@kunal_dutt/token-lens 0.2.23 → 0.2.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/check-agent.mjs +22 -22
- package/gateway.mjs +58 -58
- package/package.json +1 -1
- package/sdk.mjs +13 -12
- package/server.mjs +44 -44
- package/token-lens.mcpb +0 -0
- package/types/src/core/advice.d.ts +1 -1
- package/types/src/core/budget.d.ts +5 -0
package/check-agent.mjs
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import { createRequire as __cr } from 'node:module'; const require = __cr(import.meta.url);
|
|
2
|
-
import{execFileSync as
|
|
2
|
+
import{execFileSync as Pi}from"node:child_process";import{existsSync as Ii,readFileSync as Mi,writeFileSync as ji}from"node:fs";import{join as _a}from"node:path";var A="<|endoftext|>",j="<|fim_prefix|>",x="<|fim_middle|>",D="<|fim_suffix|>",F="<|im_start|>",G="<|im_end|>",N="<|im_sep|>",L="<|endofprompt|>",he="<|startoftext|>",$="<|start|>",Y="<|end|>",X="<|message|>",Q="<|channel|>",Z="<|return|>",ee="<|constrain|>",ae="<|call|>";var je=e=>e<=127;function be(e){if(e.length===0)return!1;let a=e.charCodeAt(e.length-1);return a>=55296&&a<=56319}function Ra(e){let a=0;for(;a<e.length;){let i=e[a],n=0,r=0;if(i<=127)n=1,r=i;else if((i&224)===192){if(n=2,r=i&31,i<=193)return!1}else if((i&240)===224)n=3,r=i&15;else if((i&248)===240){if(n=4,r=i&7,i>244)return!1}else return!1;if(a+n>e.length)return!1;for(let t=1;t<n;t++){let o=e[a+t];if(o===void 0||(o&192)!==128)return!1;r=r<<6|o&63}if(n===2&&r<128||n===3&&r<2048||n===4&&r<65536||r>=55296&&r<=57343||r>1114111)return!1;a+=n}return!0}var za=new TextDecoder("utf8",{fatal:!1});function xe(e){if(Ra(e))return za.decode(e)}function De(e,a){let i=Math.min(e.length,a.length);for(let n=0;n<i;n++)if(e[n]!==a[n])return e[n]-a[n];return e.length-a.length}function Ne(e){let a=0;return e.forEach(i=>{a=Math.max(a,i)}),a}function ke(e){return e.replace(/[$()*+.?[\\\]^{|}]/g,"\\$&")}function fe(e){let a=[...e].map(ke).join("|");return new RegExp(`(${a})`)}var se=new Uint8Array(0),U=new TextDecoder("utf8"),Oe=class{mergeableBytePairRankCount;bytePairRankDecoder;bytePairNonUtfRankDecoder=new Map;bytePairNonUtfSortedEncoder;bytePairStringRankEncoder;tokenSplitRegex;specialTokensEncoder;specialTokensDecoder;specialTokenPatternRegex;textEncoder=new TextEncoder;mergeCache;mergeCacheSize;constructor({bytePairRankDecoder:e,specialTokensEncoder:a,tokenSplitRegex:i,mergeCacheSize:n=1e5}){this.bytePairRankDecoder=e,this.bytePairStringRankEncoder=new Map,this.mergeCacheSize=n,n>0&&(this.mergeCache=new Map),this.mergeableBytePairRankCount=Object.keys(e).length;let r=[];e.forEach((o,s)=>{if(typeof o=="string"){this.bytePairStringRankEncoder.set(o,s);return}let l=new Uint8Array(o);r.push([l,s]),this.bytePairNonUtfRankDecoder.set(s,l)}),this.bytePairNonUtfSortedEncoder=r.sort((o,s)=>De(o[0],s[0])),this.specialTokensEncoder=a??new Map,this.specialTokensDecoder=a?new Map([...a].map(([o,s])=>[s,o])):new Map,this.tokenSplitRegex=i;let t=[...this.specialTokensEncoder.keys()].map(ke).join("|");try{this.specialTokenPatternRegex=new RegExp(t,"y")}catch{throw new Error("Invalid regular expression pattern.")}}setMergeCacheSize(e){this.mergeCacheSize===0&&e>0&&(this.mergeCache=new Map),this.mergeCacheSize=e,e===0&&(this.mergeCache=void 0)}clearMergeCache(){this.mergeCache?.clear()}*encodeNativeGenerator(e,a){let i=0,n=0;for(;;){let r=this.findNextSpecialToken(e,a,i),t=r?.[0],o=t??e.length,s=i===0&&o===e.length?e:e.slice(i,o);for(let[l]of s.matchAll(this.tokenSplitRegex)){let c=this.getBpeRankFromString(l);if(c!==void 0){n=1,yield[c];continue}let d=this.bytePairEncode(l);n=d.length,yield d}if(t!==void 0){let l=r[1],c=this.specialTokensEncoder.get(l);if(c===void 0)throw new Error(`Special token "${l}" is not in the special token encoder.`);yield[c],i=t+l.length,n=1}else break}return n}encodeNative(e,a){let i=0,n=[];for(;;){let r=this.findNextSpecialToken(e,a,i),t=r?.[0],o=t??e.length,s=i===0&&o===e.length?e:e.slice(i,o);for(let[l]of s.matchAll(this.tokenSplitRegex)){let c=this.getBpeRankFromString(l);if(c!==void 0){n.push(c);continue}let d=this.bytePairEncode(l);n.push(...d)}if(t!==void 0){let l=r[1],c=this.specialTokensEncoder.get(l);if(c===void 0)throw new Error(`Special token "${l}" is not in the special token encoder.`);n.push(c),i=t+l.length}else break}return n}countNative(e,a){let i=0,n=0;for(;;){let r=this.findNextSpecialToken(e,a,i),t=r?.[0],o=t??e.length,s=i===0&&o===e.length?e:e.slice(i,o);for(let[l]of s.matchAll(this.tokenSplitRegex)){if(this.getBpeRankFromString(l)!==void 0){n++;continue}let c=this.bytePairEncode(l);n+=c.length}if(t!==void 0){let l=r[1];if(this.specialTokensEncoder.get(l)===void 0)throw new Error(`Special token "${l}" is not in the special token encoder.`);n++,i=t+l.length}else break}return n}*decodeNativeGenerator(e){for(let a of e){let i=this.tryDecodeToken(a);i&&(yield i)}}decodeNative(e){let a="",i=se;for(let n of e){let r=this.tryDecodeToken(n);if(r===void 0)throw new Error(`Token ${n} is not in the byte pair encoder.`);if(typeof r=="string")i!==se&&(a+=U.decode(i,{stream:!0}),i=se),a+=r;else{let t=new Uint8Array(i.length+r.length);t.set(i),t.set(r,i.length),i=t}}return i!==se&&(a+=U.decode(i,{stream:!0})),a}async*decodeNativeAsyncIterable(e){for await(let a of e){let i=this.tryDecodeToken(a);i&&(yield i)}}getBpeRankFromString(e){return this.bytePairStringRankEncoder.get(e)}getBpeRankFromStringOrThrow(e){let a=this.getBpeRankFromString(e);if(a===void 0)throw new Error(`The byte-pair encoding does not contain a value for: ${e}`);return a}getBpeRankFromBytes(e){let a=xe(e);if(a!==void 0)return this.getBpeRankFromString(a);let i=this.binarySearch(e);if(i!==-1)return this.bytePairNonUtfSortedEncoder[i][1]}getBpeRankFromBytesOrThrow(e){let a=this.getBpeRankFromBytes(e);if(a===void 0)throw new Error(`The byte-pair encoding does not contain a value for: ${e.toString()}`);return a}binarySearch(e){let a=0,i=this.bytePairNonUtfSortedEncoder.length-1;for(;a<=i;){let n=a+i>>>1,r=this.bytePairNonUtfSortedEncoder[n][0],t=0,o=Math.min(r.length,e.length);for(let s=0;s<o&&(t=r[s]-e[s],t===0);s++);if(t===0&&(t=r.length-e.length),t===0)return n;t<0?a=n+1:i=n-1}return-1}findNextSpecialToken(e,a,i){let n=i;for(;;){this.specialTokenPatternRegex.lastIndex=n;let r=this.specialTokenPatternRegex.exec(e);if(!r)return;let t=r[0];if(a?.has(t))return[r.index+n,t];n=r.index+n+1}}tryDecodeToken(e){let a=this.bytePairRankDecoder[e];if(typeof a=="string")return a;if(typeof a=="object"){let i=this.bytePairNonUtfRankDecoder.get(e);if(i)return i}return this.specialTokensDecoder.get(e)}addToMergeCache(e,a){if(this.mergeCache){if(this.mergeCache.size>=this.mergeCacheSize){let i=this.mergeCache.keys().next().value;this.mergeCache.delete(i)}this.mergeCache.set(e,a)}}bytePairEncode(e){if(e.length===1&&je(e.codePointAt(0)))return[this.getBpeRankFromStringOrThrow(e)];if(this.mergeCache?.has(e)){let n=this.mergeCache.get(e);return this.mergeCache.delete(e),this.mergeCache.set(e,n),n}let a=this.textEncoder.encode(e),i=this.bytePairMerge(a);return this.addToMergeCache(e,i),i}bytePairMerge(e){let a=[],i=[],n=(t,o=a[t],s=a[t+2])=>{if(s===void 0)return Number.POSITIVE_INFINITY;let l=e.subarray(o,s);return this.getBpeRankFromBytes(l)??Number.POSITIVE_INFINITY};for(let t=0;t<=e.length;t++)a.push(t),t<e.length-1?i.push(n(t,t,t+2)):i.push(Number.POSITIVE_INFINITY);for(;a.length>1;){let t=Number.POSITIVE_INFINITY,o=-1;for(let s=0;s<i.length-1;s++){let l=i[s];l<t&&(t=l,o=s)}if(t===Number.POSITIVE_INFINITY||o===-1)break;a.splice(o+1,1),i.splice(o,1),i[o]=n(o),o>0&&(i[o-1]=n(o-1))}let r=[];for(let t=0;t<a.length-1;t++){let o=a[t],s=a[t+1],l=this.getBpeRankFromBytesOrThrow(e.subarray(o,s));r.push(l)}return r}};function Pa(e,a){let i=0;if(e.role&&(i+=a(e.role)),e.content&&(i+=a(e.content)),e.name&&(i+=a(e.name)+1),e.function_call){let{name:n,arguments:r}=e.function_call;n&&(i+=a(n)),r&&(i+=a(r)),i+=3}return i+=3,e.role==="function"&&(i-=2),i}function Be(e,a,i){if(!e.properties)return"";let n=[],r=new Set(e.required??[]),t=" ".repeat(a);for(let[o,s]of Object.entries(e.properties)){s.description&&a<2&&n.push(`${t}// ${s.description}`);let l=r.has(o),c=i(s,a);n.push(`${t}${o}${l?"":"?"}: ${c},`)}return n.join(`
|
|
3
3
|
`)}function ve(e,a){switch(e.type){case"string":return e.enum?.map(i=>JSON.stringify(i)).join(" | ")??"string";case"integer":case"number":return e.enum?.map(i=>`${i}`).join(" | ")??"number";case"boolean":return"boolean";case"null":return"null";case"array":return e.items?`${ve(e.items,a)}[]`:"any[]";case"object":return`{
|
|
4
4
|
${Be(e,a+2,ve)}
|
|
5
|
-
${" ".repeat(a)}}`;default:return"any"}}function
|
|
6
|
-
`)}function
|
|
5
|
+
${" ".repeat(a)}}`;default:return"any"}}function Ia(e){let a=["namespace functions {",""];for(let i of e){i.description&&a.push(`// ${i.description}`);let{parameters:n}=i,r=n?.properties;if(!n||!r||Object.keys(r).length===0)a.push(`type ${i.name} = () => any;`);else{a.push(`type ${i.name} = (_: {`);let t=Be(n,0,ve);t.length>0&&a.push(t),a.push("}) => any;")}a.push("")}return a.push("} // namespace functions"),a.join(`
|
|
6
|
+
`)}function Ma(e,a){let i=a(Ia(e));return i+=9,i}function ja(e,a,i){return!a||i||e.role!=="system"||!e.content||e.content.endsWith(`
|
|
7
7
|
`)?e:{...e,content:`${e.content}
|
|
8
|
-
`}}function Fe(e,a){let{messages:i,functions:n,function_call:r}=e,t=!!(n&&n.length>0),o=!1,s=0;for(let l of i){let c=
|
|
8
|
+
`}}function Fe(e,a){let{messages:i,functions:n,function_call:r}=e,t=!!(n&&n.length>0),o=!1,s=0;for(let l of i){let c=ja(l,t,o);(c!==l&&l.role==="system"||l.role==="system"&&t&&!o)&&(o=!0),s+=Pa(c,a)}return s+=3,t&&n&&(s+=Ma(n,a),i.some(l=>l.role==="system")&&(s-=4)),r&&r!=="auto"&&(r==="none"?s+=1:typeof r=="object"&&r.name&&(s+=a(r.name)+4)),s}var Ge=["chat-latest","chatgpt-4o-latest","codex-mini-latest","computer-use-preview","computer-use-preview-2025-03-11","daybreak-blue-latest","daybreak-red-latest","gpt-3.5","gpt-3.5-0301","gpt-3.5-turbo","gpt-3.5-turbo-0125","gpt-3.5-turbo-0613","gpt-3.5-turbo-1106","gpt-3.5-turbo-16k-0613","gpt-3.5-turbo-instruct","gpt-4","gpt-4-0125-preview","gpt-4-0314","gpt-4-0613","gpt-4-1106-preview","gpt-4-1106-vision-preview","gpt-4-32k","gpt-4-turbo","gpt-4-turbo-2024-04-09","gpt-4-turbo-preview","gpt-4.1","gpt-4.1-2025-04-14","gpt-4.1-mini","gpt-4.1-mini-2025-04-14","gpt-4.1-nano","gpt-4.1-nano-2025-04-14","gpt-4.5-preview","gpt-4.5-preview-2025-02-27","gpt-4o","gpt-4o-2024-05-13","gpt-4o-2024-08-06","gpt-4o-2024-11-20","gpt-4o-audio-preview","gpt-4o-audio-preview-2024-10-01","gpt-4o-audio-preview-2024-12-17","gpt-4o-audio-preview-2025-06-03","gpt-4o-mini","gpt-4o-mini-2024-07-18","gpt-4o-mini-audio-preview","gpt-4o-mini-audio-preview-2024-12-17","gpt-4o-mini-search-preview","gpt-4o-mini-search-preview-2025-03-11","gpt-4o-search-preview","gpt-4o-search-preview-2025-03-11","gpt-5","gpt-5-2025-08-07","gpt-5-chat-latest","gpt-5-codex","gpt-5-mini","gpt-5-mini-2025-08-07","gpt-5-nano","gpt-5-nano-2025-08-07","gpt-5-pro","gpt-5-pro-2025-10-06","gpt-5.1","gpt-5.1-2025-11-13","gpt-5.1-chat-latest","gpt-5.1-codex","gpt-5.1-codex-max","gpt-5.1-codex-mini","gpt-5.2","gpt-5.2-2025-12-11","gpt-5.2-chat-latest","gpt-5.2-codex","gpt-5.2-pro","gpt-5.2-pro-2025-12-11","gpt-5.3-chat-latest","gpt-5.3-codex","gpt-5.4","gpt-5.4-2026-03-05","gpt-5.4-mini","gpt-5.4-mini-2026-03-17","gpt-5.4-nano","gpt-5.4-nano-2026-03-17","gpt-5.4-pro","gpt-5.4-pro-2026-03-05","gpt-5.5","gpt-5.5-2026-04-23","gpt-5.5-pro","gpt-5.5-pro-2026-04-23","gpt-5.6-cyber","gpt-5.6-luna","gpt-5.6-sol","gpt-5.6-terra","gpt-audio","gpt-audio-1.5","gpt-audio-2025-08-28","gpt-audio-mini","gpt-audio-mini-2025-10-06","gpt-audio-mini-2025-12-15","gpt-image-1","gpt-oss-120b","gpt-oss-20b","o1","o1-2024-12-17","o1-mini","o1-mini-2024-09-12","o1-preview","o1-preview-2024-09-12","o1-pro","o1-pro-2025-03-19","o3","o3-2025-04-16","o3-deep-research","o3-deep-research-2025-06-26","o3-mini","o3-mini-2025-01-31","o3-pro","o3-pro-2025-06-10","o4-mini","o4-mini-2025-04-16","o4-mini-deep-research","o4-mini-deep-research-2025-06-26"];var qe=Object.defineProperty,He=(e,a)=>{let i={};for(var n in e)qe(i,n,{get:e[n],enumerable:!0});return a||qe(i,Symbol.toStringTag,{value:"Module"}),i};var Ue=He({cl100k_base:()=>Oa,gpt2:()=>xa,o200k_base:()=>Ba,o200k_harmony:()=>Fa,p50k_base:()=>Da,p50k_edit:()=>La,r50k_base:()=>Na}),xa=["gpt2","gpt-2"],Da=["text-davinci-002","text-davinci-003","code-davinci-001","code-davinci-002","davinci-codex","code-cushman-001","code-cushman-002","cushman-codex"],Na=["text-ada-001","text-babbage-001","text-curie-001","text-davinci-001","ada","babbage","curie","davinci","code-search-ada-code-001","code-search-ada-text-001","text-similarity-ada-001","text-search-ada-doc-001","text-search-ada-query-001","text-similarity-babbage-001","text-search-babbage-doc-001","text-search-babbage-query-001","code-search-babbage-code-001","code-search-babbage-text-001","text-similarity-curie-001","text-search-curie-doc-001","text-search-curie-query-001","text-similarity-davinci-001","text-search-davinci-doc-001","text-search-davinci-query-001"],La=["code-davinci-edit-001","text-davinci-edit-001"],Oa=["gpt-3.5","gpt-3.5-0301","gpt-3.5-turbo","gpt-3.5-turbo-0125","gpt-3.5-turbo-0613","gpt-3.5-turbo-1106","gpt-3.5-turbo-16k-0613","gpt-3.5-turbo-instruct","gpt-4","gpt-4-0125-preview","gpt-4-0314","gpt-4-0613","gpt-4-1106-preview","gpt-4-1106-vision-preview","gpt-4-32k","gpt-4-turbo","gpt-4-turbo-2024-04-09","gpt-4-turbo-preview","text-embedding-3-large","text-embedding-3-small","text-embedding-ada-002","babbage-002","davinci-002"],Ba=[],Fa=["gpt-oss-20b","gpt-oss-120b"];var Ve=Object.fromEntries(Object.entries(Ue).flatMap(([e,a])=>a.map(i=>[i,e]))),Ga={messageSeparator:`
|
|
9
9
|
`,roleSeparator:`
|
|
10
|
-
`},Ga={messageSeparator:"",roleSeparator:N},We=Object.fromEntries(Ge.flatMap(e=>e.startsWith("gpt-3.5")?[[e,Fa]]:[[e,Ga]]));var V=new RegExp("'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)|\\s+","gu"),Je=String.raw`'(?:[sS]|[dD]|[mM]|[tT]|[lL][lL]|[vV][eE]|[rR][eE])`,Ke=String.raw`(?:${Je})?`,qa=String.raw`${Je}|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s+$|\s*[\r\n]|\s+(?!\S)|\s`,$e=new RegExp(qa,"gu"),Ha=String.raw`[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+${Ke}|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*${Ke}|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n/]*|\s*[\r\n]+|\s+(?!\S)|\s+`,oe=new RegExp(Ha,"gu");function Ye(e){return{tokenSplitRegex:$e,bytePairRankDecoder:e,specialTokensEncoder:new Map([[A,100257],[j,100258],[x,100259],[D,100260],[F,100264],[G,100265],[N,100266],[L,100276]])}}var Ua=[[A,199999],[j,2e5],[x,200001],[D,200002],[F,200003],[G,200004],[N,200005],[L,200006]],Va=()=>new Map(Ua);function Xe(e){return{tokenSplitRegex:oe,bytePairRankDecoder:e,specialTokensEncoder:Va()}}var Wa=200013,Ka=201088,Ja=[[he,199998],[A,199999],["<|reserved_200000|>",2e5],["<|reserved_200001|>",200001],[Z,200002],[ee,200003],["<|reserved_200004|>",200004],[Q,200005],[$,200006],[Y,200007],[X,200008],["<|reserved_200009|>",200009],["<|reserved_200010|>",200010],["<|reserved_200011|>",200011],[ae,200012]];function Qe(e){let a=new Map(Ja);for(let i=Wa;i<Ka;i+=1)a.set(`<|reserved_${i}|>`,i);return a.set(L,200018),{tokenSplitRegex:oe,bytePairRankDecoder:e,specialTokensEncoder:a,chatFormatter:"harmony"}}function Ze(e){return{expectedVocabularySize:50281,tokenSplitRegex:V,bytePairRankDecoder:e,specialTokensEncoder:new Map([[A,50256]])}}function ea(e){return{tokenSplitRegex:V,bytePairRankDecoder:e,specialTokensEncoder:new Map([[A,50256],[j,50281],[x,50282],[D,50283]])}}function aa(e){return{expectedVocabularySize:50257,tokenSplitRegex:V,bytePairRankDecoder:e,specialTokensEncoder:new Map([[A,50256]])}}function ye(e,a){let i=a(e);switch(e.toLowerCase()){case"gpt2":case"r50k_base":return aa(i);case"p50k_base":return Ze(i);case"p50k_edit":return ea(i);case"cl100k_base":return Ye(i);case"o200k_base":return Xe(i);case"o200k_harmony":return Qe(i);default:throw new Error(`Unknown encoding name: ${e}`)}}var ia=class we{static EndOfPrompt=L;static EndOfText=A;static FimMiddle=x;static FimPrefix=j;static FimSuffix=D;modelName;modelSpec;bytePairEncodingCoreProcessor;specialTokensEncoder;specialTokensSet;allSpecialTokenRegex;defaultSpecialTokenConfig;chatFormatter;countChatCompletionTokens;vocabularySize;constructor({bytePairRankDecoder:a,specialTokensEncoder:i,expectedVocabularySize:n,modelName:r,modelSpec:t,chatFormatter:o,...s}){this.specialTokensEncoder=i,this.specialTokensSet=new Set(this.specialTokensEncoder.keys()),this.allSpecialTokenRegex=fe(this.specialTokensSet),this.bytePairEncodingCoreProcessor=new Oe({bytePairRankDecoder:a,specialTokensEncoder:i,...s}),this.defaultSpecialTokenConfig=this.processSpecialTokens();let l=Math.max(a.length-1,Ne(i));if(this.vocabularySize=this.bytePairEncodingCoreProcessor.mergeableBytePairRankCount+i.size,n!==void 0){if(this.vocabularySize!==n)throw new Error("The number of mergeable tokens and special tokens must be equal to expectedVocabularySize.");if(l!==n-1)throw new Error(`The model encodings are invalid. The maximum token value must be equal to expectedVocabularySize - 1. Currently ${l}, expected ${n-1}`)}this.encode=this.encode.bind(this),this.decode=this.decode.bind(this),this.encodeGenerator=this.encodeGenerator.bind(this),this.decodeGenerator=this.decodeGenerator.bind(this),this.decodeAsyncGenerator=this.decodeAsyncGenerator.bind(this),this.decodeAsync=this.decodeAsync.bind(this),this.isWithinTokenLimit=this.isWithinTokenLimit.bind(this),this.encodeChat=this.encodeChat.bind(this),this.encodeChatGenerator=this.encodeChatGenerator.bind(this),this.countTokens=this.countTokens.bind(this),this.setMergeCacheSize=this.setMergeCacheSize.bind(this),this.clearMergeCache=this.clearMergeCache.bind(this),this.estimateCost=this.estimateCost.bind(this),t?.supported_features?.includes("function_calling")&&(this.countChatCompletionTokens=this.countChatCompletionTokensInternal.bind(this)),this.modelName=r,this.modelSpec=t,this.chatFormatter=o??"chatml"}*encodeHarmonyChatGenerator(a,i){let n=this.specialTokensEncoder.get($),r=this.specialTokensEncoder.get(X),t=this.specialTokensEncoder.get(Y),o=this.specialTokensEncoder.get(Z),s=this.specialTokensEncoder.get(ae),l=this.specialTokensEncoder.get(Q),c=this.specialTokensEncoder.get(ee);if(n===void 0||r===void 0||t===void 0||o===void 0||s===void 0||l===void 0||c===void 0)throw new Error("Harmony chat format requires dedicated special tokens.");let d=p=>p.length>0?this.encode(p):[],u=p=>{switch(p){case"<|return|>":return o;case"<|call|>":return s;default:return t}};for(let p of a){if(p.content===void 0)throw new Error("Content must be defined for all messages.");let y=p.name??p.role??"user";yield[n],yield d(y);let h=p.recipient&&(p.recipientPlacement==="role"||!p.channel),f=p.recipient&&!h;h&&(yield d(` to=${p.recipient}`)),p.channel&&(yield[l],yield d(p.channel),f&&(yield d(` to=${p.recipient}`))),p.constraint&&(yield[c],yield d(p.constraint)),yield[r],yield*this.encodeGenerator(p.content,i),yield[u(p.terminator)]}let g=i?.primeWithAssistantResponse??"assistant";g.length>0&&(yield[n],yield d(g))}static getEncodingApi(a,i){let n=ye(a,i);return new we(n)}static getEncodingApiForModel(a,i,n){let r=Ve[a]??"o200k_base",t=ye(r,i);return new we({...t,modelName:a,modelSpec:n})}processSpecialTokens({allowedSpecial:a,disallowedSpecial:i}={}){let n;if(a==="all"||a?.has("all")){a=new Set(this.specialTokensSet);let r=a;if(i==="all")throw new Error('allowedSpecial and disallowedSpecial cannot both be set to "all".');typeof i=="object"?i.forEach(t=>r.delete(t)):i=new Set}if(!i||i==="all"||i.has("all")){i=new Set(this.specialTokensSet);let r=i;a?.size?(a.forEach(t=>r.delete(t)),i.forEach(t=>a.delete(t)),i.size>0&&(n=fe(i))):n=this.allSpecialTokenRegex}return{allowedSpecial:a,regexPattern:n}}encodeGenerator(a,i){let n=i?this.processSpecialTokens(i):this.defaultSpecialTokenConfig;if(n.regexPattern){let r=a.match(n.regexPattern);if(r!==null)throw new Error(`Disallowed special token found: ${r[0]}`)}return this.bytePairEncodingCoreProcessor.encodeNativeGenerator(a,n.allowedSpecial)}encode(a,i){let n=i?this.processSpecialTokens(i):this.defaultSpecialTokenConfig;if(n.regexPattern){let r=a.match(n.regexPattern);if(r!==null)throw new Error(`Disallowed special token found: ${r[0]}`)}return this.bytePairEncodingCoreProcessor.encodeNative(a,n.allowedSpecial)}*encodeChatGenerator(a,i=this.modelName,n){if(!i)throw new Error("Model name must be provided either during initialization or passed in to the method.");let r=We[i];if(!r)throw new Error(`Model '${i}' does not support chat.`);if(this.chatFormatter==="harmony"){yield*this.encodeHarmonyChatGenerator(a,n);return}let t=this.specialTokensEncoder.get(F),o=this.specialTokensEncoder.get(G);if(t===void 0||o===void 0)throw new Error(`Model '${i}' does not support chat.`);let s=new Set([N]),{messageSeparator:l,roleSeparator:c}=r,d=l.length>0?this.encode(l):[],u=c.length>0?this.encode(c,{allowedSpecial:s}):[],g=new Map;for(let{role:y="system",name:h=y,content:f}of a){if(f===void 0)throw new Error("Content must be defined for all messages.");yield[t];let _=g.get(h)??this.encode(h);g.set(h,_),yield _,u.length>0&&(yield u),yield*this.encodeGenerator(f,n),yield[o],yield d}let p=n?.primeWithAssistantResponse??"assistant";p.length>0&&(yield[t],yield*this.encodeGenerator(p,n)),u.length>0&&(yield u)}encodeChat(a,i=this.modelName,n){return[...this.encodeChatGenerator(a,i,n)].flat()}isWithinTokenLimit(a,i,n){let r=typeof a=="string"?this.encodeGenerator(a,n):this.encodeChatGenerator(a,void 0,n),t=0;for(let o of r)if(t+=o.length,t>i)return!1;return t}countTokens(a,i){if(typeof a=="string"){let t=i?this.processSpecialTokens(i):this.defaultSpecialTokenConfig;if(t.regexPattern){let o=a.match(t.regexPattern);if(o!==null)throw new Error(`Disallowed special token found: ${o[0]}`)}return this.bytePairEncodingCoreProcessor.countNative(a,t.allowedSpecial)}let n=this.encodeChatGenerator(a,void 0,i),r=0;for(let t of n)r+=t.length;return r}countStringTokens(a){return a?this.bytePairEncodingCoreProcessor.countNative(a):0}countChatCompletionTokensInternal(a){return Fe(a,i=>this.countStringTokens(i))}setMergeCacheSize(a){this.bytePairEncodingCoreProcessor.setMergeCacheSize(a)}clearMergeCache(){this.bytePairEncodingCoreProcessor.clearMergeCache()}decode(a){return this.bytePairEncodingCoreProcessor.decodeNative(a)}*decodeGenerator(a){let i=this.bytePairEncodingCoreProcessor.decodeNativeGenerator(a),n="";for(let r of i)n+=typeof r=="string"?r:U.decode(r,{stream:!0}),!(n.length===0||be(n))&&(yield n,n="");n.length>0&&(yield n)}async*decodeAsyncGenerator(a){let i=this.bytePairEncodingCoreProcessor.decodeNativeAsyncIterable(a),n="";for await(let r of i)n+=typeof r=="string"?r:U.decode(r,{stream:!0}),!(n.length===0||be(n))&&(yield n,n="");n.length>0&&(yield n)}async decodeAsync(a){let i=this.bytePairEncodingCoreProcessor.decodeNativeAsyncIterable(a),n="";for await(let r of i)n+=typeof r=="string"?r:U.decode(r,{stream:!0});return n}estimateCost(a,i=this.modelSpec){if(!i)throw new Error("Model spec must be provided either during initialization or passed in to the method.");if(!i.price_data)throw new Error(`No cost information available for model: ${i.name}`);let n=i.price_data,r={},t=a/1e6;return n.main&&(r.main={input:n.main.input&&n.main.input*t,output:n.main.output&&n.main.output*t,cached_input:n.main.cached_input&&n.main.cached_input*t,cached_output:n.main.cached_output&&n.main.cached_output*t}),n.batch&&(r.batch={input:n.batch.input&&n.batch.input*t,output:n.batch.output&&n.batch.output*t,cached_input:n.batch.cached_input&&n.batch.cached_input*t,cached_output:n.batch.cached_output&&n.batch.cached_output*t}),r}};var na=["!",'"',"#","$","%","&","'","(",")","*","+",",","-",".","/","0","1","2","3","4","5","6","7","8","9",":",";","<","=",">","?","@","A","B","C","D","E","F","G","H","I","J","K","L","M","N","O","P","Q","R","S","T","U","V","W","X","Y","Z","[","\\","]","^","_","`","a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z","{","|","}","~",[161],[162],[163],[164],[165],[166],[167],[168],[169],[170],[171],[172],[174],[175],[176],[177],[178],[179],[180],[181],[182],[183],[184],[185],[186],[187],[188],[189],[190],[191],[192],[193],[194],[195],[196],[197],[198],[199],[200],[201],[202],[203],[204],[205],[206],[207],[208],[209],[210],[211],[212],[213],[214],[215],[216],[217],[218],[219],[220],[221],[222],[223],[224],[225],[226],[227],[228],[229],[230],[231],[232],[233],[234],[235],[236],[237],[238],[239],[240],[241],[242],[243],[244],[245],[246],[247],[248],[249],[250],[251],[252],[253],[254],[255],"\0","","","","","","","\x07","\b"," ",`
|
|
10
|
+
`},qa={messageSeparator:"",roleSeparator:N},We=Object.fromEntries(Ge.flatMap(e=>e.startsWith("gpt-3.5")?[[e,Ga]]:[[e,qa]]));var V=new RegExp("'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)|\\s+","gu"),Je=String.raw`'(?:[sS]|[dD]|[mM]|[tT]|[lL][lL]|[vV][eE]|[rR][eE])`,Ke=String.raw`(?:${Je})?`,Ha=String.raw`${Je}|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s+$|\s*[\r\n]|\s+(?!\S)|\s`,$e=new RegExp(Ha,"gu"),Ua=String.raw`[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+${Ke}|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*${Ke}|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n/]*|\s*[\r\n]+|\s+(?!\S)|\s+`,oe=new RegExp(Ua,"gu");function Ye(e){return{tokenSplitRegex:$e,bytePairRankDecoder:e,specialTokensEncoder:new Map([[A,100257],[j,100258],[x,100259],[D,100260],[F,100264],[G,100265],[N,100266],[L,100276]])}}var Va=[[A,199999],[j,2e5],[x,200001],[D,200002],[F,200003],[G,200004],[N,200005],[L,200006]],Wa=()=>new Map(Va);function Xe(e){return{tokenSplitRegex:oe,bytePairRankDecoder:e,specialTokensEncoder:Wa()}}var Ka=200013,Ja=201088,$a=[[he,199998],[A,199999],["<|reserved_200000|>",2e5],["<|reserved_200001|>",200001],[Z,200002],[ee,200003],["<|reserved_200004|>",200004],[Q,200005],[$,200006],[Y,200007],[X,200008],["<|reserved_200009|>",200009],["<|reserved_200010|>",200010],["<|reserved_200011|>",200011],[ae,200012]];function Qe(e){let a=new Map($a);for(let i=Ka;i<Ja;i+=1)a.set(`<|reserved_${i}|>`,i);return a.set(L,200018),{tokenSplitRegex:oe,bytePairRankDecoder:e,specialTokensEncoder:a,chatFormatter:"harmony"}}function Ze(e){return{expectedVocabularySize:50281,tokenSplitRegex:V,bytePairRankDecoder:e,specialTokensEncoder:new Map([[A,50256]])}}function ea(e){return{tokenSplitRegex:V,bytePairRankDecoder:e,specialTokensEncoder:new Map([[A,50256],[j,50281],[x,50282],[D,50283]])}}function aa(e){return{expectedVocabularySize:50257,tokenSplitRegex:V,bytePairRankDecoder:e,specialTokensEncoder:new Map([[A,50256]])}}function ye(e,a){let i=a(e);switch(e.toLowerCase()){case"gpt2":case"r50k_base":return aa(i);case"p50k_base":return Ze(i);case"p50k_edit":return ea(i);case"cl100k_base":return Ye(i);case"o200k_base":return Xe(i);case"o200k_harmony":return Qe(i);default:throw new Error(`Unknown encoding name: ${e}`)}}var ia=class we{static EndOfPrompt=L;static EndOfText=A;static FimMiddle=x;static FimPrefix=j;static FimSuffix=D;modelName;modelSpec;bytePairEncodingCoreProcessor;specialTokensEncoder;specialTokensSet;allSpecialTokenRegex;defaultSpecialTokenConfig;chatFormatter;countChatCompletionTokens;vocabularySize;constructor({bytePairRankDecoder:a,specialTokensEncoder:i,expectedVocabularySize:n,modelName:r,modelSpec:t,chatFormatter:o,...s}){this.specialTokensEncoder=i,this.specialTokensSet=new Set(this.specialTokensEncoder.keys()),this.allSpecialTokenRegex=fe(this.specialTokensSet),this.bytePairEncodingCoreProcessor=new Oe({bytePairRankDecoder:a,specialTokensEncoder:i,...s}),this.defaultSpecialTokenConfig=this.processSpecialTokens();let l=Math.max(a.length-1,Ne(i));if(this.vocabularySize=this.bytePairEncodingCoreProcessor.mergeableBytePairRankCount+i.size,n!==void 0){if(this.vocabularySize!==n)throw new Error("The number of mergeable tokens and special tokens must be equal to expectedVocabularySize.");if(l!==n-1)throw new Error(`The model encodings are invalid. The maximum token value must be equal to expectedVocabularySize - 1. Currently ${l}, expected ${n-1}`)}this.encode=this.encode.bind(this),this.decode=this.decode.bind(this),this.encodeGenerator=this.encodeGenerator.bind(this),this.decodeGenerator=this.decodeGenerator.bind(this),this.decodeAsyncGenerator=this.decodeAsyncGenerator.bind(this),this.decodeAsync=this.decodeAsync.bind(this),this.isWithinTokenLimit=this.isWithinTokenLimit.bind(this),this.encodeChat=this.encodeChat.bind(this),this.encodeChatGenerator=this.encodeChatGenerator.bind(this),this.countTokens=this.countTokens.bind(this),this.setMergeCacheSize=this.setMergeCacheSize.bind(this),this.clearMergeCache=this.clearMergeCache.bind(this),this.estimateCost=this.estimateCost.bind(this),t?.supported_features?.includes("function_calling")&&(this.countChatCompletionTokens=this.countChatCompletionTokensInternal.bind(this)),this.modelName=r,this.modelSpec=t,this.chatFormatter=o??"chatml"}*encodeHarmonyChatGenerator(a,i){let n=this.specialTokensEncoder.get($),r=this.specialTokensEncoder.get(X),t=this.specialTokensEncoder.get(Y),o=this.specialTokensEncoder.get(Z),s=this.specialTokensEncoder.get(ae),l=this.specialTokensEncoder.get(Q),c=this.specialTokensEncoder.get(ee);if(n===void 0||r===void 0||t===void 0||o===void 0||s===void 0||l===void 0||c===void 0)throw new Error("Harmony chat format requires dedicated special tokens.");let d=p=>p.length>0?this.encode(p):[],u=p=>{switch(p){case"<|return|>":return o;case"<|call|>":return s;default:return t}};for(let p of a){if(p.content===void 0)throw new Error("Content must be defined for all messages.");let y=p.name??p.role??"user";yield[n],yield d(y);let h=p.recipient&&(p.recipientPlacement==="role"||!p.channel),f=p.recipient&&!h;h&&(yield d(` to=${p.recipient}`)),p.channel&&(yield[l],yield d(p.channel),f&&(yield d(` to=${p.recipient}`))),p.constraint&&(yield[c],yield d(p.constraint)),yield[r],yield*this.encodeGenerator(p.content,i),yield[u(p.terminator)]}let g=i?.primeWithAssistantResponse??"assistant";g.length>0&&(yield[n],yield d(g))}static getEncodingApi(a,i){let n=ye(a,i);return new we(n)}static getEncodingApiForModel(a,i,n){let r=Ve[a]??"o200k_base",t=ye(r,i);return new we({...t,modelName:a,modelSpec:n})}processSpecialTokens({allowedSpecial:a,disallowedSpecial:i}={}){let n;if(a==="all"||a?.has("all")){a=new Set(this.specialTokensSet);let r=a;if(i==="all")throw new Error('allowedSpecial and disallowedSpecial cannot both be set to "all".');typeof i=="object"?i.forEach(t=>r.delete(t)):i=new Set}if(!i||i==="all"||i.has("all")){i=new Set(this.specialTokensSet);let r=i;a?.size?(a.forEach(t=>r.delete(t)),i.forEach(t=>a.delete(t)),i.size>0&&(n=fe(i))):n=this.allSpecialTokenRegex}return{allowedSpecial:a,regexPattern:n}}encodeGenerator(a,i){let n=i?this.processSpecialTokens(i):this.defaultSpecialTokenConfig;if(n.regexPattern){let r=a.match(n.regexPattern);if(r!==null)throw new Error(`Disallowed special token found: ${r[0]}`)}return this.bytePairEncodingCoreProcessor.encodeNativeGenerator(a,n.allowedSpecial)}encode(a,i){let n=i?this.processSpecialTokens(i):this.defaultSpecialTokenConfig;if(n.regexPattern){let r=a.match(n.regexPattern);if(r!==null)throw new Error(`Disallowed special token found: ${r[0]}`)}return this.bytePairEncodingCoreProcessor.encodeNative(a,n.allowedSpecial)}*encodeChatGenerator(a,i=this.modelName,n){if(!i)throw new Error("Model name must be provided either during initialization or passed in to the method.");let r=We[i];if(!r)throw new Error(`Model '${i}' does not support chat.`);if(this.chatFormatter==="harmony"){yield*this.encodeHarmonyChatGenerator(a,n);return}let t=this.specialTokensEncoder.get(F),o=this.specialTokensEncoder.get(G);if(t===void 0||o===void 0)throw new Error(`Model '${i}' does not support chat.`);let s=new Set([N]),{messageSeparator:l,roleSeparator:c}=r,d=l.length>0?this.encode(l):[],u=c.length>0?this.encode(c,{allowedSpecial:s}):[],g=new Map;for(let{role:y="system",name:h=y,content:f}of a){if(f===void 0)throw new Error("Content must be defined for all messages.");yield[t];let _=g.get(h)??this.encode(h);g.set(h,_),yield _,u.length>0&&(yield u),yield*this.encodeGenerator(f,n),yield[o],yield d}let p=n?.primeWithAssistantResponse??"assistant";p.length>0&&(yield[t],yield*this.encodeGenerator(p,n)),u.length>0&&(yield u)}encodeChat(a,i=this.modelName,n){return[...this.encodeChatGenerator(a,i,n)].flat()}isWithinTokenLimit(a,i,n){let r=typeof a=="string"?this.encodeGenerator(a,n):this.encodeChatGenerator(a,void 0,n),t=0;for(let o of r)if(t+=o.length,t>i)return!1;return t}countTokens(a,i){if(typeof a=="string"){let t=i?this.processSpecialTokens(i):this.defaultSpecialTokenConfig;if(t.regexPattern){let o=a.match(t.regexPattern);if(o!==null)throw new Error(`Disallowed special token found: ${o[0]}`)}return this.bytePairEncodingCoreProcessor.countNative(a,t.allowedSpecial)}let n=this.encodeChatGenerator(a,void 0,i),r=0;for(let t of n)r+=t.length;return r}countStringTokens(a){return a?this.bytePairEncodingCoreProcessor.countNative(a):0}countChatCompletionTokensInternal(a){return Fe(a,i=>this.countStringTokens(i))}setMergeCacheSize(a){this.bytePairEncodingCoreProcessor.setMergeCacheSize(a)}clearMergeCache(){this.bytePairEncodingCoreProcessor.clearMergeCache()}decode(a){return this.bytePairEncodingCoreProcessor.decodeNative(a)}*decodeGenerator(a){let i=this.bytePairEncodingCoreProcessor.decodeNativeGenerator(a),n="";for(let r of i)n+=typeof r=="string"?r:U.decode(r,{stream:!0}),!(n.length===0||be(n))&&(yield n,n="");n.length>0&&(yield n)}async*decodeAsyncGenerator(a){let i=this.bytePairEncodingCoreProcessor.decodeNativeAsyncIterable(a),n="";for await(let r of i)n+=typeof r=="string"?r:U.decode(r,{stream:!0}),!(n.length===0||be(n))&&(yield n,n="");n.length>0&&(yield n)}async decodeAsync(a){let i=this.bytePairEncodingCoreProcessor.decodeNativeAsyncIterable(a),n="";for await(let r of i)n+=typeof r=="string"?r:U.decode(r,{stream:!0});return n}estimateCost(a,i=this.modelSpec){if(!i)throw new Error("Model spec must be provided either during initialization or passed in to the method.");if(!i.price_data)throw new Error(`No cost information available for model: ${i.name}`);let n=i.price_data,r={},t=a/1e6;return n.main&&(r.main={input:n.main.input&&n.main.input*t,output:n.main.output&&n.main.output*t,cached_input:n.main.cached_input&&n.main.cached_input*t,cached_output:n.main.cached_output&&n.main.cached_output*t}),n.batch&&(r.batch={input:n.batch.input&&n.batch.input*t,output:n.batch.output&&n.batch.output*t,cached_input:n.batch.cached_input&&n.batch.cached_input*t,cached_output:n.batch.cached_output&&n.batch.cached_output*t}),r}};var na=["!",'"',"#","$","%","&","'","(",")","*","+",",","-",".","/","0","1","2","3","4","5","6","7","8","9",":",";","<","=",">","?","@","A","B","C","D","E","F","G","H","I","J","K","L","M","N","O","P","Q","R","S","T","U","V","W","X","Y","Z","[","\\","]","^","_","`","a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z","{","|","}","~",[161],[162],[163],[164],[165],[166],[167],[168],[169],[170],[171],[172],[174],[175],[176],[177],[178],[179],[180],[181],[182],[183],[184],[185],[186],[187],[188],[189],[190],[191],[192],[193],[194],[195],[196],[197],[198],[199],[200],[201],[202],[203],[204],[205],[206],[207],[208],[209],[210],[211],[212],[213],[214],[215],[216],[217],[218],[219],[220],[221],[222],[223],[224],[225],[226],[227],[228],[229],[230],[231],[232],[233],[234],[235],[236],[237],[238],[239],[240],[241],[242],[243],[244],[245],[246],[247],[248],[249],[250],[251],[252],[253],[254],[255],"\0","","","","","","","\x07","\b"," ",`
|
|
11
11
|
`,"\v","\f","\r","","","","","","","","","","","","","","\x1B"," "," "," "," "," ","\x7F",[128],[129],[130],[131],[132],[133],[134],[135],[136],[137],[138],[139],[140],[141],[142],[143],[144],[145],[146],[147],[148],[149],[150],[151],[152],[153],[154],[155],[156],[157],[158],[159],[160],[173]," "," ","in","er"," t"," a","en","on","re"," s","at","or","es"," ","an"," "," d","he"," c"," p","is","ar","it",`
|
|
12
12
|
|
|
13
13
|
`,"al",[224,164],"le","ou"," m"," f"," w"," b","as","ing"," the","ic","et"," o","ion","ed","el"," n","ro","ent",[32,208],"nd","st",[225,131],"\u0430"," l"," in",`;
|
|
@@ -3266,33 +3266,33 @@ ${" ".repeat(a)}}`;default:return"any"}}function Pa(e){let a=["namespace functio
|
|
|
3266
3266
|
`," ffur","-All","Constru","Retrieved"," incr\xEDveis"," \u0438\u043C\u043A\u043E"," foliga"," Balanced","\u1021\u101E"," schlechten"," \u03B1\u03BB\u03BB\u03B1\u03B3"," \u0436\u0430\u0441\u0430\u043F"," S\u01A1n","\u0438\u0440\u0430\u0438","\u10D8\u10DC\u10D4"," milioane","EMU","\u2019\xE9viter"," hyr"," encontre"," hallar"," tablas","_Long","<Cell"," interacted"," figsize"," \u10D6\u10E6"," \u0447\u043E\u0432\u0435\u043A"," Codigo"," dedica\xE7\xE3o"," stolet"," Orten","Philip"," \xE9mission","\u0E32\u0E2B\u0E25\u0E35"," bearbeiten"," mierda","holde","\u0443\u0432\u0430\u0432","<Component"," WPA","\u05E6\u05D9\u05DC\u05D5\u05DD","SSA"," Allies"," B\xE1r","_mer","_SMS"," \u0918\u0942\u092E"," Fixtures"," Bask"," \u5F90"," tradu\xE7\xE3o","blicke"," hammered"," \u0441\u04AF\u043F"," \u0431\u0438\u0433","olu\xE7\xE3o"," electricians","\u0631\u06CC\u0634\u0646"," pitsaaner"," cier"," Receita","\u9EBB\u5C06"," \u0437\u0430\u043A\u043E\u043D\u043E\u0434\u0430\u0442\u0435\u043B\u044C\u0441\u0442\u0432\u0430"," \u10DE\u10D0\u10E1\u10E3\u10EE\u10D8\u10E1\u10DB\u10D2","-operated","\u0D4A\u0D2A\u0D4D\u0D2A\u0D02"," hassles","\u093E\u0924\u0930","\u0430\u043D\u0430\u0433\u0430\u0440\u0430","\u05E0\u05D8\u05E8","Japgolly"," doeleinden"," \u043F\u0435\u0440\u0435\u0439\u0442\u0438"," temsil"," ol\xEDmp"," Spazier"," \u0627\u0644\u0633\u0645\u0627"," pressione"," fiscais","l\u0259rini"," abode"," tantric","()],","rello"," \u0437\u0430\u0447\u0430\u0441\u0442\u0443\u044E","\u5FAE\u8F6F\u96C5\u9ED1","\u0633\u0631\u0627\u0626\u064A\u0644\u064A\u0629"," uitzending","flok"," escult"," Cerro"," is\xE6r"," kobiet","uree","\u0C95\u0CCA","encha"," adhesives"," paura","\u30C3\u30D4\u30F3\u30B0","\u0E27\u0E31\u0E22"," names"," slimmer","milk"," \u0A95\u0AB0\u0AA4\u0AC0"," Cim"," \u0441\u0432\u044F\u0437\u0430\u043D","\u200C\u0C28\u0C4D"," Chauff"," vingers","\u0430\u0446\u0430\u0440\u0442"," webcast",".notice"," m\u1EB7c","Passengers"," \u05E4\u05BC\u05E8\u05D0\u05B8\u05D3\u05D5\u05E7\u05D8\u05DF","\u0430\u043C\u0435\u0442\u0440"," VBA"," \u0440\u0430\u0437\u0432\u0438\u0432\u0430\u0435\u0442\u0441\u044F",`])));
|
|
3267
3267
|
`," surfers"," \u05DE\u05EA\u05D0\u05D9\u05DD",`')):
|
|
3268
3268
|
`,".apk","\u1EDBm"," \u05D4\u05DE\u05D6"," brincar"," AMOLED"," nate"," ayaan"," phosphorylation"," \u0985\u09AA\u09C7\u0995\u09CD\u09B7"," Eats"," Converse","\u094D\u092F\u0942\u091F\u0930","Traveler","inggi","\u05D5\u05E8\u05D2"," almoh"," \u0625\u0641\u0631\u064A\u0642\u064A\u0627"," Magazin"," Arme","\u0E23\u0E01\u0E34\u0E08"," a\u015Fk","-lines"," \u0627\u0644\u0623\u0645\u0627\u0643\u0646","\u516C\u5E73"," kv\xE4ll"," mlx"," \u092E\u0928\u094D\u0924\u094D\u0930\u0940","\u10D0\u10D5\u10D8\u10D0"," teks"," lumea"," legyen"," zwing"," boost"," J\xFA","\u5F69\u7968\u8BBA\u575B"," \u0434\u044D"," \u043E\u0448\u0438\u0431\u043E\u043A","\u0561\u056F\u0561\u0576\u056B","\u0E24\u0E29\u0E20\u0E32\u0E04\u0E21","ebwa"," tipu"," domini"," Gurgaon","combat"," Resist"," \u0930\u094B\u091C\u0917\u093E\u0930","strategie","\u0A17\u0A40","(locator","MONTH"," atlik"," \uB79C","\u0627\u0644\u0633\u064A"," rzecz"," W\xE4"," deportivos"," Facilit","-able"," evitare"," thong"," \u6D3B","\uBB34\uB8CC"," mace"," Extraordinary","\u0927\u093E"," konflik"," livelihoods"," \u043A\u043E\u043C\u043F\u043E\u043D\u0435\u043D\u0442\u043E\u0432"," kuitenkaan","\u5E78\u8FD0\u98DE\u8247","Corpus"," besch\xE4d"," \u05D3\u05D5\u05D3"," dentant"," nweta"," heath"," \u0936\u0941\u0915\u094D\u0930\u092C\u093E\u0930"," mnemonic",".Filters",`','".$`," \u0905\u0938\u094D\u0924","isite"," asawa","\u5E74\u6708"," gotas"," \u0C2A\u0C28\u0C3F\u0C1A","Dek"," Pact","'agence","fei\xE7o"," B\xF6yle","venteen","notif"," \u0410\u043D\u0434\u0440\u0435\u0439","\u591A\u5E74","mittedly","\u8CA1\u5E03"," \u043A\u043E\u043C\u043F\u0430\u043A\u0442","trk","\u0928\u094D\u0926\u094D\u0930\u0940\u092F"," miesto"," Kait"," \u0CAC\u0CB8"," spec"," Injector"," logarith","entions"," \u0447\u0435\u043B","Collateral"," confes","\u093F\u0939\u0930"," sufri\xF3","CRI"," tuge","urllib"," k\u013C"," verpleeg"," application's","(Constructor","\u09BE\u09B2\u09BE"," vulputate"," \u043E\u043F\u0443\u0445"," Nordrhein","allage","\xEDsimas","ornado","\u0438\u0434\u0435\u0440","m\xE9nyek","Qua"," pelicula","\uC131\uACFC","\u3011-"," hemi"," touristes"," absentee"," Nether",".PAGE"," i\u015Fg\xE4r"," efectivos"," saav"," ","flashdata"," sembler"," ikuku"," \u099F\u09BE\u0987",` \u201D
|
|
3269
|
-
`,"\u5F69\u7968\u5E73\u53F0\u5F00\u6237","\u65C1",".\u0413","uckles"," \u0627\u0644\u062E\u0627\u0631","\u0432\u043E\u043B\u044C"," \u043D\u0435\u0432\u043E\u0437\u043C\u043E\u0436",".gradient"," ebile"," Athlete"," \u0442\u04B1\u0440\u0493\u0430\u043D"," Syrie",".processing"," Consultancy"," thrills","\u0642\u0631\u0627\u0621","(varargin","k\xFCnfte"," \u5F6D","-knit","riedenheit"," t\xE4nka","']//","udies","hacer"," Th\xE9\xE2tre","av\u7535\u5F71"," Armada","-techn","ibris","veli"," razloga","-Hop"," \u05DE\u05D7\u05D3\u05E9"," Bloggers"," controversies"," \u0C1A\u0C30\u0C4D\u0C2F"," \u0D2A\u0D4D\u0D30\u0D2E\u0D41\u0D16","_unc","werfen"," \u043B\u043E\u043C"," \u043F\u043E\u043A\u0443\u043F\u0430\u0442\u0435\u043B\u0435\u0439"," \u0433\u043B\u0430\u0437\u0430\u043C\u0438"," \u05D1\u05E2\u05D5\u05D3"," nanot"," ws"," \u043A\u0435\u0437\u0434\u0435\u0441","):("," Produit","Aircraft","iffen"," patrones"," par\xE2metros","Cursos"," cocos"],ta=na.concat(ra);na.length=0;ra.length=0;var
|
|
3270
|
-
`),i=e.match(
|
|
3269
|
+
`,"\u5F69\u7968\u5E73\u53F0\u5F00\u6237","\u65C1",".\u0413","uckles"," \u0627\u0644\u062E\u0627\u0631","\u0432\u043E\u043B\u044C"," \u043D\u0435\u0432\u043E\u0437\u043C\u043E\u0436",".gradient"," ebile"," Athlete"," \u0442\u04B1\u0440\u0493\u0430\u043D"," Syrie",".processing"," Consultancy"," thrills","\u0642\u0631\u0627\u0621","(varargin","k\xFCnfte"," \u5F6D","-knit","riedenheit"," t\xE4nka","']//","udies","hacer"," Th\xE9\xE2tre","av\u7535\u5F71"," Armada","-techn","ibris","veli"," razloga","-Hop"," \u05DE\u05D7\u05D3\u05E9"," Bloggers"," controversies"," \u0C1A\u0C30\u0C4D\u0C2F"," \u0D2A\u0D4D\u0D30\u0D2E\u0D41\u0D16","_unc","werfen"," \u043B\u043E\u043C"," \u043F\u043E\u043A\u0443\u043F\u0430\u0442\u0435\u043B\u0435\u0439"," \u0433\u043B\u0430\u0437\u0430\u043C\u0438"," \u05D1\u05E2\u05D5\u05D3"," nanot"," ws"," \u043A\u0435\u0437\u0434\u0435\u0441","):("," Produit","Aircraft","iffen"," patrones"," par\xE2metros","Cursos"," cocos"],ta=na.concat(ra);na.length=0;ra.length=0;var Ya=ia.getEncodingApi("o200k_base",()=>ta),{decode:Qn,decodeAsyncGenerator:Zn,decodeGenerator:er,encode:ar,encodeGenerator:ir,isWithinTokenLimit:nr,countTokens:sa,encodeChat:rr,encodeChatGenerator:tr,vocabularySize:sr,setMergeCacheSize:or,clearMergeCache:lr,estimateCost:cr}=Ya;var Xa=/[A-Za-z]+|\d+|[^\sA-Za-z\d]/g;function Se(e,a=4){if(!e)return 0;let i=e.length/a,n=0;for(let r of e.match(Xa)??[]){let t=/^[A-Za-z]/.test(r)?6:/^\d/.test(r)?3:1;n+=Math.max(1,Math.ceil(r.length/t))}return Math.max(1,Math.round(Math.max(i,n)))}function I(e,a){if(!e)return{tokens:0,exact:a.tokenizer!=="heuristic"};if(a.tokenizer==="o200k")try{return{tokens:sa(e),exact:!0}}catch{}return{tokens:Se(e,a.charsPerToken),exact:!1}}var Qa={debugging:[/\b(error|exception|traceback|stack ?trace|crash(es|ed|ing)?|segfault|undefined is not|cannot read propert|null pointer|stack overflow)\b/],coding:[/\b(code|function|method|class|script|refactor|implement|api|endpoint|regex|sql|query|unit tests?|typescript|javascript|python|java|rust|golang|c\+\+|react|component|compile|deploy)\b/],editing:[/\b(proofread|rewrite|rephrase|reword|edit|polish|improve|fix (the )?grammar|make (it|this) (sound|more|less)|tone of)\b/],summarizing:[/\b(summari[sz]e|summary|tl;?dr|key (points|takeaways)|condense|recap|gist)\b/],translation:[/\btranslat(e|ion)\b|\b(in|into) (english|hindi|spanish|french|german|japanese|chinese|arabic|portuguese|italian|korean|russian)\b/],analysis:[/\b(analy[sz]e|analysis|dataset|csv|spreadsheet|table|chart|trend|average|median|correlat|statistics?|metrics?)\b/],brainstorming:[/\b(brainstorm|ideas? for|suggest (some|a few)|come up with|names? for|list of ideas)\b/],writing:[/\b(write|draft|compose|blog|essay|article|email|cover letter|post|story|poem|speech|announcement|outline)\b/],research:[/^(what|why|how|when|where|who|which|is|are|can|does|do|should|explain|compare|tell me)\b/,/\b(explain|difference between|compare|pros and cons|research|sources?)\b/]},Za=/\b(bug|broken|fails?|failing|doesn'?t work|not working|wrong output|fix (this|the|my|it))\b/,oa=["debugging","translation","summarizing","editing","analysis","coding","brainstorming","writing","research"],ei=/```[\s\S]*?```|~~~[\s\S]*?~~~/g,_e=/^\s*(def |class |function |const |let |var |import |from \S+ import|#include|public |private |return\b|if\s*\(|for\s*\(|while\s*\(|\}|\{|<\/?[a-z][\w-]*[ >])|;\s*$/,le=/^\s*(\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}|\[\d{2}:\d{2}:\d{2}|\[?(INFO|WARN|WARNING|ERROR|DEBUG|TRACE|FATAL)\]?\b|at .+[(:]\d+|File ".+", line \d+|Traceback \(most recent call last\)|\w+(Error|Exception)(: |$)|Caused by:)/,ai=/^\s*([^,\t\n]+[,\t]){2,}[^,\t\n]+\s*$|^\s*[[{]\s*"\w+"\s*:/,ii=/\b(in|under|within|at most|max(imum)?|no more than|about|around|exactly|only)\s+\d+\s*(words?|sentences?|bullets?|bullet points|lines?|paragraphs?|points?|items?|characters?|chars?|tokens?)\b|\b(\d+|one|two|three|four|five|six|seven|eight|nine|ten)[- ](words?|sentences?|bullets?|bullet points|lines?|paragraphs?|points?|ideas?|items?)\b|\b(brief(ly)?|concise(ly)?|short(ly)?|succinct(ly)?|tl;?dr|one[- ]liner|yes or no|just the (answer|code|diff))\b/,ni=/\b(full|entire|whole|complete)\s+(file|code|script|program|document|text|essay)\b|\brewrite (the )?(whole|entire)\b/;function ri(e,a){let i=e.filter(n=>n.trim());return i.length?i.filter(n=>a.test(n)).length/i.length:0}function Ee(e){let a=e.replace(/^\s+/,""),i=e.length-a.length,n=/\n\s*\n|:\s*\n/.exec(a);if(!n)return{instruction:e,material:""};let r=i+n.index+n[0].length;return{instruction:e.slice(0,r),material:e.slice(r)}}function ie(e){let a=e.split(`
|
|
3270
|
+
`),i=e.match(ei)??[],{instruction:n,material:r}=Ee(e),t=n.toLowerCase().trim(),o={hasCode:i.length>0||ri(a,_e)>.3,hasLogs:a.filter(d=>le.test(d)).length>=2,hasData:a.filter(d=>ai.test(d)).length>=3,pastedTokens:Se(r),hasLengthLimit:ii.test(e.toLowerCase()),asksForFullOutput:ni.test(t)},s=new Map;for(let d of oa){let u=Qa[d],g=u.reduce((p,y)=>p+(y.test(t)?2:0),0);!g&&r&&(g=u.reduce((p,y)=>p+(y.test(r.slice(0,2e3).toLowerCase())?.5:0),0)),s.set(d,g)}(o.hasCode||o.hasLogs)&&Za.test(t)&&s.set("debugging",(s.get("debugging")??0)+2),o.hasLogs&&s.set("debugging",(s.get("debugging")??0)+2),o.hasCode&&s.set("coding",(s.get("coding")??0)+1.5),o.hasData&&s.set("analysis",(s.get("analysis")??0)+1);let l="chat",c=0;for(let d of oa){let u=s.get(d)??0;u>c&&(l=d,c=u)}return{task:l,confidence:Math.min(1,c/4),signals:o}}var la=new Set(["chat","writing","brainstorming","research"]);var ti=[["number",/\b\d+(?:[.,]\d+)*%?/g],["url",/https?:\/\/\S+/g],["code",/`[^`\n]+`/g],["quote",/"[^"\n]{1,200}"|“[^”\n]{1,200}”/g],["negation",/\b(?:not|no|never|none|nothing|nor|without|cannot|can't|don't|doesn't|didn't|won't|wouldn't|shouldn't|isn't|aren't|wasn't|mustn't)\b/gi],["constraint",/\b(?:must|only|exactly|always|avoid|at least|at most|required|mandatory|strictly|unless|except)\b/gi]];function ce(e){let a=new Map;for(let[i,n]of ti)for(let r of e.match(n)??[]){let t=`${i}:${r.toLowerCase()}`;a.set(t,(a.get(t)??0)+1)}return a}function ne(e,a,i="count"){let n=ce(e),r=ce(a),t=[];for(let[o,s]of n)(r.get(o)??0)<(i==="count"?s:1)&&t.push(o);return{ok:t.length===0,lost:t}}function Ae(e,a){let i=ce(e),n=ce(a),r={};for(let t of i.keys()){let o=t.slice(0,t.indexOf(":")),[s,l]=r[o]??[0,0];r[o]=[s+1,l+(n.has(t)?1:0)]}return r}var si=new Set("please kindly thank thanks much lot advance hi hello hey there chatgpt claude gemini basically actually really very just simply quite literally totally definitely important worth noting note make sure order regard respect fact purpose event point time near future large number majority able wondering like love want need you your would could will can helpful assistant language model act possible mind think step breath deep take reasoning thought process work before answering answer show explain spite due lets let's the that this with for are was is it's its and".split(" ")),ca=/[a-z][a-z'-]{2,}/g;function Ce(e,a){let i=new Set(a.toLowerCase().match(ca)??[]),n=new Set;for(let r of e.toLowerCase().match(ca)??[])!i.has(r)&&!si.has(r)&&n.add(r);return[...n]}var b=(e,a,i,n,r=!1)=>({re:new RegExp(e,"gim"),repl:a,name:i,level:n,reasoningOnly:r}),oi=[b(String.raw`\bdue to the fact that\b`,"because","wordy phrasing","light"),b(String.raw`\bin spite of the fact that\b`,"although","wordy phrasing","light"),b(String.raw`\bin the event that\b`,"if","wordy phrasing","light"),b(String.raw`\bfor the purpose of\b`,"for","wordy phrasing","light"),b(String.raw`\bin order to\b`,"to","wordy phrasing","light"),b(String.raw`\bwith (?:regard|respect) to\b`,"about","wordy phrasing","light"),b(String.raw`\bat this point in time\b`,"now","wordy phrasing","light"),b(String.raw`\bin the near future\b`,"soon","wordy phrasing","light"),b(String.raw`\ba large number of\b`,"many","wordy phrasing","light"),b(String.raw`\bthe majority of\b`,"most","wordy phrasing","light"),b(String.raw`\b(is|are) able to\b`,"can","wordy phrasing","light"),b(String.raw`\bmake sure that\b`,"ensure","wordy phrasing","light"),b(String.raw`\b(?:let'?s|please)?\s*think (?:about (?:this|it) )?step[- ]by[- ]step\b[.,]?\s*`,"","step-by-step (model already reasons)","light",!0),b(String.raw`\btake a deep breath(?: and)?[.,]?\s*`,"","step-by-step (model already reasons)","light",!0),b(String.raw`\b(?:show|explain) your (?:reasoning|thought process|work) (?:step[- ]by[- ]step )?(?:before (?:answering|you answer))[.,]?\s*`,"","step-by-step (model already reasons)","light",!0),b(String.raw`^\s*(?:hi|hello|hey)(?: there)?(?: chatgpt| claude| gemini)?[,!.]?\s*`,"","politeness","balanced"),b(String.raw`(^|[.!?]\s+)(?:could|can|would|will) you (?:please )?(?:kindly )?([^.!?\n]*)\?`,"$1$2.","politeness","balanced"),b(String.raw`(^|[.!?]\s+)(?:could|can|would|will) you (?:please )?(?:kindly )?`,"$1","politeness","balanced"),b(String.raw`\bi would (?:really )?(?:like|love) (?:for )?you to\b\s*`,"","politeness","balanced"),b(String.raw`\bi (?:want|need) you to\b\s*`,"","politeness","balanced"),b(String.raw`\bplease\b,?\s*`,"","politeness","balanced"),b(String.raw`\bkindly\b\s*`,"","politeness","balanced"),b(String.raw`\bthank you(?: so much| very much| a lot)?(?: in advance)?[.!]?`,"","politeness","balanced"),b(String.raw`\bthanks(?: so much| very much| a lot)?(?: in advance)?[.!]?`,"","politeness","balanced"),b(String.raw`\b(?:it is|it's) (?:important|worth noting) (?:to note )?that\b\s*`,"","filler","balanced"),b(String.raw`\b(?:please )?note that\b\s*`,"","filler","balanced"),b(String.raw`\b(?:you are|act as|as) (?:an?|my) (?:ai|large language model|helpful assistant|ai assistant)[,.]?\s*`,"","filler","balanced"),b(String.raw`\bi was wondering if\b\s*`,"","filler","balanced"),b(String.raw`(?<!\bnot )\b(?:basically|actually|really|very|just|simply|quite|literally|totally|definitely)\b\s*`,"","hedges","aggressive"),b(String.raw`\b(?:if possible|if you can|if you don't mind)\b[,.]?\s*`,"","hedges","aggressive")],ue={light:0,balanced:1,aggressive:2},li=/(```[\s\S]*?```|~~~[\s\S]*?~~~)/,ci=new RegExp(["`[^`\\n]+`","https?:\\/\\/\\S+","<([A-Za-z][\\w-]*)[^>]*>[\\s\\S]*?<\\/\\1>",'"""[\\s\\S]*?"""'].join("|"),"g");function ui(e){return/^\s*>/.test(e)||/^\s{4,}\S/.test(e)||_e.test(e)||le.test(e)}function di(e,a){return a&&/^[A-Z]/.test(e)?a[0].toUpperCase()+a.slice(1):a}function mi(e,a,i){for(let n of oi){if(ue[n.level]>ue[a.level]||n.reasoningOnly&&!a.profile.reasoning)continue;n.re.lastIndex=0;let r=e.replace(n.re,(...t)=>{let o=t[0],s=t.slice(1,-2),l=n.repl.replace(/\$(\d)/g,(d,u)=>s[Number(u)-1]??""),c=n.repl.startsWith("$1")?s[0]??"":"";return c+di(o.slice(c.length),l.slice(c.length))});r!==e&&(i.push(n.name),e=r)}return e}function pi(e,a){let i=new Set;return e.split(`
|
|
3271
3271
|
`).map(n=>n.split(/(?<=[.!?])\s+/).filter(r=>{let t=r.replace(/\W+/g," ").trim().toLowerCase();return t.split(" ").length<4?!0:i.has(t)?(a.push("repeated sentence"),!1):(i.add(t),!0)}).join(" ")).join(`
|
|
3272
|
-
`)}function
|
|
3272
|
+
`)}function gi(e){return e.replace(/[ \t]+/g," ").replace(/ +([,.;:!?])/g,"$1").replace(/([,;:])(?=[.!?])/g,"").replace(/^[ ,;:]+/gm,"").replace(/ +$/gm,"").replace(/\n{3,}/g,`
|
|
3273
3273
|
|
|
3274
|
-
`).replace(/(^|[.!?]\s+)([a-z])/gm,(a,i,n)=>i+n.toUpperCase())}function ua(e,a,i){if(!e.trim())return e;let n=e.slice(0,e.length-e.trimStart().length),r=e.slice(e.trimEnd().length),t=[],o=d=>`\0${t.push(d)-1}\0`,s=e.trim().replace(
|
|
3275
|
-
`).map(d=>
|
|
3276
|
-
`),ue[a.level]>=ue.balanced&&(s=
|
|
3277
|
-
`),r=[];for(let s=0;s<n.length;){let l=s,c=0;for(;l<n.length&&pa.test(n[l]);)l+=/^\s*File "/.test(n[l])&&l+1<n.length&&/^\s{4,}\S/.test(n[l+1])&&!/^\s*(File|at) /.test(n[l+1])?2:1,c++;if(c>=3){let d=/^\s*/.exec(n[s])[0];r.push(`${d}... ${c} library frames omitted`),i.push("library stack frames"),s=l}else r.push(n[s]),s++}let t=a==="aggressive"?
|
|
3274
|
+
`).replace(/(^|[.!?]\s+)([a-z])/gm,(a,i,n)=>i+n.toUpperCase())}function ua(e,a,i){if(!e.trim())return e;let n=e.slice(0,e.length-e.trimStart().length),r=e.slice(e.trimEnd().length),t=[],o=d=>`\0${t.push(d)-1}\0`,s=e.trim().replace(ci,o);s=s.split(`
|
|
3275
|
+
`).map(d=>ui(d)?o(d):d).join(`
|
|
3276
|
+
`),ue[a.level]>=ue.balanced&&(s=pi(s,i));let l=s,c=[];for(s=gi(mi(s,a,c)),ne(l,s).ok?i.push(...c):s=l;/\u0000\d+\u0000/.test(s);)s=s.replace(/\u0000(\d+)\u0000/g,(d,u)=>t[Number(u)]);return n+s+r}var pa=/^\s*at .*(node_modules|node:internal|internal\/|java\.base|java\.|javax\.|jdk\.internal|sun\.|org\.springframework|org\.apache|kotlin\.|scala\.|<anonymous>)|^\s*File ".*(site-packages|dist-packages|lib\/python\d|\/usr\/lib)/,hi=e=>e.replace(/\d{4}-\d{2}-\d{2}[T ][\d:.,]+Z?/g,"<ts>").replace(/\b0x[0-9a-f]+\b/gi,"<hex>").replace(/\d+/g,"#").trim();function da(e,a,i){let n=e.split(`
|
|
3277
|
+
`),r=[];for(let s=0;s<n.length;){let l=s,c=0;for(;l<n.length&&pa.test(n[l]);)l+=/^\s*File "/.test(n[l])&&l+1<n.length&&/^\s{4,}\S/.test(n[l+1])&&!/^\s*(File|at) /.test(n[l+1])?2:1,c++;if(c>=3){let d=/^\s*/.exec(n[s])[0];r.push(`${d}... ${c} library frames omitted`),i.push("library stack frames"),s=l}else r.push(n[s]),s++}let t=a==="aggressive"?hi:s=>s,o=[];for(let s=0;s<r.length;){let l=s+1;for(;l<r.length&&r[s].trim()&&t(r[l])===t(r[s]);)l++;o.push(r[s]),l-s>=3?(o.push(`... same line repeated ${l-s-1} more times`),i.push("repeated log lines")):o.push(...r.slice(s+1,l)),s=l}return o.join(`
|
|
3278
3278
|
`)}function ma(e){let a=e.split(`
|
|
3279
|
-
`).filter(i=>i.trim());return a.length>=3&&a.filter(i=>le.test(i)||pa.test(i)).length/a.length>=.4}function
|
|
3279
|
+
`).filter(i=>i.trim());return a.length>=3&&a.filter(i=>le.test(i)||pa.test(i)).length/a.length>=.4}function bi(e,a){let i=/^(?:```|~~~)\s*([\w+-]*)/.exec(e)?.[1]?.toLowerCase()??"",n=e.replace(/^(?:```|~~~).*\n?/,"");if(/^(md|markdown|diff|patch|text|txt)$/.test(i)||/"""|'''|`[^`\n]*\n|<<-?\s*['"]?\w+/.test(n))return e;let r=e.replace(/[ \t]+$/gm,"").replace(/\n{4,}/g,`
|
|
3280
3280
|
|
|
3281
|
-
`);return r!==e&&a.push("whitespace in code"),r}function O(e,a){let i=[],{instruction:n}=Ee(e),r=n.length,t=a.task==="debugging"&&a.level!=="light",o=0,s=[];for(let[u,g]of e.split(oi).entries()){let p=o;if(o+=g.length,u%2===1){s.push(t&&ma(g)?da(g,a.level,i):a.level==="light"?g:hi(g,i));continue}let y=g.slice(0,Math.max(0,r-p)),h=g.slice(y.length),f=h;t&&ma(h)?f=da(h,a.level,i):la.has(a.task)&&(f=ua(h,a,i)),s.push(ua(y,a,i)+f)}let l=s.join(""),c=I(e,a.profile).tokens,d=I(l,a.profile).tokens;return d>=c||!l.trim()?{text:e,rules:[],originalTokens:c,tokens:c,saved:0}:{text:l,rules:[...new Set(i)],originalTokens:c,tokens:d,saved:c-d}}function Te(e,a="input"){let i=e.pricing;return i?(a==="output"?i.outputPerMTok:a==="cached"?i.cachedInputPerMTok??i.inputPerMTok:i.inputPerMTok)/1e6:null}function W(e,a,i="input"){let n=Te(a,i);return n===null?null:e*n}function q(e,a,i="input"){let n=W(e,a,i);return{tokens:e,usd:n,usdPer1k:n===null?null:n*1e3}}function ga(e,a){let i=Te(a,"input"),n=Te(a,"cached"),r=i===null||n===null?null:e*(i-n);return{tokens:e,usd:r,usdPer1k:r===null?null:r*1e3}}function K(e){if(e==null||!Number.isFinite(e))return"n/a";let a=Math.abs(e),i=e<0?"-":"";if(a===0)return"$0";if(a>=100)return`${i}$${a.toLocaleString("en-US",{maximumFractionDigits:0})}`;if(a>=.1)return`${i}$${a.toFixed(2)}`;let n=Math.min(8,Math.max(2,1-Math.floor(Math.log10(a))));return`${i}$${a.toFixed(n).replace(/(\.\d*?[1-9])0+$/,"$1")}`}var v=e=>e,H={"gpt-astra":v({id:"gpt-astra",label:"GPT-6 Astra",family:"openai",contextWindow:105e4,tokenizer:"heuristic",charsPerToken:4.2,reasoning:!0,pricing:{inputPerMTok:10,outputPerMTok:50}}),"gpt-reasoning":v({id:"gpt-reasoning",label:"GPT (thinking)",family:"openai",contextWindow:196e3,tokenizer:"o200k",charsPerToken:4.2,reasoning:!0,lighterAlternative:"the Instant / non-thinking model",pricing:{inputPerMTok:5,outputPerMTok:30,cachedInputPerMTok:.5}}),gpt:v({id:"gpt",label:"GPT",family:"openai",contextWindow:128e3,tokenizer:"o200k",charsPerToken:4.2,reasoning:!1,pricing:{inputPerMTok:2.5,outputPerMTok:15,cachedInputPerMTok:.25}}),"claude-fable":v({id:"claude-fable",label:"Claude Fable",family:"anthropic",contextWindow:1e6,tokenizer:"heuristic",charsPerToken:3.5,reasoning:!0,notes:["Claude reads XML-style tags well: wrap pasted material in <doc>\u2026</doc> instead of explaining it."],pricing:{inputPerMTok:10,outputPerMTok:50,cachedInputPerMTok:.25}}),"claude-opus":v({id:"claude-opus",label:"Claude Opus",family:"anthropic",contextWindow:2e5,tokenizer:"heuristic",charsPerToken:3.5,reasoning:!1,lighterAlternative:"Sonnet or Haiku",notes:["Claude reads XML-style tags well: wrap pasted material in <doc>\u2026</doc> instead of explaining it."],pricing:{inputPerMTok:4,outputPerMTok:20,cachedInputPerMTok:.2}}),"claude-sonnet":v({id:"claude-sonnet",label:"Claude Sonnet",family:"anthropic",contextWindow:2e5,tokenizer:"heuristic",charsPerToken:3.5,reasoning:!1,lighterAlternative:"Haiku",notes:["Claude reads XML-style tags well: wrap pasted material in <doc>\u2026</doc> instead of explaining it."],pricing:{inputPerMTok:2,outputPerMTok:10,cachedInputPerMTok:.2}}),"claude-haiku":v({id:"claude-haiku",label:"Claude Haiku",family:"anthropic",contextWindow:2e5,tokenizer:"heuristic",charsPerToken:3.5,reasoning:!1,pricing:{inputPerMTok:.1,outputPerMTok:.5,cachedInputPerMTok:.01}}),"gemini-pro":v({id:"gemini-pro",label:"Gemini Pro",family:"google",contextWindow:1e6,tokenizer:"heuristic",charsPerToken:4,reasoning:!0,lighterAlternative:"Flash",pricing:{inputPerMTok:2,outputPerMTok:12,cachedInputPerMTok:.2,longContext:{aboveTokens:2e5,inputPerMTok:4,outputPerMTok:18}}}),"gemini-flash":v({id:"gemini-flash",label:"Gemini Flash",family:"google",contextWindow:1e6,tokenizer:"heuristic",charsPerToken:4,reasoning:!1,pricing:{inputPerMTok:1.5,outputPerMTok:9,cachedInputPerMTok:.15}}),deepseek:v({id:"deepseek",label:"DeepSeek",family:"deepseek",contextWindow:128e3,tokenizer:"heuristic",charsPerToken:3.8,reasoning:!1,pricing:{inputPerMTok:.28,outputPerMTok:.42,cachedInputPerMTok:.028}}),"deepseek-reasoning":v({id:"deepseek-reasoning",label:"DeepSeek (thinking)",family:"deepseek",contextWindow:128e3,tokenizer:"heuristic",charsPerToken:3.8,reasoning:!0,lighterAlternative:"the non-thinking mode",pricing:{inputPerMTok:.55,outputPerMTok:2.19,cachedInputPerMTok:.14}}),mistral:v({id:"mistral",label:"Mistral",family:"mistral",contextWindow:128e3,tokenizer:"heuristic",charsPerToken:3.8,reasoning:!1,pricing:{inputPerMTok:2,outputPerMTok:6}}),grok:v({id:"grok",label:"Grok",family:"xai",contextWindow:128e3,tokenizer:"heuristic",charsPerToken:4,reasoning:!1,pricing:{inputPerMTok:2,outputPerMTok:6}}),llama:v({id:"llama",label:"Llama",family:"other",contextWindow:128e3,tokenizer:"heuristic",charsPerToken:3.9,reasoning:!1,pricing:{inputPerMTok:.27,outputPerMTok:.85}}),qwen:v({id:"qwen",label:"Qwen",family:"other",contextWindow:128e3,tokenizer:"heuristic",charsPerToken:3.8,reasoning:!1,pricing:{inputPerMTok:.4,outputPerMTok:1.2}}),kimi:v({id:"kimi",label:"Kimi",family:"other",contextWindow:256e3,tokenizer:"heuristic",charsPerToken:3.8,reasoning:!1,pricing:{inputPerMTok:.6,outputPerMTok:2.5,cachedInputPerMTok:.15}}),glm:v({id:"glm",label:"GLM",family:"other",contextWindow:2e5,tokenizer:"heuristic",charsPerToken:3.8,reasoning:!1,pricing:{inputPerMTok:.6,outputPerMTok:2.2,cachedInputPerMTok:.11}}),command:v({id:"command",label:"Cohere Command",family:"other",contextWindow:256e3,tokenizer:"heuristic",charsPerToken:4,reasoning:!1,pricing:{inputPerMTok:2.5,outputPerMTok:10}}),sonar:v({id:"sonar",label:"Perplexity Sonar",family:"other",contextWindow:128e3,tokenizer:"heuristic",charsPerToken:4,reasoning:!1,pricing:{inputPerMTok:1,outputPerMTok:1}}),gemma:v({id:"gemma",label:"Gemma",family:"other",contextWindow:128e3,tokenizer:"heuristic",charsPerToken:3.9,reasoning:!1,pricing:{inputPerMTok:.1,outputPerMTok:.2}}),phi:v({id:"phi",label:"Phi",family:"other",contextWindow:16e3,tokenizer:"heuristic",charsPerToken:3.9,reasoning:!1,pricing:{inputPerMTok:.07,outputPerMTok:.14}}),generic:v({id:"generic",label:"Unknown model",family:"other",contextWindow:128e3,tokenizer:"heuristic",charsPerToken:4,reasoning:!1,pricing:{inputPerMTok:2,outputPerMTok:10}})},bi=[[/\b(fable|mythos)\b/,"claude-fable"],[/\bastra\b|\bgpt[- ]?6\b/,"gpt-astra"],[/\bclaude\b.*\bopus\b|\bopus\b/,"claude-opus"],[/\bsonnet\b/,"claude-sonnet"],[/\bhaiku\b/,"claude-haiku"],[/\b(qwen|qwq)/,"qwen"],[/\b(glm|chatglm|zhipu)\b|\bglm-?\d/,"glm"],[/\b(kimi|moonshot)/,"kimi"],[/\bgemma/,"gemma"],[/\bgemini\b.*\bflash\b|\bflash\b/,"gemini-flash"],[/\bgemini\b.*\b(pro|ultra|deep think)\b|^\s*\d(\.\d)?\s*pro\b|\bdeep think\b/,"gemini-pro"],[/\bdeepseek\b.*\b(r1|reason|think|deepthink)|\bdeepthink\b/,"deepseek-reasoning"],[/\bdeepseek\b/,"deepseek"],[/\b(mistral|le chat|magistral|codestral)\b/,"mistral"],[/\bgrok\b/,"grok"],[/\bllama|codellama/,"llama"],[/\bcommand[- ]?(r|a)\b|\bcohere\b/,"command"],[/\bsonar\b|\bperplexity\b/,"sonar"],[/\bphi-?\d/,"phi"],[/\bo[1-9](-mini|-pro)?\b|\bgpt[- ]?\S*\s*(thinking|pro)\b|\bthinking\b/,"gpt-reasoning"],[/\b(gpt|chatgpt)\b|\bgpt-?\d/,"gpt"],[/\bgemini\b/,"gemini-pro"],[/\bclaude\b/,"claude-sonnet"]];var ki={chatgpt:"gpt",claude:"claude-sonnet",gemini:"gemini-flash",deepseek:"deepseek",mistral:"mistral",grok:"grok",generic:"generic"},fi=[];var vr={...H};function re(e,a="generic"){let i=(e??"").toLowerCase().replace(/\s+/g," ").trim();if(i){for(let[n,r]of fi)if(n.test(i))return H[r];if(a==="gemini"&&/\b(pro|deep think)\b/.test(i))return H["gemini-pro"];if(a==="gemini"&&/\bflash\b/.test(i))return H["gemini-flash"];if(a==="deepseek"&&/\b(deepthink|r1|think)/.test(i))return H["deepseek-reasoning"];for(let[n,r]of bi)if(n.test(i))return H[r]}return H[ki[a]??"generic"]}function ze(e,a){return e===a||ne(e,a,"presence").ok&&Ce(e,a).length===0}function Re(e){return Array.isArray(e)?e.map(Re):!e||typeof e!="object"?e:Object.fromEntries(Object.entries(e).filter(([a])=>a!=="description").map(([a,i])=>[a,Re(i)]))}function ha(e){let a=[];e.system&&a.push(["system prompt",e.system]);for(let i of e.tools??[]){i.description&&a.push([`tool ${i.name}`,i.description]);let n=(r,t)=>{if(Array.isArray(r))r.forEach((o,s)=>n(o,`${t}[${s}]`));else if(r&&typeof r=="object")for(let[o,s]of Object.entries(r))o==="description"&&typeof s=="string"?a.push([`tool ${i.name} ${t}`,s]):n(s,`${t}.${o}`)};n(i.parameters,"params")}for(let[i,n]of(e.examples??[]).entries())a.push([`example ${i+1}`,n]);return e.prompt&&a.push(["next prompt",e.prompt]),a}var de=(e,a)=>`${e} ${a}${e===1?"":"s"}`;function ba(e,a){let i=[],n=ha(e),r=new Map(ha(a)),t={},o=[];for(let[h,f]of n){let _=r.get(h)??"";for(let[T,[P,ge]]of Object.entries(Ae(f,_))){let[C,te]=t[T]??[0,0];t[T]=[C+P,te+ge]}ne(f,_,"presence").ok||o.push(h)}let s=Object.entries(t).map(([h,[f,_]])=>`${_}/${f} ${h}s`);i.push({name:"Facts and constraints kept",passed:o.length===0,detail:o.length?`Lost in: ${o.join(", ")}`:s.length?`Kept ${s.join(", ")}.`:"No numbers, links, code or constraint words to keep."});let l=[];for(let[h,f]of n){let _=Ce(f,r.get(h)??"");_.length&&l.push(`${h}: ${_.join(", ")}`)}i.push({name:"Instructions kept",passed:l.length===0,detail:l.length?`Missing words: ${l.join("; ")}`:`Every content word of ${de(n.length,"text")} is still there; only filler was removed.`});let c=(h=[])=>JSON.stringify(h.map(f=>[f.name,Re(f.parameters??null)])),d=c(e.tools)===c(a.tools);i.push({name:"Tool interface unchanged",passed:d,detail:d?`${de(e.tools?.length??0,"tool")}: same names, parameters, types, required fields and enums.`:"Tool names or parameter schemas differ."});let u=e.messages??[],g=a.messages??[],p=0,y=u.length===g.length&&u.every((h,f)=>{let _=g[f];if(_.role!==h.role)return!1;if(_.content===h.content)return!0;let T=/^\[Same content as message (\d+) above\.\]$/.exec(_.content),P=!!T&&u[Number(T[1])-1]?.content.trim()===h.content.trim();return P&&p++,P});return u.length&&i.push({name:"Conversation unchanged",passed:y,detail:y?`${de(u.length,"message")} in order${p?`; ${de(p,"exact repeat")} replaced by a pointer to the first copy`:", all verbatim"}.`:"Messages were dropped, reordered or edited."}),i.push({name:"Model and examples unchanged",passed:(e.examples?.length??0)===(a.examples?.length??0),detail:"Token Lens never switches the model or drops examples on its own."}),{passed:i.every(h=>h.passed),checks:i}}var S=e=>e.toLocaleString("en-US"),ka=(e,a)=>a?Math.round(e/a*100):0;function fa(e){return`${e.name}
|
|
3281
|
+
`);return r!==e&&a.push("whitespace in code"),r}function O(e,a){let i=[],{instruction:n}=Ee(e),r=n.length,t=a.task==="debugging"&&a.level!=="light",o=0,s=[];for(let[u,g]of e.split(li).entries()){let p=o;if(o+=g.length,u%2===1){s.push(t&&ma(g)?da(g,a.level,i):a.level==="light"?g:bi(g,i));continue}let y=g.slice(0,Math.max(0,r-p)),h=g.slice(y.length),f=h;t&&ma(h)?f=da(h,a.level,i):la.has(a.task)&&(f=ua(h,a,i)),s.push(ua(y,a,i)+f)}let l=s.join(""),c=I(e,a.profile).tokens,d=I(l,a.profile).tokens;return d>=c||!l.trim()?{text:e,rules:[],originalTokens:c,tokens:c,saved:0}:{text:l,rules:[...new Set(i)],originalTokens:c,tokens:d,saved:c-d}}function Te(e,a="input"){let i=e.pricing;return i?(a==="output"?i.outputPerMTok:a==="cached"?i.cachedInputPerMTok??i.inputPerMTok:i.inputPerMTok)/1e6:null}function W(e,a,i="input"){let n=Te(a,i);return n===null?null:e*n}function q(e,a,i="input"){let n=W(e,a,i);return{tokens:e,usd:n,usdPer1k:n===null?null:n*1e3}}function ga(e,a){let i=Te(a,"input"),n=Te(a,"cached"),r=i===null||n===null?null:e*(i-n);return{tokens:e,usd:r,usdPer1k:r===null?null:r*1e3}}function K(e){if(e==null||!Number.isFinite(e))return"n/a";let a=Math.abs(e),i=e<0?"-":"";if(a===0)return"$0";if(a>=100)return`${i}$${a.toLocaleString("en-US",{maximumFractionDigits:0})}`;if(a>=.1)return`${i}$${a.toFixed(2)}`;let n=Math.min(8,Math.max(2,1-Math.floor(Math.log10(a))));return`${i}$${a.toFixed(n).replace(/(\.\d*?[1-9])0+$/,"$1")}`}var v=e=>e,H={"gpt-astra":v({id:"gpt-astra",label:"GPT-6 Astra",family:"openai",contextWindow:105e4,tokenizer:"heuristic",charsPerToken:4.2,reasoning:!0,pricing:{inputPerMTok:10,outputPerMTok:50}}),"gpt-reasoning":v({id:"gpt-reasoning",label:"GPT (thinking)",family:"openai",contextWindow:196e3,tokenizer:"o200k",charsPerToken:4.2,reasoning:!0,lighterAlternative:"the Instant / non-thinking model",pricing:{inputPerMTok:5,outputPerMTok:30,cachedInputPerMTok:.5}}),gpt:v({id:"gpt",label:"GPT",family:"openai",contextWindow:128e3,tokenizer:"o200k",charsPerToken:4.2,reasoning:!1,pricing:{inputPerMTok:2.5,outputPerMTok:15,cachedInputPerMTok:.25}}),"claude-fable":v({id:"claude-fable",label:"Claude Fable",family:"anthropic",contextWindow:1e6,tokenizer:"heuristic",charsPerToken:3.5,reasoning:!0,notes:["Claude reads XML-style tags well: wrap pasted material in <doc>\u2026</doc> instead of explaining it."],pricing:{inputPerMTok:10,outputPerMTok:50,cachedInputPerMTok:.25}}),"claude-opus":v({id:"claude-opus",label:"Claude Opus",family:"anthropic",contextWindow:2e5,tokenizer:"heuristic",charsPerToken:3.5,reasoning:!1,lighterAlternative:"Sonnet or Haiku",notes:["Claude reads XML-style tags well: wrap pasted material in <doc>\u2026</doc> instead of explaining it."],pricing:{inputPerMTok:4,outputPerMTok:20,cachedInputPerMTok:.2}}),"claude-sonnet":v({id:"claude-sonnet",label:"Claude Sonnet",family:"anthropic",contextWindow:2e5,tokenizer:"heuristic",charsPerToken:3.5,reasoning:!1,lighterAlternative:"Haiku",notes:["Claude reads XML-style tags well: wrap pasted material in <doc>\u2026</doc> instead of explaining it."],pricing:{inputPerMTok:2,outputPerMTok:10,cachedInputPerMTok:.2}}),"claude-haiku":v({id:"claude-haiku",label:"Claude Haiku",family:"anthropic",contextWindow:2e5,tokenizer:"heuristic",charsPerToken:3.5,reasoning:!1,pricing:{inputPerMTok:.1,outputPerMTok:.5,cachedInputPerMTok:.01}}),"gemini-pro":v({id:"gemini-pro",label:"Gemini Pro",family:"google",contextWindow:1e6,tokenizer:"heuristic",charsPerToken:4,reasoning:!0,lighterAlternative:"Flash",pricing:{inputPerMTok:2,outputPerMTok:12,cachedInputPerMTok:.2,longContext:{aboveTokens:2e5,inputPerMTok:4,outputPerMTok:18}}}),"gemini-flash":v({id:"gemini-flash",label:"Gemini Flash",family:"google",contextWindow:1e6,tokenizer:"heuristic",charsPerToken:4,reasoning:!1,pricing:{inputPerMTok:1.5,outputPerMTok:9,cachedInputPerMTok:.15}}),deepseek:v({id:"deepseek",label:"DeepSeek",family:"deepseek",contextWindow:128e3,tokenizer:"heuristic",charsPerToken:3.8,reasoning:!1,pricing:{inputPerMTok:.28,outputPerMTok:.42,cachedInputPerMTok:.028}}),"deepseek-reasoning":v({id:"deepseek-reasoning",label:"DeepSeek (thinking)",family:"deepseek",contextWindow:128e3,tokenizer:"heuristic",charsPerToken:3.8,reasoning:!0,lighterAlternative:"the non-thinking mode",pricing:{inputPerMTok:.55,outputPerMTok:2.19,cachedInputPerMTok:.14}}),mistral:v({id:"mistral",label:"Mistral",family:"mistral",contextWindow:128e3,tokenizer:"heuristic",charsPerToken:3.8,reasoning:!1,pricing:{inputPerMTok:2,outputPerMTok:6}}),grok:v({id:"grok",label:"Grok",family:"xai",contextWindow:128e3,tokenizer:"heuristic",charsPerToken:4,reasoning:!1,pricing:{inputPerMTok:2,outputPerMTok:6}}),llama:v({id:"llama",label:"Llama",family:"other",contextWindow:128e3,tokenizer:"heuristic",charsPerToken:3.9,reasoning:!1,pricing:{inputPerMTok:.27,outputPerMTok:.85}}),qwen:v({id:"qwen",label:"Qwen",family:"other",contextWindow:128e3,tokenizer:"heuristic",charsPerToken:3.8,reasoning:!1,pricing:{inputPerMTok:.4,outputPerMTok:1.2}}),kimi:v({id:"kimi",label:"Kimi",family:"other",contextWindow:256e3,tokenizer:"heuristic",charsPerToken:3.8,reasoning:!1,pricing:{inputPerMTok:.6,outputPerMTok:2.5,cachedInputPerMTok:.15}}),glm:v({id:"glm",label:"GLM",family:"other",contextWindow:2e5,tokenizer:"heuristic",charsPerToken:3.8,reasoning:!1,pricing:{inputPerMTok:.6,outputPerMTok:2.2,cachedInputPerMTok:.11}}),command:v({id:"command",label:"Cohere Command",family:"other",contextWindow:256e3,tokenizer:"heuristic",charsPerToken:4,reasoning:!1,pricing:{inputPerMTok:2.5,outputPerMTok:10}}),sonar:v({id:"sonar",label:"Perplexity Sonar",family:"other",contextWindow:128e3,tokenizer:"heuristic",charsPerToken:4,reasoning:!1,pricing:{inputPerMTok:1,outputPerMTok:1}}),gemma:v({id:"gemma",label:"Gemma",family:"other",contextWindow:128e3,tokenizer:"heuristic",charsPerToken:3.9,reasoning:!1,pricing:{inputPerMTok:.1,outputPerMTok:.2}}),phi:v({id:"phi",label:"Phi",family:"other",contextWindow:16e3,tokenizer:"heuristic",charsPerToken:3.9,reasoning:!1,pricing:{inputPerMTok:.07,outputPerMTok:.14}}),generic:v({id:"generic",label:"Unknown model",family:"other",contextWindow:128e3,tokenizer:"heuristic",charsPerToken:4,reasoning:!1,pricing:{inputPerMTok:2,outputPerMTok:10}})},ki=[[/\b(fable|mythos)\b/,"claude-fable"],[/\bastra\b|\bgpt[- ]?6\b/,"gpt-astra"],[/\bclaude\b.*\bopus\b|\bopus\b/,"claude-opus"],[/\bsonnet\b/,"claude-sonnet"],[/\bhaiku\b/,"claude-haiku"],[/\b(qwen|qwq)/,"qwen"],[/\b(glm|chatglm|zhipu)\b|\bglm-?\d/,"glm"],[/\b(kimi|moonshot)/,"kimi"],[/\bgemma/,"gemma"],[/\bgemini\b.*\bflash\b|\bflash\b/,"gemini-flash"],[/\bgemini\b.*\b(pro|ultra|deep think)\b|^\s*\d(\.\d)?\s*pro\b|\bdeep think\b/,"gemini-pro"],[/\bdeepseek\b.*\b(r1|reason|think|deepthink)|\bdeepthink\b/,"deepseek-reasoning"],[/\bdeepseek\b/,"deepseek"],[/\b(mistral|le chat|magistral|codestral)\b/,"mistral"],[/\bgrok\b/,"grok"],[/\bllama|codellama/,"llama"],[/\bcommand[- ]?(r|a)\b|\bcohere\b/,"command"],[/\bsonar\b|\bperplexity\b/,"sonar"],[/\bphi-?\d/,"phi"],[/\bo[1-9](-mini|-pro)?\b|\bgpt[- ]?\S*\s*(thinking|pro)\b|\bthinking\b/,"gpt-reasoning"],[/\b(gpt|chatgpt)\b|\bgpt-?\d/,"gpt"],[/\bgemini\b/,"gemini-pro"],[/\bclaude\b/,"claude-sonnet"]];var fi={chatgpt:"gpt",claude:"claude-sonnet",gemini:"gemini-flash",deepseek:"deepseek",mistral:"mistral",grok:"grok",generic:"generic"},vi=[];var wr={...H};function re(e,a="generic"){let i=(e??"").toLowerCase().replace(/\s+/g," ").trim();if(i){for(let[n,r]of vi)if(n.test(i))return H[r];if(a==="gemini"&&/\b(pro|deep think)\b/.test(i))return H["gemini-pro"];if(a==="gemini"&&/\bflash\b/.test(i))return H["gemini-flash"];if(a==="deepseek"&&/\b(deepthink|r1|think)/.test(i))return H["deepseek-reasoning"];for(let[n,r]of ki)if(n.test(i))return H[r]}return H[fi[a]??"generic"]}function ze(e,a){return e===a||ne(e,a,"presence").ok&&Ce(e,a).length===0}function Re(e){return Array.isArray(e)?e.map(Re):!e||typeof e!="object"?e:Object.fromEntries(Object.entries(e).filter(([a])=>a!=="description").map(([a,i])=>[a,Re(i)]))}function ha(e){let a=[];e.system&&a.push(["system prompt",e.system]);for(let i of e.tools??[]){i.description&&a.push([`tool ${i.name}`,i.description]);let n=(r,t)=>{if(Array.isArray(r))r.forEach((o,s)=>n(o,`${t}[${s}]`));else if(r&&typeof r=="object")for(let[o,s]of Object.entries(r))o==="description"&&typeof s=="string"?a.push([`tool ${i.name} ${t}`,s]):n(s,`${t}.${o}`)};n(i.parameters,"params")}for(let[i,n]of(e.examples??[]).entries())a.push([`example ${i+1}`,n]);return e.prompt&&a.push(["next prompt",e.prompt]),a}var de=(e,a)=>`${e} ${a}${e===1?"":"s"}`;function ba(e,a){let i=[],n=ha(e),r=new Map(ha(a)),t={},o=[];for(let[h,f]of n){let _=r.get(h)??"";for(let[T,[P,ge]]of Object.entries(Ae(f,_))){let[C,te]=t[T]??[0,0];t[T]=[C+P,te+ge]}ne(f,_,"presence").ok||o.push(h)}let s=Object.entries(t).map(([h,[f,_]])=>`${_}/${f} ${h}s`);i.push({name:"Facts and constraints kept",passed:o.length===0,detail:o.length?`Lost in: ${o.join(", ")}`:s.length?`Kept ${s.join(", ")}.`:"No numbers, links, code or constraint words to keep."});let l=[];for(let[h,f]of n){let _=Ce(f,r.get(h)??"");_.length&&l.push(`${h}: ${_.join(", ")}`)}i.push({name:"Instructions kept",passed:l.length===0,detail:l.length?`Missing words: ${l.join("; ")}`:`Every content word of ${de(n.length,"text")} is still there; only filler was removed.`});let c=(h=[])=>JSON.stringify(h.map(f=>[f.name,Re(f.parameters??null)])),d=c(e.tools)===c(a.tools);i.push({name:"Tool interface unchanged",passed:d,detail:d?`${de(e.tools?.length??0,"tool")}: same names, parameters, types, required fields and enums.`:"Tool names or parameter schemas differ."});let u=e.messages??[],g=a.messages??[],p=0,y=u.length===g.length&&u.every((h,f)=>{let _=g[f];if(_.role!==h.role)return!1;if(_.content===h.content)return!0;let T=/^\[Same content as message (\d+) above\.\]$/.exec(_.content),P=!!T&&u[Number(T[1])-1]?.content.trim()===h.content.trim();return P&&p++,P});return u.length&&i.push({name:"Conversation unchanged",passed:y,detail:y?`${de(u.length,"message")} in order${p?`; ${de(p,"exact repeat")} replaced by a pointer to the first copy`:", all verbatim"}.`:"Messages were dropped, reordered or edited."}),i.push({name:"Model and examples unchanged",passed:(e.examples?.length??0)===(a.examples?.length??0),detail:"Token Lens never switches the model or drops examples on its own."}),{passed:i.every(h=>h.passed),checks:i}}var S=e=>e.toLocaleString("en-US"),ka=(e,a)=>a?Math.round(e/a*100):0;function fa(e){return`${e.name}
|
|
3282
3282
|
${e.description??""}
|
|
3283
|
-
${e.parameters?JSON.stringify(e.parameters):""}`}function wa(e,a=[]){if(e&&typeof e=="object")for(let[i,n]of Object.entries(e))i==="description"&&typeof n=="string"?a.push(n):wa(n,a);return a}function va(e){let a=(e.description??"").replace(/\s+/g," ").trim().split(/(?<=[.!?])\s/)[0]??"";return a.length>100?`${a.slice(0,97).trimEnd()}...`:a}function
|
|
3283
|
+
${e.parameters?JSON.stringify(e.parameters):""}`}function wa(e,a=[]){if(e&&typeof e=="object")for(let[i,n]of Object.entries(e))i==="description"&&typeof n=="string"?a.push(n):wa(n,a);return a}function va(e){let a=(e.description??"").replace(/\s+/g," ").trim().split(/(?<=[.!?])\s/)[0]??"";return a.length>100?`${a.slice(0,97).trimEnd()}...`:a}function yi(e){let a=e.map(n=>`- ${n.name}${va(n)?`: ${va(n)}`:""}`).join(`
|
|
3284
3284
|
`),i=e.map(n=>n.name);return{tools:[{name:"get_tool_schema",description:`Get the full definition (description and parameters) of a tool before calling it. Available tools:
|
|
3285
|
-
${a}`,parameters:{type:"object",properties:{tool_name:{type:"string",enum:i}},required:["tool_name"]}},{name:"invoke_tool",description:"Call a tool by name with arguments that match the schema from get_tool_schema.",parameters:{type:"object",properties:{tool_name:{type:"string",enum:i},tool_input:{type:"object"}},required:["tool_name","tool_input"]}}],schemas:Object.fromEntries(e.map(n=>[n.name,n]))}}var
|
|
3286
|
-
`))/^\s*```/.test(t)&&(n=!n),(!n&&
|
|
3285
|
+
${a}`,parameters:{type:"object",properties:{tool_name:{type:"string",enum:i}},required:["tool_name"]}},{name:"invoke_tool",description:"Call a tool by name with arguments that match the schema from get_tool_schema.",parameters:{type:"object",properties:{tool_name:{type:"string",enum:i},tool_input:{type:"object"}},required:["tool_name","tool_input"]}}],schemas:Object.fromEntries(e.map(n=>[n.name,n]))}}var wi=8,Si=2e3,_i=/\b\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2})?)?|\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b|\b(?:today is|current (?:date|time)|timestamp)\b|\b(?:user|customer|account|session|request)[ _-]?(?:name|id)\s*[:=]/i;function Ei(e){let a=[],i=[],n=!1;for(let t of e.split(`
|
|
3286
|
+
`))/^\s*```/.test(t)&&(n=!n),(!n&&_i.test(t)?i:a).push(t);return i.length?e.trimEnd().split(`
|
|
3287
3287
|
`).slice(-i.length).join(`
|
|
3288
3288
|
`)===i.join(`
|
|
3289
3289
|
`)?{text:e,moved:[]}:{text:`${a.join(`
|
|
3290
3290
|
`).trimEnd()}
|
|
3291
3291
|
|
|
3292
3292
|
${i.join(`
|
|
3293
|
-
`)}`,moved:i}:{text:e,moved:i}}var
|
|
3294
|
-
`)),n=i===void 0?{type:"object",properties:{},additionalProperties:!0}:Pe(i),r=n.type==="object"?n:{type:"object",properties:{items:n},required:["items"],additionalProperties:!1},t=a.family==="anthropic"?{tools:[{name:"return_result",description:"Return the result.",input_schema:r,strict:!0}],tool_choice:{type:"tool",name:"return_result"}}:a.family==="google"?{generationConfig:{responseMimeType:"application/json",responseSchema:r}}:{response_format:{type:"json_schema",json_schema:{name:"result",strict:!0,schema:r}}};return{fromExample:i!==void 0,setup:t}}var ya=1024;function me(e,a){let i=typeof a=="string"?re(a):a,n=m=>m?I(m,i).tokens:0,r=n(e.system),t=(e.tools??[]).map(m=>({tool:m,tokens:n(fa(m))})),o=t.reduce((m,k)=>m+k.tokens,0),s=(e.examples??[]).map(n),l=s.reduce((m,k)=>m+k,0),c=(e.messages??[]).map(m=>({m,tokens:m.tokens??n(m.content)})),d=c.reduce((m,k)=>m+k.tokens,0),u=n(e.prompt),g=r+o+l,p=g+d+u,y=[{area:"system",label:"System prompt",tokens:r},{area:"tools",label:`Tool definitions (${t.length})`,tokens:o},{area:"examples",label:`Examples (${s.length})`,tokens:l},{area:"history",label:`Conversation (${c.length} messages)`,tokens:d},{area:"prompt",label:"Next prompt",tokens:u}].filter(m=>m.tokens>0),h=c.filter(m=>m.m.role==="user").length,f=h?Math.round(d/h):0,_=i.contextWindow-p,T=f>0?Math.max(0,Math.floor(_/f)):null,P=[],ge=new Set(["prompt-caching","volatile-prefix"]),C=m=>m.saves>0&&P.push({...m,usd:(ge.has(m.id)?ga(m.saves,i):q(m.saves,i)).usd});if(e.system){let m=O(e.system,{profile:i,task:"chat",level:"balanced"});C({id:"system-wording",area:"system",issue:`System prompt is ${S(r)} tokens and is re-sent on every call.`,fix:`Tighten its wording (${m.rules.join(", ")||"filler"}); meaning and constraints are kept.`,risk:"safe",saves:m.saved});let k=_i(e.system);k.moved.length&&g>=ya&&C({id:"volatile-prefix",area:"system",issue:`The system prompt has ${k.moved.length} line${k.moved.length>1?"s":""} that change between calls (${k.moved.map(w=>`"${w.trim().slice(0,40)}"`).join(", ")}) ahead of the rest, so the provider's discount for a repeated opening can't apply.`,fix:"Move those lines to the end of the system prompt (ready to paste below): same content, so the model reads the same facts, and everything before them is billed at the cached price.",risk:"safe",saves:Math.round(g*.9)})}g>=ya&&(i.family==="anthropic"||i.family==="openai"||i.family==="google")&&C({id:"prompt-caching",area:"system",issue:`${S(g)} tokens of system prompt, tools and examples are identical on every call.`,fix:i.family==="anthropic"?"Mark the end of the tools and system prompt with cache_control so repeat calls read it from cache at a fraction of the price. The saving shown is net of the cached-read price; each cache write (when the cache is cold) costs about 25% extra on those tokens.":i.family==="openai"?"Keep this prefix byte-identical and first in every request; OpenAI caches repeated prefixes automatically and bills them at a discount.":"Use context caching for this prefix so repeat calls don't pay full price for it. The saving shown is net of the cached price, before any cache storage charge.",risk:"safe",saves:Math.round(g*.9)});let te=new Set((e.messages??[]).flatMap(m=>m.toolCalls??[]));for(let{tool:m,tokens:k}of t){let z=[m.description??"",...wa(m.parameters)].filter(Boolean).reduce((E,B)=>E+O(B,{profile:i,task:"chat",level:"balanced"}).saved,0);C({id:`tool-wording:${m.name}`,area:"tools",issue:`Tool "${m.name}" definition is ${S(k)} tokens.`,fix:"Tighten the tool and parameter descriptions; names, types and constraints are kept.",risk:"safe",saves:z})}if(t.length>=yi||t.length>=3&&o>=wi){let k=vi(e.tools??[]).tools.reduce((w,z)=>w+n(fa(z)),0);C({id:"lazy-tools",area:"tools",issue:`${t.length} tool definitions (${S(o)} tokens) are sent in full on every call, whichever tools the call uses.`,fix:`Load tool definitions on demand: send a one-line catalog plus get_tool_schema and invoke_tool (${S(k)} tokens) and return each full definition only when the agent asks for it. Same tools and model; the first use of each tool in a run adds one short lookup.`,risk:"safe",saves:o-k})}if(te.size&&h>=3){let m=t.filter(w=>!te.has(w.tool.name)),k=m.reduce((w,z)=>w+z.tokens,0);m.length&&C({id:"unused-tools",area:"tools",issue:`${m.length} of ${t.length} tools were never called in ${h} turns (${m.map(w=>w.tool.name).join(", ")}).`,fix:"Load them only for the tasks that need them, or merge them into fewer tools.",risk:"review",saves:k})}if(s.length>3||l>Math.max(1500,r)){let m=[...s].sort((k,w)=>k-w).slice(0,Math.min(3,s.length));C({id:"examples",area:"examples",issue:`${s.length} examples take ${S(l)} tokens (${ka(l,p)}% of each call).`,fix:"Keep the 2\u20133 most different examples, or shorten each to the part that shows the format.",risk:"review",saves:l-m.reduce((k,w)=>k+w,0)})}if(c.length){let m=new Map,k=0;c.forEach(({m:E,tokens:B},J)=>{let Me=E.content.trim();B>=200&&m.has(Me)?k+=B-10:m.set(Me,J)}),C({id:"duplicate-history",area:"history",issue:"The same large content appears more than once in the conversation.",fix:"Replace repeats with a pointer to the first copy.",risk:"safe",saves:k});let z=c.filter(E=>E.m.role==="tool").filter(E=>E.tokens>2e3);if(z.length){let E=z.reduce((B,J)=>B+J.tokens,0);C({id:"large-tool-results",area:"history",issue:`${z.length} tool results are over 2,000 tokens (${S(E)} in total, ${ka(E,p)}% of the context) and stay in every later call.`,fix:"Have tools return only the fields the agent uses, paginate, or replace old results with a short summary once used.",risk:"review",saves:Math.round(E*.7)})}if(d>i.contextWindow*.5||T!==null&&T<5||d>4e4){let E=c.slice(-6).reduce((B,J)=>B+J.tokens,0);C({id:"summarize-history",area:"history",issue:`History is ${S(d)} tokens and grows ~${S(f)} per turn${T!==null?`; about ${T} turns until the context is full`:""}.`,fix:"Summarize older turns into a short brief and keep the last few verbatim.",risk:"review",saves:Math.max(0,d-E-400)})}}if(e.prompt){let m=O(e.prompt,{profile:i,task:ie(e.prompt).task,level:"balanced"});C({id:"prompt-wording",area:"prompt",issue:`The next prompt is ${S(u)} tokens.`,fix:`Tighten its wording (${m.rules.join(", ")}); code and pasted material are untouched.`,risk:"safe",saves:m.saved})}let R=i.pricing?.longContext;if(R&&i.pricing&&p>=R.aboveTokens*.85){let m=p>R.aboveTokens,k=p*(R.inputPerMTok-i.pricing.inputPerMTok)/1e6,w=[...y].sort((z,E)=>E.tokens-z.tokens)[0];P.push({id:"price-threshold",area:w?.area??"history",issue:m?`This call is ${S(p)} tokens, over ${S(R.aboveTokens)}, so ${i.label} bills the whole request at $${R.inputPerMTok}/M input instead of $${i.pricing.inputPerMTok}/M (\u2248$${k.toFixed(4)} extra per call).`:`This call is ${S(p)} tokens, ${S(R.aboveTokens-p)} below ${S(R.aboveTokens)}; past that, ${i.label} bills the whole request at $${R.inputPerMTok}/M input instead of $${i.pricing.inputPerMTok}/M.`,fix:`The biggest part is ${w?.label??"the conversation"} (${S(w?.tokens??0)} tokens). Keeping the call under ${S(R.aboveTokens)} (cache or trim it, or start from a handoff brief) avoids the higher rate. Dropping content can change the answer, so check the output before you rely on it.`,risk:"review",saves:m?p-R.aboveTokens:0,usd:m?k:null})}return[e.system,e.prompt,...e.examples??[]].some(m=>m&&Ei.test(m))&&P.push({id:"structured-output",area:"output",issue:"The agent asks for JSON in words, so a reply that doesn't parse costs a retry: the whole call again.",fix:`Turn on the provider's strict JSON mode (setup below, schema ${Ci(e,i).fromExample?"taken from your example":"to fill in"}); every reply then matches the schema and no call is spent on a retry. The instructions stay as they are, so the content asked for is the same; check a few outputs once after switching.`,risk:"review",saves:0,usd:null}),P.sort((m,k)=>k.saves-m.saves),{profile:i,total:p,costPerCall:W(p,i),fixedOverhead:g,breakdown:y,growthPerTurn:f,turnsLeft:T,findings:P}}function Sa(e,a,i="balanced"){let n=me(e,a),r=n.profile,t=(u,g=ie(u).task)=>{let p=O(u,{profile:r,task:g,level:i}).text;return ze(u,p)?p:u},o=u=>Array.isArray(u)?u.map(o):!u||typeof u!="object"?u:Object.fromEntries(Object.entries(u).map(([g,p])=>[g,g==="description"&&typeof p=="string"?t(p,"chat"):o(p)])),s={...e};if(e.system&&(s.system=t(e.system,"chat")),e.tools&&(s.tools=e.tools.map(u=>({...u,description:u.description===void 0?void 0:t(u.description,"chat"),parameters:o(u.parameters)}))),e.messages){let u=new Map;s.messages=e.messages.map((g,p)=>{let y=g.content.trim(),h=u.get(y);return h!==void 0&&I(y,r).tokens>=200?{...g,content:`[Same content as message ${h+1} above.]`}:(u.set(y,p),g)})}e.prompt&&(s.prompt=t(e.prompt));let l=new Set(["system-wording","duplicate-history","prompt-wording"]);for(let u of n.findings)(l.has(u.id)||u.id.startsWith("tool-wording:"))&&(u.applied=!0);let c=ba(e,s);if(!c.passed){for(let u of n.findings)u.applied=!1;return{spec:e,report:n,before:n.total,after:n.total,savings:q(0,r),verification:c}}let d=me(s,r).total;return{spec:s,report:n,before:n.total,after:d,savings:q(n.total-d,r),verification:c}}function ji(e){let a={paths:[]};for(let i=0;i<e.length;i++){let n=e[i],r=()=>e[++i];n==="--model"?a.model=r():n==="--base"?a.base=r():n==="--out"?a.out=r():n==="--calls-per-month"?a.callsPerMonth=Number(r()):n==="--fail-above"?a.failAbove=Number(r()):a.paths.push(n)}return a}var Ie=(...e)=>zi("git",e,{encoding:"utf8",maxBuffer:64<<20,stdio:["ignore","pipe","pipe"]});function xi(){let e=process.env.GITHUB_BASE_REF??process.env.CI_MERGE_REQUEST_TARGET_BRANCH_NAME;return e?`origin/${e}`:"HEAD~1"}function Ea(e){let a=e?.type==="function"&&e.function?e.function:e;return!a||typeof a.name!="string"?null:{name:a.name,description:a.description,parameters:a.parameters??a.input_schema??a.inputSchema}}function Aa(e,a){if(!e.endsWith(".json"))return{system:a};let i=JSON.parse(a);if(Array.isArray(i))return{tools:i.map(Ea).filter(t=>!!t)};let n=[i.system,i.instructions,i.system_prompt].map(t=>Array.isArray(t)?t.map(o=>o?.text??"").join(`
|
|
3295
|
-
`):t).find(t=>typeof t=="string"),r=Array.isArray(i.tools)?i.tools.map(Ea).filter(t=>!!t):void 0;return{system:n,tools:r,examples:Array.isArray(i.examples)?i.examples.map(String):void 0,messages:Array.isArray(i.messages)?i.messages:void 0,prompt:typeof i.prompt=="string"?i.prompt:void 0}}var M=e=>e.toLocaleString("en-US"),
|
|
3293
|
+
`)}`,moved:i}:{text:e,moved:i}}var Ai=/\b(?:respond|reply|return|output|answer|format|give)\w*\b[^.\n]{0,60}\bjson\b|\bvalid json\b|\bjson (?:object|format|schema|only)\b/i;function Ci(e){for(let a=0;a<e.length;a++){let i=e[a];if(i!=="{"&&i!=="[")continue;let n=i==="{"?"}":"]",r=0;for(let t=a;t<e.length;t++)if(e[t]===i)r++;else if(e[t]===n&&--r===0){try{let o=JSON.parse(e.slice(a,t+1));if(o&&typeof o=="object")return o}catch{}break}}}function Pe(e){if(Array.isArray(e))return{type:"array",items:e.length?Pe(e[0]):{}};if(e===null)return{type:"null"};if(typeof e=="object"){let a=Object.fromEntries(Object.entries(e).map(([i,n])=>[i,Pe(n)]));return{type:"object",properties:a,required:Object.keys(a),additionalProperties:!1}}return typeof e=="number"?{type:Number.isInteger(e)?"integer":"number"}:typeof e=="boolean"?{type:"boolean"}:{type:"string"}}function Ti(e,a){let i=Ci([e.system,...e.examples??[],e.prompt].filter(Boolean).join(`
|
|
3294
|
+
`)),n=i===void 0?{type:"object",properties:{},additionalProperties:!0}:Pe(i),r=n.type==="object"?n:{type:"object",properties:{items:n},required:["items"],additionalProperties:!1},t=a.family==="anthropic"?{tools:[{name:"return_result",description:"Return the result.",input_schema:r,strict:!0}],tool_choice:{type:"tool",name:"return_result"}}:a.family==="google"?{generationConfig:{responseMimeType:"application/json",responseSchema:r}}:{response_format:{type:"json_schema",json_schema:{name:"result",strict:!0,schema:r}}};return{fromExample:i!==void 0,setup:t}}var ya=1024;function me(e,a){let i=typeof a=="string"?re(a):a,n=m=>m?I(m,i).tokens:0,r=n(e.system),t=(e.tools??[]).map(m=>({tool:m,tokens:n(fa(m))})),o=t.reduce((m,k)=>m+k.tokens,0),s=(e.examples??[]).map(n),l=s.reduce((m,k)=>m+k,0),c=(e.messages??[]).map(m=>({m,tokens:m.tokens??n(m.content)})),d=c.reduce((m,k)=>m+k.tokens,0),u=n(e.prompt),g=r+o+l,p=g+d+u,y=[{area:"system",label:"System prompt",tokens:r},{area:"tools",label:`Tool definitions (${t.length})`,tokens:o},{area:"examples",label:`Examples (${s.length})`,tokens:l},{area:"history",label:`Conversation (${c.length} messages)`,tokens:d},{area:"prompt",label:"Next prompt",tokens:u}].filter(m=>m.tokens>0),h=c.filter(m=>m.m.role==="user").length,f=h?Math.round(d/h):0,_=i.contextWindow-p,T=f>0?Math.max(0,Math.floor(_/f)):null,P=[],ge=new Set(["prompt-caching","volatile-prefix"]),C=m=>m.saves>0&&P.push({...m,usd:(ge.has(m.id)?ga(m.saves,i):q(m.saves,i)).usd});if(e.system){let m=O(e.system,{profile:i,task:"chat",level:"balanced"});C({id:"system-wording",area:"system",issue:`System prompt is ${S(r)} tokens and is re-sent on every call.`,fix:`Tighten its wording (${m.rules.join(", ")||"filler"}); meaning and constraints are kept.`,risk:"safe",saves:m.saved});let k=Ei(e.system);k.moved.length&&g>=ya&&C({id:"volatile-prefix",area:"system",issue:`The system prompt has ${k.moved.length} line${k.moved.length>1?"s":""} that change between calls (${k.moved.map(w=>`"${w.trim().slice(0,40)}"`).join(", ")}) ahead of the rest, so the provider's discount for a repeated opening can't apply.`,fix:"Move those lines to the end of the system prompt (ready to paste below): same content, so the model reads the same facts, and everything before them is billed at the cached price.",risk:"safe",saves:Math.round(g*.9)})}g>=ya&&(i.family==="anthropic"||i.family==="openai"||i.family==="google")&&C({id:"prompt-caching",area:"system",issue:`${S(g)} tokens of system prompt, tools and examples are identical on every call.`,fix:i.family==="anthropic"?"Mark the end of the tools and system prompt with cache_control so repeat calls read it from cache at a fraction of the price. The saving shown is net of the cached-read price; each cache write (when the cache is cold) costs about 25% extra on those tokens.":i.family==="openai"?"Keep this prefix byte-identical and first in every request; OpenAI caches repeated prefixes automatically and bills them at a discount.":"Use context caching for this prefix so repeat calls don't pay full price for it. The saving shown is net of the cached price, before any cache storage charge.",risk:"safe",saves:Math.round(g*.9)});let te=new Set((e.messages??[]).flatMap(m=>m.toolCalls??[]));for(let{tool:m,tokens:k}of t){let z=[m.description??"",...wa(m.parameters)].filter(Boolean).reduce((E,B)=>E+O(B,{profile:i,task:"chat",level:"balanced"}).saved,0);C({id:`tool-wording:${m.name}`,area:"tools",issue:`Tool "${m.name}" definition is ${S(k)} tokens.`,fix:"Tighten the tool and parameter descriptions; names, types and constraints are kept.",risk:"safe",saves:z})}if(t.length>=wi||t.length>=3&&o>=Si){let k=yi(e.tools??[]).tools.reduce((w,z)=>w+n(fa(z)),0);C({id:"lazy-tools",area:"tools",issue:`${t.length} tool definitions (${S(o)} tokens) are sent in full on every call, whichever tools the call uses.`,fix:`Load tool definitions on demand: send a one-line catalog plus get_tool_schema and invoke_tool (${S(k)} tokens) and return each full definition only when the agent asks for it. Same tools and model; the first use of each tool in a run adds one short lookup.`,risk:"safe",saves:o-k})}if(te.size&&h>=3){let m=t.filter(w=>!te.has(w.tool.name)),k=m.reduce((w,z)=>w+z.tokens,0);m.length&&C({id:"unused-tools",area:"tools",issue:`${m.length} of ${t.length} tools were never called in ${h} turns (${m.map(w=>w.tool.name).join(", ")}).`,fix:"Load them only for the tasks that need them, or merge them into fewer tools.",risk:"review",saves:k})}if(s.length>3||l>Math.max(1500,r)){let m=[...s].sort((k,w)=>k-w).slice(0,Math.min(3,s.length));C({id:"examples",area:"examples",issue:`${s.length} examples take ${S(l)} tokens (${ka(l,p)}% of each call).`,fix:"Keep the 2\u20133 most different examples, or shorten each to the part that shows the format.",risk:"review",saves:l-m.reduce((k,w)=>k+w,0)})}if(c.length){let m=new Map,k=0;c.forEach(({m:E,tokens:B},J)=>{let Me=E.content.trim();B>=200&&m.has(Me)?k+=B-10:m.set(Me,J)}),C({id:"duplicate-history",area:"history",issue:"The same large content appears more than once in the conversation.",fix:"Replace repeats with a pointer to the first copy.",risk:"safe",saves:k});let z=c.filter(E=>E.m.role==="tool").filter(E=>E.tokens>2e3);if(z.length){let E=z.reduce((B,J)=>B+J.tokens,0);C({id:"large-tool-results",area:"history",issue:`${z.length} tool results are over 2,000 tokens (${S(E)} in total, ${ka(E,p)}% of the context) and stay in every later call.`,fix:"Have tools return only the fields the agent uses, paginate, or replace old results with a short summary once used.",risk:"review",saves:Math.round(E*.7)})}if(d>i.contextWindow*.5||T!==null&&T<5||d>4e4){let E=c.slice(-6).reduce((B,J)=>B+J.tokens,0);C({id:"summarize-history",area:"history",issue:`History is ${S(d)} tokens and grows ~${S(f)} per turn${T!==null?`; about ${T} turns until the context is full`:""}.`,fix:"Summarize older turns into a short brief and keep the last few verbatim.",risk:"review",saves:Math.max(0,d-E-400)})}}if(e.prompt){let m=O(e.prompt,{profile:i,task:ie(e.prompt).task,level:"balanced"});C({id:"prompt-wording",area:"prompt",issue:`The next prompt is ${S(u)} tokens.`,fix:`Tighten its wording (${m.rules.join(", ")}); code and pasted material are untouched.`,risk:"safe",saves:m.saved})}let R=i.pricing?.longContext;if(R&&i.pricing&&p>=R.aboveTokens*.85){let m=p>R.aboveTokens,k=p*(R.inputPerMTok-i.pricing.inputPerMTok)/1e6,w=[...y].sort((z,E)=>E.tokens-z.tokens)[0];P.push({id:"price-threshold",area:w?.area??"history",issue:m?`This call is ${S(p)} tokens, over ${S(R.aboveTokens)}, so ${i.label} bills the whole request at $${R.inputPerMTok}/M input instead of $${i.pricing.inputPerMTok}/M (\u2248$${k.toFixed(4)} extra per call).`:`This call is ${S(p)} tokens, ${S(R.aboveTokens-p)} below ${S(R.aboveTokens)}; past that, ${i.label} bills the whole request at $${R.inputPerMTok}/M input instead of $${i.pricing.inputPerMTok}/M.`,fix:`The biggest part is ${w?.label??"the conversation"} (${S(w?.tokens??0)} tokens). Keeping the call under ${S(R.aboveTokens)} (cache or trim it, or start from a handoff brief) avoids the higher rate. Dropping content can change the answer, so check the output before you rely on it.`,risk:"review",saves:m?p-R.aboveTokens:0,usd:m?k:null})}return[e.system,e.prompt,...e.examples??[]].some(m=>m&&Ai.test(m))&&P.push({id:"structured-output",area:"output",issue:"The agent asks for JSON in words, so a reply that doesn't parse costs a retry: the whole call again.",fix:`Turn on the provider's strict JSON mode (setup below, schema ${Ti(e,i).fromExample?"taken from your example":"to fill in"}); every reply then matches the schema and no call is spent on a retry. The instructions stay as they are, so the content asked for is the same; check a few outputs once after switching.`,risk:"review",saves:0,usd:null}),P.sort((m,k)=>k.saves-m.saves),{profile:i,total:p,costPerCall:W(p,i),fixedOverhead:g,breakdown:y,growthPerTurn:f,turnsLeft:T,findings:P}}function Sa(e,a,i="balanced"){let n=me(e,a),r=n.profile,t=(u,g=ie(u).task)=>{let p=O(u,{profile:r,task:g,level:i}).text;return ze(u,p)?p:u},o=u=>Array.isArray(u)?u.map(o):!u||typeof u!="object"?u:Object.fromEntries(Object.entries(u).map(([g,p])=>[g,g==="description"&&typeof p=="string"?t(p,"chat"):o(p)])),s={...e};if(e.system&&(s.system=t(e.system,"chat")),e.tools&&(s.tools=e.tools.map(u=>({...u,description:u.description===void 0?void 0:t(u.description,"chat"),parameters:o(u.parameters)}))),e.messages){let u=new Map;s.messages=e.messages.map((g,p)=>{let y=g.content.trim(),h=u.get(y);return h!==void 0&&I(y,r).tokens>=200?{...g,content:`[Same content as message ${h+1} above.]`}:(u.set(y,p),g)})}e.prompt&&(s.prompt=t(e.prompt));let l=new Set(["system-wording","duplicate-history","prompt-wording"]);for(let u of n.findings)(l.has(u.id)||u.id.startsWith("tool-wording:"))&&(u.applied=!0);let c=ba(e,s);if(!c.passed){for(let u of n.findings)u.applied=!1;return{spec:e,report:n,before:n.total,after:n.total,savings:q(0,r),verification:c}}let d=me(s,r).total;return{spec:s,report:n,before:n.total,after:d,savings:q(n.total-d,r),verification:c}}function xi(e){let a={paths:[]};for(let i=0;i<e.length;i++){let n=e[i],r=()=>e[++i];n==="--model"?a.model=r():n==="--base"?a.base=r():n==="--out"?a.out=r():n==="--calls-per-month"?a.callsPerMonth=Number(r()):n==="--fail-above"?a.failAbove=Number(r()):a.paths.push(n)}return a}var Ie=(...e)=>Pi("git",e,{encoding:"utf8",maxBuffer:64<<20,stdio:["ignore","pipe","pipe"]});function Di(){let e=process.env.GITHUB_BASE_REF??process.env.CI_MERGE_REQUEST_TARGET_BRANCH_NAME;return e?`origin/${e}`:"HEAD~1"}function Ea(e){let a=e?.type==="function"&&e.function?e.function:e;return!a||typeof a.name!="string"?null:{name:a.name,description:a.description,parameters:a.parameters??a.input_schema??a.inputSchema}}function Aa(e,a){if(!e.endsWith(".json"))return{system:a};let i=JSON.parse(a);if(Array.isArray(i))return{tools:i.map(Ea).filter(t=>!!t)};let n=[i.system,i.instructions,i.system_prompt].map(t=>Array.isArray(t)?t.map(o=>o?.text??"").join(`
|
|
3295
|
+
`):t).find(t=>typeof t=="string"),r=Array.isArray(i.tools)?i.tools.map(Ea).filter(t=>!!t):void 0;return{system:n,tools:r,examples:Array.isArray(i.examples)?i.examples.map(String):void 0,messages:Array.isArray(i.messages)?i.messages:void 0,prompt:typeof i.prompt=="string"?i.prompt:void 0}}var M=e=>e.toLocaleString("en-US"),Ni=e=>e>0?`+${M(e)}`:M(e),pe=e=>e===null?"n/a":e>0?`+${K(e)}`:K(e),Ca=(e,a)=>e===null?null:e*a;function Li(e,a,i,n){let r=d=>{if(d===null)return{tokens:0,usd:0};let u=me(Aa(e,d),n);return{tokens:u.total,usd:u.costPerCall}},t=r(a),o=r(i),s=0,l=0,c=[];if(i!==null){let d=Sa(Aa(e,i),n);s=d.savings.tokens,l=d.savings.usd;for(let u of d.report.findings.slice(0,3))u.saves>0&&c.push(`${u.issue} ${u.fix} (saves ${M(u.saves)} tokens${u.usd?`, ${K(u.usd)}`:""} per call${u.risk==="review"?", review first":""})`)}return{path:e,before:t.tokens,after:o.tokens,usdBefore:t.usd,usdAfter:o.usd,canSave:s,canSaveUsd:l,fixes:c}}function Oi(e,a,i){let n=e.reduce((s,l)=>s+l.after-l.before,0),r=e.some(s=>s.usdAfter===null||s.usdBefore===null)?null:e.reduce((s,l)=>s+l.usdAfter-l.usdBefore,0),t=["### Token Lens: agent cost check","",`This change ${n>0?"adds":n<0?"removes":"keeps"} **${M(Math.abs(n))} input tokens per call** (${pe(r)} per call, ${pe(Ca(r,1e3))} per 1,000 calls`+(i?`, ${pe(Ca(r,i))} per month at ${M(i)} calls`:"")+`) on ${a.label}.`,"","| File | Before | After | Change | Per call | Same output, fewer tokens |","| --- | ---: | ---: | ---: | ---: | ---: |"];for(let s of e){let l=s.usdAfter===null||s.usdBefore===null?null:s.usdAfter-s.usdBefore;t.push(`| \`${s.path}\` | ${M(s.before)} | ${M(s.after)} | ${Ni(s.after-s.before)} | ${pe(l)} | `+(s.canSave?`\u2212${M(s.canSave)} (${K(s.canSaveUsd)})`:"\u2013")+" |")}let o=e.filter(s=>s.fixes.length);if(o.length){t.push("","**Where to trim**","");for(let s of o)for(let l of s.fixes)t.push(`- \`${s.path}\`: ${l}`)}return t.push("",`<sub>Input tokens re-sent on every call (system prompt, tools, examples), counted for ${a.label} at ${K(a.pricing?a.pricing.inputPerMTok:null)} per 1M input tokens. "Same output" fixes keep every fact, number, constraint and tool interface. Computed in this CI job; nothing was sent anywhere. [Token Lens](https://www.npmjs.com/package/@kunal_dutt/token-lens)</sub>`),t.join(`
|
|
3296
3296
|
`)+`
|
|
3297
|
-
`}function
|
|
3298
|
-
`).filter(Boolean),t=Ie("rev-parse","--show-toplevel").trim();if(!r.length)return console.log("token-lens: no agent files changed."),0;let o=r.map(c=>{let d=null;try{d=Ie("show",`${n}:${c}`)}catch{}let u=
|
|
3297
|
+
`}function Rt(e){let a=xi(e);if(!a.model||!a.paths.length)return console.error("usage: token-lens check-agent --model <the model your agent uses> [--base <ref>] [--out file.md] <pathspec>..."),2;let i=re(a.model),n=a.base??Di(),r=Ie("diff","--name-only","--diff-filter=ACMRD",n,"--",...a.paths.map(c=>`:(glob)${c}`)).split(`
|
|
3298
|
+
`).filter(Boolean),t=Ie("rev-parse","--show-toplevel").trim();if(!r.length)return console.log("token-lens: no agent files changed."),0;let o=r.map(c=>{let d=null;try{d=Ie("show",`${n}:${c}`)}catch{}let u=Ii(_a(t,c))?Mi(_a(t,c),"utf8"):null;try{return Li(c,d,u,i)}catch(g){return console.error(`token-lens: skipped ${c}: ${g.message}`),null}}).filter(c=>!!c),s=Oi(o,i,a.callsPerMonth);a.out?ji(a.out,s):process.stdout.write(s);let l=o.reduce((c,d)=>c+d.after-d.before,0);return a.failAbove!==void 0&&l>a.failAbove?(console.error(`token-lens: this change adds ${M(l)} tokens per call, above --fail-above ${M(a.failAbove)}.`),1):0}export{Li as checkFile,Rt as main,Aa as parseSpec,Oi as render};
|