npm - @librechat/agents - Versions diffs - 3.6.6 → 3.6.7 - Mend

@librechat/agents 3.6.6 → 3.6.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (6) hide show

package/dist/cjs/utils/tokens.cjs +9 -5
package/dist/cjs/utils/tokens.cjs.map +1 -1
package/dist/esm/utils/tokens.mjs +9 -5
package/dist/esm/utils/tokens.mjs.map +1 -1
package/package.json +1 -1
package/src/utils/tokens.ts +21 -15

package/dist/cjs/utils/tokens.cjs CHANGED Viewed

@@ -51,6 +51,8 @@ const TIMED_MEDIA_TYPES = new Set([
 const MAX_STRUCTURED_TOKENIZATION_CHARS = 2e5;
 const MAX_TOKENS_PER_OMITTED_STRUCTURED_CHAR = 4;
 const MAX_STRUCTURED_SAFETY_INSPECTION_WORK = 1e4;
+const MAX_TEXT_TOKENIZATION_CHARS = 8192;
+const TEXT_TOKEN_CHUNK_BOUNDARY_SAFETY_TOKENS = 4;
 function extractImageDimensions(base64Data) {
 	const raw = base64Data.startsWith("data:") ? base64Data.slice(base64Data.indexOf(",") + 1) : base64Data;
 	if (raw.length < 32) return null;
@@ -383,11 +385,13 @@ function hasUnsafeStructuredSerialization(value) {
 	return false;
 }
 function getBoundedTextTokenCount(value, getTokenCount) {
-	const preview = value.length > MAX_STRUCTURED_TOKENIZATION_CHARS ? value.slice(0, MAX_STRUCTURED_TOKENIZATION_CHARS) : value;
-	const previewTokens = ensureSafeTokenMeasurement(getTokenCount(preview), "tokenizer");
-	const omittedChars = value.length - preview.length;
-	if (omittedChars <= 0) return previewTokens;
-	return Math.min(Number.MAX_SAFE_INTEGER, previewTokens + omittedChars * MAX_TOKENS_PER_OMITTED_STRUCTURED_CHAR);
+	let numTokens = 0;
+	for (let offset = 0; offset < value.length; offset += MAX_TEXT_TOKENIZATION_CHARS) {
+		const chunkTokens = ensureSafeTokenMeasurement(getTokenCount(value.slice(offset, offset + MAX_TEXT_TOKENIZATION_CHARS)), "tokenizer");
+		const boundaryTokens = offset > 0 ? TEXT_TOKEN_CHUNK_BOUNDARY_SAFETY_TOKENS : 0;
+		numTokens = Math.min(Number.MAX_SAFE_INTEGER, numTokens + chunkTokens + boundaryTokens);
+	}
+	return numTokens;
 }
 function getBoundedStructuredTokenCount(value, getTokenCount) {
 	const serialized = require_toolContent.serializeStructuredValueBounded(value, MAX_STRUCTURED_TOKENIZATION_CHARS);

package/dist/cjs/utils/tokens.cjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"tokens.cjs","names":["Tokenizer","serializeStructuredValueBounded","isAtomicToolContentBlock","isComputerCallOutputContent"],"sources":["../../../src/utils/tokens.ts"],"mappings":";;;;;AAmBA,IAAa,8BAAb,cAAiD,MAAM;CACrD,OAAgB;CAChB;CACA;CAEA,YAAY,EACV,QACA,QAIC;EACD,MACE,KAAK,UAAU;GACb,MAAM;GACN;GACA;EACF,CAAC,CACH;EACA,KAAK,OAAO;EACZ,KAAK,SAAS;EACd,KAAK,OAAO;CACd;AACF;AAEA,SAAS,2BAA2B,OAAe,MAAsB;CACvE,IACE,CAAC,OAAO,cAAc,KAAK,KAC3B,QAAQ,KACR,SAAS,OAAO,kBAEhB,MAAM,IAAI,4BAA4B;EACpC,QAAQ;EACR;CACF,CAAC;CAEH,OAAO;AACT;AAGA,MAAM,6BAA6B;AAEnC,MAAM,0BAA0B;AAEhC,MAAM,0BAA0B;AAEhC,MAAM,yBAAyB;AAE/B,MAAM,+BAA+B;AAIrC,MAAa,4BAA4B;AAMzC,MAAM,gCAAgC;AAEtC,MAAM,6BAA6B;AAInC,MAAM,4BAA4B;AAElC,MAAM,+BAA+B;AAUrC,MAAM,0BAA0B;AAEhC,MAAM,0BAA0B;AAEhC,MAAM,yBAAyB;AAC/B,MAAM,yBAAyB;AAE/B,MAAM,4BAA4B;AAClC,MAAM,4BAA4B;AAGlC,MAAM,oBAAoB,IAAI,IAAI;CAChC;CACA;CACA;CACA;CACA;AACF,CAAC;AAMD,MAAM,oCAAoC;AAE1C,MAAM,yCAAyC;AAC/C,MAAM,wCAAwC;AAO9C,SAAgB,uBACd,YAC0C;CAC1C,MAAM,MAAM,WAAW,WAAW,OAAO,IACrC,WAAW,MAAM,WAAW,QAAQ,GAAG,IAAI,CAAC,IAC5C;CAEJ,IAAI,IAAI,SAAS,IACf,OAAO;CAGT,MAAM,QAAQ,IAAI,WAAW,OAAO,KAAK,IAAI,MAAM,GAAG,EAAE,GAAG,QAAQ,CAAC;CAEpE,IAAI,MAAM,OAAO,OAAQ,MAAM,OAAO,IAMpC,OAAO;EAAE,OAHN,MAAM,OAAO,KAAO,MAAM,OAAO,KAAO,MAAM,OAAO,IAAK,MAAM;EAGnD,QADb,MAAM,OAAO,KAAO,MAAM,OAAO,KAAO,MAAM,OAAO,IAAK,MAAM;CAC5C;CAGzB,IAAI,MAAM,OAAO,OAAQ,MAAM,OAAO,KAAM;EAE1C,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,SAAS,GAAG,KACpC,IACE,MAAM,OAAO,QACZ,MAAM,IAAI,OAAO,OAAQ,MAAM,IAAI,OAAO,MAC3C;GACA,MAAM,SAAU,MAAM,IAAI,MAAM,IAAK,MAAM,IAAI;GAE/C,OAAO;IAAE,OADM,MAAM,IAAI,MAAM,IAAK,MAAM,IAAI;IAC9B;GAAO;EACzB;EAEF,OAAO;CACT;CAEA,IAAI,MAAM,OAAO,MAAQ,MAAM,OAAO,MAAQ,MAAM,OAAO,IAIzD,OAAO;EAAE,OAFK,MAAM,KAAM,MAAM,MAAM;EAEtB,QADD,MAAM,KAAM,MAAM,MAAM;CAChB;CAGzB,IACE,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,QAAQ,MACd,MAAM,QAAQ,IACd;EAEA,IAAI,MAAM,SAAS,IAGjB,OAAO;GAAE,QAFM,MAAM,MAAO,MAAM,OAAO,KAAM;GAE/B,SADA,MAAM,MAAO,MAAM,OAAO,KAAM;EACzB;EAEzB,OAAO;CACT;CAEA,OAAO;AACT;AAGA,SAAgB,6BACd,OACA,QACQ;CACR,OAAO,KAAK,IACV,4BACA,KAAK,KAAM,QAAQ,SAAU,uBAAuB,CACtD;AACF;AAGA,SAAgB,0BACd,OACA,QACA,SAAiB,QACT;CACR,IAAI,WAAW,OACb,OAAO;CAKT,OAAO,0BAFL,KAAK,KAAK,QAAQ,sBAAsB,IACxC,KAAK,KAAK,SAAS,sBAAsB,IACF;AAC3C;AAOA,SAAgB,yBACd,OACA,UACQ;CACR,IAAI;CAEJ,IACE,MAAM,SAAA,eACN,MAAM,SAAS,eACf,MAAM,SAAS,yBACf,MAAM,SAAS,eACf;EACA,MAAM,WAAW,MAAM;EACvB,MAAM,MAAM,OAAO,aAAa,WAAW,WAAW,UAAU;EAChE,IAAI,OAAO,QAAQ,YAAY,IAAI,WAAW,OAAO,GACnD,aAAa;OACR,IAAI,OAAO,MAAM,YAAY,YAAY,MAAM,YAAY,IAChE,OAAO,MAAM,WAAW,QACpB,0BACA;OAEJ,OAAO;CAEX,OAAO,IAAI,MAAM,SAAS,SAAS;EACjC,MAAM,SAAS,MAAM;EACrB,IAAI,QAAQ,SAAS,YAAY,OAAO,OAAO,SAAS,UACtD,aAAa,OAAO;OAEpB,OAAO;CAEX,OACE,OAAO;CAGT,MAAM,OAAO,uBAAuB,UAAU;CAC9C,IAAI,QAAQ,MACV,OAAO;CAGT,IAAI,aAAa,UACf,OAAO,6BAA6B,KAAK,OAAO,KAAK,MAAM;CAE7D,OAAO,0BAA0B,KAAK,OAAO,KAAK,MAAM;AAC1D;AAWA,SAAgB,4BACd,OACA,UACA,eACQ;CACR,MAAM,mBACJ,aAAa,WACT,gCACA;CAGN,MAAM,aAAa,MAAM;CACzB,IAAI,OAAO,eAAe,UAAU;EAClC,MAAM,YAAa,MAAM,aAAoC,GAAA,CAAI,MAC/D,GACF,CAAC,CAAC;EAEF,IAAI,eAAe,UAAU,OAAO,MAAM,SAAS,UACjD,OAAO,cAAc,MAAM,IAAc;EAG3C,IAAI,eAAe,YAAY,OAAO,MAAM,SAAS,UAAU;GAC7D,IAAI,aAAa,qBAAqB,aAAa,IAKjD,OAJqB,KAAK,IACxB,GACA,KAAK,KAAM,MAAM,KAAgB,SAAS,yBAAyB,CAEnD,IAAI;GAGxB,IAAI,SAAS,WAAW,QAAQ,GAC9B,OAAO,yBACL;IACE,GAAG;IACH,MAAM;IACN,QAAQ;KAAE,MAAM;KAAU,MAAM,MAAM;IAAK;GAC7C,GACA,QACF;GAEF,OAAO,cAAc,MAAM,IAAc;EAC3C;EAEA,IAAI,eAAe,OACjB,OAAO;EAGT,OAAO;CACT;CAGA,MAAM,SAAS,MAAM;CASrB,IAAI,UAAU,MACZ,OAAO;CAGT,IAAI,OAAO,SAAS,UAAU,OAAO,OAAO,SAAS,UACnD,OAAO,cAAc,OAAO,IAAI;CAGlC,IAAI,OAAO,SAAS,YAAY,OAAO,OAAO,SAAS,UAAU;EAC/D,MAAM,aAAa,OAAO,cAAc,GAAA,CAAI,MAAM,GAAG,CAAC,CAAC;EACvD,IAAI,cAAc,qBAAqB,cAAc,IAKnD,OAJqB,KAAK,IACxB,GACA,KAAK,KAAK,OAAO,KAAK,SAAS,yBAAyB,CAExC,IAAI;EAExB,IAAI,UAAU,WAAW,QAAQ,GAC/B,OAAO,yBACL;GAAE,MAAM;GAAS,QAAQ;IAAE,MAAM;IAAU,MAAM,OAAO;GAAK;EAAE,GAC/D,QACF;EAEF,OAAO,cAAc,OAAO,IAAI;CAClC;CAEA,IAAI,OAAO,SAAS,OAClB,OAAO;CAIT,IAAI,OAAO,SAAS,aAAa,MAAM,QAAQ,OAAO,OAAO,GAAG;EAC9D,IAAI,QAAQ;EACZ,KAAK,MAAM,SAAS,OAAO,SACzB,IAAI,SAAS,QAAQ,OAAO,UAAU,YAAY,UAAU,OAAO;GACjE,MAAM,aAAa;GACnB,IAAI,WAAW,SAAS,SACtB,SAAS,yBAAyB,YAAY,QAAQ;EAE1D;EAEF,OAAO,QAAQ,IAAI,QAAQ;CAC7B;CAEA,OAAO;AACT;AAKA,SAAS,iBAAiB,OAAmC;CAC3D,IAAI,OAAO,UAAU,YAAY,MAAM,WAAW,GAChD,OAAO;CAET,IAAI,MAAM,WAAW,OAAO,GAAG;EAC7B,MAAM,QAAQ,MAAM,QAAQ,GAAG;EAC/B,OAAO,QAAQ,IAAI,IAAI,KAAK,OAAQ,MAAM,SAAS,QAAQ,KAAK,IAAK,CAAC;CACxE;CACA,IAAI,uBAAuB,KAAK,KAAK,GACnC,OAAO;CAET,OAAO,KAAK,MAAO,MAAM,SAAS,IAAK,CAAC;AAC1C;AAEA,SAAS,iBACP,OACA,gBACA,iBACA,aACQ;CACR,IAAI,SAAS,GACX,OAAO;CAET,OAAO,KAAK,IACV,iBACA,KAAK,KAAM,QAAQ,iBAAkB,eAAe,CACtD;AACF;AAMA,SAAS,qBAAqB,OAAwC;CACpE,MAAM,OAAO,MAAM;CACnB,IAAI,OAAO,SAAS,UAClB,OAAO,iBAAiB,IAAI;CAE9B,IAAI,gBAAgB,YAClB,OAAO,KAAK;CAEd,IAAI,OAAO,MAAM,QAAQ,UACvB,OAAO,iBAAiB,MAAM,GAAG;CAEnC,MAAM,SAAU,MAAM,SAAS,MAAM;CAGrC,IAAI,UAAU,MAAM;EAClB,IAAI,OAAO,OAAO,SAAS,UACzB,OAAO,iBAAiB,OAAO,IAAI;EAErC,IAAI,OAAO,OAAO,QAAQ,UACxB,OAAO,iBAAiB,OAAO,GAAG;EAEpC,IAAI,OAAO,QAAQ,iBAAiB,YAClC,OAAO,OAAO,OAAO,MAAM;CAE/B;CACA,OAAO;AACT;AAMA,SAAS,eACP,OAC0B;CAC1B,MAAM,OACJ,OAAO,MAAM,SAAS,WAAW,uBAAuB,MAAM,IAAI,IAAI;CACxE,IAAI,SAAS,iBAAiB,SAAS,SACrC,OAAO;CAET,IAAI,SAAS,eAAe,SAAS,SACnC,OAAO;CAET,MAAM,YAAY,iBAAiB,KAAK;CACxC,MAAM,OAAO,SAAS,WAAW,aAAa,OAAO,YAAY;CACjE,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,OAAO;AACT;AAEA,SAAS,iBAAiB,OAAoD;CAC5E,IAAI,OAAO,MAAM,aAAa,UAC5B,OAAO,uBAAuB,MAAM,QAAQ;CAE9C,IAAI,OAAO,MAAM,cAAc,UAC7B,OAAO,uBAAuB,MAAM,SAAS;AAGjD;AAEA,SAAS,uBAAuB,UAA0B;CACxD,OAAO,SAAS,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,YAAY;AACnD;AAKA,SAAS,iBAAiB,MAAuB;CAC/C,MAAM,iBAAiB,uBAAuB,IAAI;CAClD,OACE,kBAAkB,IAAI,cAAc,KACpC,eAAe,WAAW,QAAQ,KAClC,eAAe,WAAW,QAAQ;AAEtC;AAGA,SAAS,gCAAgC,MAAuB;CAC9D,IAAI,OAAO,SAAS,UAAU;EAC5B,IAAI,iBAAiB,IAAI,KAAK,GAC5B,OAAO;EAET,OAAO,KAAK,IAAI,8BAA8B,KAAK,MAAM;CAC3D;CACA,IAAI,gBAAgB,eAAe,YAAY,OAAO,IAAI,GACxD,OAAO,KAAK,IACV,8BACA,KAAK,KAAM,KAAK,aAAa,IAAK,CAAC,CACrC;CAEF,OAAO;AACT;AAMA,SAAS,+BACP,OACA,UACA,eACoB;CACpB,MAAM,YAAY,iBAAiB,KAAK;CACxC,IAAI,MAAM,SAAS,WAAW,aAAa,MACzC;CAGF,MAAM,WAAW;CACjB,MAAM,cACJ,MAAM,SAAS,QAAQ,OAAO,MAAM,UAAU,WACzC,MAAM,QACP,KAAA;CACN,MAAM,OACJ,MAAM,QAAQ,MAAM,SAAS,aAAa,QAAQ,aAAa;CACjE,MAAM,YACJ,MAAM,OACN,MAAM,WACN,MAAM,UACN,aAAa,OACb,aAAa,WACb,aAAa;CAEf,IAAI,SAAS,WAAW,QAAQ,GAAG;EACjC,IAAI;EACJ,IAAI,OAAO,SAAS,UAClB,eAAe;OACV,IAAI,gBAAgB,eAAe,YAAY,OAAO,IAAI,GAAG;GAClE,MAAM,OACJ,gBAAgB,cACZ,IAAI,WAAW,IAAI,IACnB,IAAI,WAAW,KAAK,QAAQ,KAAK,YAAY,KAAK,UAAU;GAClE,eAAe,OAAO,KAAK,KAAK,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS,QAAQ;EACpE;EAEA,IAAI,gBAAgB,QAAQ,aAAa,SAAS,GAChD,OAAO,yBACL;GACE,MAAM;GACN,QAAQ;IAAE,MAAM;IAAU,MAAM;GAAa;EAC/C,GACA,QACF;EAEF,OAAO,yBACL;GACE,MAAM;GACN,WAAW,EAAE,KAAK,OAAO,cAAc,WAAW,YAAY,GAAG;EACnE,GACA,QACF;CACF;CAEA,IAAI,aAAa,mBAAmB;EAClC,IAAI,OAAO,SAAS,YAAY,KAAK,SAAS,GAAG;GAC/C,MAAM,QAAQ,KAAK,WAAW,OAAO,IAAI,KAAK,QAAQ,GAAG,IAAI;GAE7D,OAAO,4BACL;IACE,MAAM;IACN,aAAa;IACb,WAAW;IACX,MANe,SAAS,IAAI,KAAK,MAAM,QAAQ,CAAC,IAAI;GAOtD,GACA,UACA,aACF;EACF;EACA,IAAI,gBAAgB,eAAe,YAAY,OAAO,IAAI,GAAG;GAC3D,MAAM,uBAAuB,KAAK,KAAM,KAAK,aAAa,IAAK,CAAC;GAChE,MAAM,mBACJ,aAAa,WACT,gCACA;GACN,OACE,KAAK,IACH,GACA,KAAK,KAAK,uBAAuB,yBAAyB,CAC5D,IAAI;EAER;EACA,OAAO;CACT;CAEA,IAAI,SAAS,WAAW,QAAQ,KAAK,SAAS,WAAW,QAAQ,GAC/D;CAEF,OAAO,gCAAgC,IAAI;AAC7C;AAaA,SAAgB,8BACd,OACQ;CACR,MAAM,OAAO,eAAe,KAAK;CACjC,IAAI,QAAQ,MACV,OAAO;CAET,IAAI;CACJ,IAAI,MAAM,SAAS,eAAe;EAChC,MAAM,QAAQ,MAAM;EACpB,QAAQ,iBAAiB,OAAO,IAAI;CACtC,OAAO,IAAI,MAAM,SAAS,aAAa;EACrC,MAAM,QAAQ,MAAM;EACpB,QAAQ,iBAAiB,OAAO,UAAU,WAAW,QAAQ,OAAO,GAAG;CACzE,OACE,QAAQ,qBAAqB,KAAK;CAEpC,IAAI,SAAS,SACX,OAAO,iBACL,OACA,wBACA,yBACA,yBACF;CAEF,OAAO,iBACL,OACA,wBACA,yBACA,yBACF;AACF;AAEA,MAAM,aAAuD,CAAC;AAE9D,eAAe,aACb,WAAyB,cACL;CACpB,MAAM,SAAS,WAAW;CAC1B,IAAI,QACF,OAAO;CAMT,MAAM,WAAW,IAAIA,aAAAA,UAHnB,aAAa,WACT,MAAM,OAAO,kCACb,MAAM,OAAO,mCACgB;CACnC,WAAW,YAAY;CACvB,OAAO;AACT;AAEA,SAAgB,iBAAiB,OAA6B;CAC5D,IAAI,MAAM,YAAY,CAAC,CAAC,SAAS,QAAQ,GACvC,OAAO;CAET,OAAO;AACT;AAMA,SAAS,4BACP,YACS;CACT,IAAI,WAAW,aAAa,GAC1B,OAAO;CAET,WAAW;CACX,OAAO;AACT;AAEA,SAAS,gBACP,OACA,YACS;CACT,IAAI,UAAyB;CAC7B,MAAM,uBAAO,IAAI,IAAY;CAC7B,IAAI;EACF,OAAO,WAAW,MAAM;GACtB,KAAA,GAAA,gBAAA,QAAA,CAAY,OAAO,GACjB,OAAO;GAET,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,4BAA4B,UAAU,GAC9D,OAAO;GAET,KAAK,IAAI,OAAO;GAChB,MAAM,aAAa,OAAO,yBAAyB,SAAS,QAAQ;GACpE,IAAI,cAAc,MAChB,OACG,WAAW,cAAc,OAAO,WAAW,UAAU,cACtD,EAAE,WAAW;GAGjB,UAAU,OAAO,eAAe,OAAO;EACzC;CACF,QAAQ;EACN,OAAO;CACT;CACA,OAAO;AACT;AAEA,MAAM,8BAA8B,OAAO,yBACzC,YAAY,WACZ,YACF,CAAC,EAAE;AAMH,SAAS,oBAAoB,OAAwB;CACnD,IAAI;EACF,IAAI,YAAY,OAAO,KAAK,GAC1B,OAAO;CAEX,QAAQ;EACN,OAAO;CACT;CACA,IAAI,+BAA+B,MACjC,OAAO;CAET,IAAI;EACF,4BAA4B,KAAK,KAAK;EACtC,OAAO;CACT,QAAQ;EACN,OAAO;CACT;AACF;AAMA,SAAS,2BACP,OACA,KACA,YACS;CACT,MAAM,OAAO,IAAI,IAAY,CAAC,KAAK,CAAC;CACpC,IAAI;CACJ,IAAI;EACF,UAAU,OAAO,eAAe,KAAK;CACvC,QAAQ;EACN,OAAO;CACT;CAEA,OAAO,WAAW,MAAM;EACtB,KAAA,GAAA,gBAAA,QAAA,CAAY,OAAO,GACjB,OAAO;EAET,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,4BAA4B,UAAU,GAC9D,OAAO;EAET,KAAK,IAAI,OAAO;EAChB,IAAI;EACJ,IAAI;GACF,aAAa,OAAO,yBAAyB,SAAS,GAAG;EAC3D,QAAQ;GACN,OAAO;EACT;EACA,IAAI,cAAc,MAChB,OAAO,EAAE,WAAW;EAEtB,IAAI;GACF,UAAU,OAAO,eAAe,OAAO;EACzC,QAAQ;GACN,OAAO;EACT;CACF;CACA,OAAO;AACT;AAQA,SAAgB,iCAAiC,OAAyB;CACxE,IAAI,OAAO,UAAU,UACnB,OAAO;CAET,IAAI,SAAS,QAAQ,OAAO,UAAU,UACpC,OAAO;CAET,KAAA,GAAA,gBAAA,QAAA,CAAY,KAAK,GACf,OAAO;CAET,IAAI,oBAAoB,KAAK,GAC3B,OAAO;CAGT,MAAM,QAAkB,CAAC,KAAK;CAC9B,MAAM,uBAAO,IAAI,IAAY;CAC7B,MAAM,aAAyC,EAC7C,WAAW,sCACb;CACA,OAAO,MAAM,SAAS,GAAG;EACvB,MAAM,UAAU,MAAM,IAAI;EAC1B,KAAA,GAAA,gBAAA,QAAA,CAAY,OAAO,GACjB,OAAO;EAET,IAAI,KAAK,IAAI,OAAO,GAClB,OAAO;EAET,KAAK,IAAI,OAAO;EAChB,IAAI,CAAC,4BAA4B,UAAU,GACzC,OAAO;EAGT,IAAI;GACF,IAAI,gBAAgB,SAAS,UAAU,GACrC,OAAO;GAET,KAAK,MAAM,OAAO,SAAS;IACzB,IAAI,CAAC,4BAA4B,UAAU,GACzC,OAAO;IAET,MAAM,aAAa,OAAO,yBAAyB,SAAS,GAAG;IAC/D,IAAI,cAAc,MAAM;KACtB,IAAI,2BAA2B,SAAS,KAAK,UAAU,GACrD,OAAO;KAET;IACF;IACA,IAAI,WAAW,eAAe,MAC5B;IAEF,IAAI,EAAE,WAAW,aACf,OAAO;IAET,IAAI,WAAW,YAAY;KACzB,IAAI,OAAO,WAAW,UAAU,UAC9B,OAAO;KAET,IACE,WAAW,SAAS,QACpB,OAAO,WAAW,UAAU,YAC5B,oBAAoB,WAAW,KAAK,GAEpC,OAAO;KAET,IACE,WAAW,SAAS,QACpB,OAAO,WAAW,UAAU,UAE5B,MAAM,KAAK,WAAW,KAAK;IAE/B;GACF;EACF,QAAQ;GACN,OAAO;EACT;CACF;CACA,OAAO;AACT;AAEA,SAAS,yBACP,OACA,eACQ;CACR,MAAM,UACJ,MAAM,SAAS,oCACX,MAAM,MAAM,GAAG,iCAAiC,IAChD;CACN,MAAM,gBAAgB,2BACpB,cAAc,OAAO,GACrB,WACF;CACA,MAAM,eAAe,MAAM,SAAS,QAAQ;CAC5C,IAAI,gBAAgB,GAClB,OAAO;CAET,OAAO,KAAK,IACV,OAAO,kBACP,gBAAgB,eAAe,sCACjC;AACF;AAEA,SAAS,+BACP,OACA,eACQ;CACR,MAAM,aAAaC,oBAAAA,gCACjB,OACA,iCACF;CACA,MAAM,gBAAgB,2BACpB,cAAc,WAAW,OAAO,GAChC,WACF;CACA,IAAI,CAAC,WAAW,WACd,OAAO;CAKT,IACE,CAAC,OAAO,cAAc,WAAW,aAAa,KAC9C,WAAW,iBAAiB,OAAO,kBAEnC,OAAO;CAGT,MAAM,eAAe,KAAK,IACxB,GACA,WAAW,gBAAgB,WAAW,QAAQ,MAChD;CACA,OAAO,KAAK,IACV,OAAO,kBACP,gBAAgB,eAAe,sCACjC;AACF;AAEA,SAAgB,wBACd,SACA,eACA,WAAyB,cACjB;CACR,MAAM,mBAAmB;CACzB,MAAM,aAAa,SACjB,yBAAyB,MAAM,aAAa;CAC9C,KAAA,GAAA,gBAAA,QAAA,CAAY,OAAO,GACjB,MAAM,IAAI,4BAA4B;EACpC,QAAQ;EACR,MAAM;CACR,CAAC;CAYH,MAAM,yCAAyB,IAAI,IAAY;CAE/C,MAAM,gBAAgB,OAAgB,SAAuB;EAC3D,IAAI,SAAS,QAAQ,OAAO,UAAU,aAAA,GAAA,gBAAA,QAAA,CAAoB,KAAK,GAC7D,MAAM,IAAI,4BAA4B;GACpC,QAAQ;GACR;EACF,CAAC;EAEH,IAAI,MAAM,QAAQ,KAAK,GACrB,KAAK,IAAI,QAAQ,GAAG,QAAQ,MAAM,QAAQ,SAAS;GACjD,MAAM,MAAM,MAAM;GAClB,IACE,OAAO,QAAQ,YACf,OAAO,QAAQ,YACf,OAAO,QAAQ,WACf;IACA,aAAa,KAAK,GAAG,KAAK,GAAG,MAAM,EAAE;IACrC;GACF;GACA,MAAM,OAAO;GACb,IAAI,QAAQ,QAAQ,OAAO,SAAS,UAClC;GAEF,KAAA,GAAA,gBAAA,QAAA,CAAY,IAAI,GACd,MAAM,IAAI,4BAA4B;IACpC,QAAQ;IACR,MAAM,GAAG,KAAK,GAAG,MAAM;GACzB,CAAC;GAEH,IAAI,OAAO,KAAK,SAAS,UAAU;IACjC,aAAa,+BAA+B,MAAM,SAAS;IAC3D;GACF;GACA,IAAI,KAAK,SAAA,eAAmC,KAAK,SAAS,YAAY;IACpE,MAAM,WACJ,OAAO,KAAK,OAAO,WAAW,KAAK,KAAK,KAAK,WAAW;IAC1D,IAAI,OAAO,aAAa,YAAY,SAAS,SAAS,GACpD,uBAAuB,IAAI,QAAQ;GAEvC;GACA,IAAI,KAAK,SAAA,SACP;GAGF,IACEC,oBAAAA,yBAAyB,IAAI,MAC5B,KAAK,SAAA,eACJ,KAAK,SAAS,eACd,KAAK,SAAS,WACd,KAAK,SAAS,yBACd,KAAK,SAAS,gBAChB;IACA,aAAa,KAAK,KAChB,yBAAyB,MAAM,QAAQ,IAAI,yBAC7C;IACA;GACF;GAEA,IACEA,oBAAAA,yBAAyB,IAAI,MAC5B,KAAK,SAAS,cACb,KAAK,SAAS,UACd,KAAK,SAAA,eACP;IACA,aAAa,KAAK,KAChB,4BAA4B,MAAM,UAAU,SAAS,IACnD,yBACJ;IACA;GACF;GAEA,IAAIA,oBAAAA,yBAAyB,IAAI,KAAK,KAAK,SAAS,SAAS;IAC3D,MAAM,oBAAoB,+BACxB,MACA,UACA,SACF;IACA,IAAI,qBAAqB,MAAM;KAC7B,aAAa,KAAK,KAChB,oBAAoB,yBACtB;KACA;IACF;GACF;GAEA,IAAIA,oBAAAA,yBAAyB,IAAI,KAAK,iBAAiB,KAAK,IAAI,GAAG;IACjE,MAAM,mBAAmB,8BAA8B,IAAI;IAC3D,IAAI,mBAAmB,GAAG;KACxB,aAAa,KAAK,KAChB,mBAAmB,yBACrB;KACA;IACF;GACF;GAEA,IAAI,KAAK,SAAA,eAAmC,KAAK,aAAa,MAAM;IAClE,MAAM,WAAW,KAAK,UAAU;IAChC,IAAI,OAAO,aAAa,YAAY,SAAS,SAAS,GACpD,aAAa,UAAU,QAAQ;IAEjC,MAAM,OAAO,KAAK,UAAU;IAC5B,IAAI,QAAQ,MACV,aACE,OAAO,SAAS,WACZ,UAAU,IAAI,IACd,+BAA+B,MAAM,SAAS;IAEtD,MAAM,SAAS,KAAK,UAAU;IAC9B,IAAI,UAAU,MACZ,aAAa,QAAQ,GAAG,KAAK,GAAG,MAAM,mBAAmB;IAE3D;GACF;GAEA,MAAM,cAAc,KAAK,KAAK;GAC9B,IAAI,eAAe,MAAM;IACvB,aAAa,+BAA+B,MAAM,SAAS;IAC3D;GACF;GAEA,aAAa,aAAa,GAAG,KAAK,GAAG,MAAM,IAAI,KAAK,MAAM;EAC5D;OACK,IAAI,OAAO,UAAU,UAC1B,aAAa,UAAU,KAAK;OACvB,IAAI,OAAO,UAAU,UAC1B,aAAa,UAAU,MAAM,SAAS,CAAC;OAClC,IAAI,OAAO,UAAU,WAC1B,aAAa,UAAU,MAAM,SAAS,CAAC;OAClC,IAAI,SAAS,QAAQ,OAAO,UAAU,UAC3C,aAAa,+BAA+B,OAAO,SAAS;CAEhE;CAEA,MAAM,sBAAsB,QAAQ;CACpC,MAAM,mBACJ,uBAAuB,QAAQ,OAAO,wBAAwB,WAC1D,sBACA,KAAA;CACN,IAAI,oBAAoB,SAAA,GAAA,gBAAA,QAAA,CAAgB,gBAAgB,GACtD,MAAM,IAAI,4BAA4B;EACpC,QAAQ;EACR,MAAM;CACR,CAAC;CAEH,IAAI;CACJ,IAAI;EACF,iBACE,oBAAoB,OAChB,OAAO,yBAAyB,kBAAkB,MAAM,IACxD,KAAA;CACR,QAAQ;EACN,MAAM,IAAI,4BAA4B;GACpC,QAAQ;GACR,MAAM;EACR,CAAC;CACH;CACA,IAAI,kBAAkB,QAAQ,EAAE,WAAW,iBACzC,MAAM,IAAI,4BAA4B;EACpC,QAAQ;EACR,MAAM;CACR,CAAC;CAGH,IAAI,YAAY;CAChB,MAAM,cAAc,QAAQ,QAAQ;CAKpC,IAHE,gBAAgB,UAChB,gBAAgB,UAAU,0BAC1BC,oBAAAA,4BAA4B,QAAQ,OAAO,KACjB,OAAO,QAAQ,YAAY,UACrD,aAAa,KAAK,KAChB,yBACE;EACE,MAAM;EACN,WAAW,QAAQ;CACrB,GACA,QACF,IAAI,yBACN;MAEA,aAAa,QAAQ,SAAS,SAAS;CAEzC,2BAA2B,WAAW,SAAS;CAC/C,MAAM,cAAe,QAA6C;CAClE,IAAI,gBAAgB,QAAQ,gBAAgB,aAAa;EACvD,MAAM,YAAa,QAAsB,cAAc,CAAC;EACxD,KAAA,GAAA,gBAAA,QAAA,CAAY,SAAS,GACnB,MAAM,IAAI,4BAA4B;GACpC,QAAQ;GACR,MAAM;EACR,CAAC;EAEH,KAAK,MAAM,YAAY,WAAW;GAChC,KAAA,GAAA,gBAAA,QAAA,CAAY,QAAQ,GAClB,MAAM,IAAI,4BAA4B;IACpC,QAAQ;IACR,MAAM;GACR,CAAC;GAEH,IACE,OAAO,SAAS,OAAO,YACvB,uBAAuB,IAAI,SAAS,EAAE,GAEtC;GAEF,IAAI,OAAO,SAAS,SAAS,YAAY,SAAS,KAAK,SAAS,GAC9D,aAAa,UAAU,SAAS,IAAI;GAEtC,MAAM,OAAgB,SAAS;GAC/B,IAAI,QAAQ,MACV,aACE,OAAO,SAAS,WACZ,UAAU,IAAI,IACd,+BAA+B,MAAM,SAAS;EAExD;EACA,IAAI;EACJ,IAAI;GACF,qBACE,oBAAoB,OAChB,OAAO,yBAAyB,kBAAkB,eAAe,IACjE,KAAA;EACR,QAAQ;GACN,MAAM,IAAI,4BAA4B;IACpC,QAAQ;IACR,MAAM;GACR,CAAC;EACH;EACA,IAAI,sBAAsB,MAAM;GAC9B,IAAI,EAAE,WAAW,qBACf,MAAM,IAAI,4BAA4B;IACpC,QAAQ;IACR,MAAM;GACR,CAAC;GAEH,IAAI,mBAAmB,SAAS,MAC9B,aAAa,+BACX,mBAAmB,OACnB,SACF;EAEJ;CACF;CACA,OAAO,2BAA2B,WAAW,SAAS;AACxD;AAQA,SAAgB,qBACd,WACA,YACA,aACwB;CACxB,MAAM,SAAiC,OAAO,OAAO,IAAI;CACzD,MAAM,aAAyD,CAAC;CAChE,IAAI,WAAW;CACf,KAAK,MAAM,CAAC,MAAM,aAAa,OAAO,QAAQ,SAAS,GAAG;EACxD,MAAM,SAAS,WAAW;EAC1B,MAAM,UAAU,KAAK,MAAM,MAAM;EACjC,OAAO,QAAQ;EACf,YAAY;EACZ,WAAW,KAAK;GAAE;GAAM,WAAW,SAAS;EAAQ,CAAC;CACvD;CACA,IAAI,WAAW,cAAc;CAC7B,IAAI,YAAY,KAAK,WAAW,WAAW,GACzC,OAAO;CAET,WAAW,MAAM,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;CACnD,KAAK,IAAI,IAAI,GAAG,WAAW,GAAG,KAAK,IAAI,KAAK,WAAW,QAAQ;EAC7D,OAAO,WAAW,EAAE,CAAC,SAAS;EAC9B;CACF;CACA,OAAO;AACT;AAOA,MAAM,0BAA0B;AAMhC,MAAa,qBAAqB,OAChC,WAAyB,iBACqB;CAC9C,MAAM,MAAM,MAAM,aAAa,QAAQ;CACvC,MAAM,eAAe,SAAyB,IAAI,MAAM,IAAI;CAC5D,MAAM,WAAW,aAAa;CAC9B,QAAQ,YAAiC;EACvC,MAAM,QAAQ,wBAAwB,SAAS,aAAa,QAAQ;EAIpE,OAAO,2BAHgB,WACnB,KAAK,KAAK,QAAQ,uBAAuB,IACzC,OAC8C,SAAS;CAC7D;AACF;AAGA,MAAa,sBAAsB;CACjC,MAAM,aAA4B,CAElC;CAEA,QAAc;EACZ,KAAK,MAAM,OAAO,OAAO,KAAK,UAAU,GACtC,OAAO,WAAW;CAEtB;CAEA,gBAAyB;EACvB,OAAO,OAAO,KAAK,UAAU,CAAC,CAAC,SAAS;CAC1C;AACF"}
1	+ {"version":3,"file":"tokens.cjs","names":["Tokenizer","serializeStructuredValueBounded","isAtomicToolContentBlock","isComputerCallOutputContent"],"sources":["../../../src/utils/tokens.ts"],"mappings":";;;;;AAmBA,IAAa,8BAAb,cAAiD,MAAM;CACrD,OAAgB;CAChB;CACA;CAEA,YAAY,EACV,QACA,QAIC;EACD,MACE,KAAK,UAAU;GACb,MAAM;GACN;GACA;EACF,CAAC,CACH;EACA,KAAK,OAAO;EACZ,KAAK,SAAS;EACd,KAAK,OAAO;CACd;AACF;AAEA,SAAS,2BAA2B,OAAe,MAAsB;CACvE,IACE,CAAC,OAAO,cAAc,KAAK,KAC3B,QAAQ,KACR,SAAS,OAAO,kBAEhB,MAAM,IAAI,4BAA4B;EACpC,QAAQ;EACR;CACF,CAAC;CAEH,OAAO;AACT;AAGA,MAAM,6BAA6B;AAEnC,MAAM,0BAA0B;AAEhC,MAAM,0BAA0B;AAEhC,MAAM,yBAAyB;AAE/B,MAAM,+BAA+B;AAIrC,MAAa,4BAA4B;AAMzC,MAAM,gCAAgC;AAEtC,MAAM,6BAA6B;AAInC,MAAM,4BAA4B;AAElC,MAAM,+BAA+B;AAUrC,MAAM,0BAA0B;AAEhC,MAAM,0BAA0B;AAEhC,MAAM,yBAAyB;AAC/B,MAAM,yBAAyB;AAE/B,MAAM,4BAA4B;AAClC,MAAM,4BAA4B;AAGlC,MAAM,oBAAoB,IAAI,IAAI;CAChC;CACA;CACA;CACA;CACA;AACF,CAAC;AAMD,MAAM,oCAAoC;AAE1C,MAAM,yCAAyC;AAC/C,MAAM,wCAAwC;AAE9C,MAAM,8BAA8B;AAEpC,MAAM,0CAA0C;AAOhD,SAAgB,uBACd,YAC0C;CAC1C,MAAM,MAAM,WAAW,WAAW,OAAO,IACrC,WAAW,MAAM,WAAW,QAAQ,GAAG,IAAI,CAAC,IAC5C;CAEJ,IAAI,IAAI,SAAS,IACf,OAAO;CAGT,MAAM,QAAQ,IAAI,WAAW,OAAO,KAAK,IAAI,MAAM,GAAG,EAAE,GAAG,QAAQ,CAAC;CAEpE,IAAI,MAAM,OAAO,OAAQ,MAAM,OAAO,IAMpC,OAAO;EAAE,OAHN,MAAM,OAAO,KAAO,MAAM,OAAO,KAAO,MAAM,OAAO,IAAK,MAAM;EAGnD,QADb,MAAM,OAAO,KAAO,MAAM,OAAO,KAAO,MAAM,OAAO,IAAK,MAAM;CAC5C;CAGzB,IAAI,MAAM,OAAO,OAAQ,MAAM,OAAO,KAAM;EAE1C,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,SAAS,GAAG,KACpC,IACE,MAAM,OAAO,QACZ,MAAM,IAAI,OAAO,OAAQ,MAAM,IAAI,OAAO,MAC3C;GACA,MAAM,SAAU,MAAM,IAAI,MAAM,IAAK,MAAM,IAAI;GAE/C,OAAO;IAAE,OADM,MAAM,IAAI,MAAM,IAAK,MAAM,IAAI;IAC9B;GAAO;EACzB;EAEF,OAAO;CACT;CAEA,IAAI,MAAM,OAAO,MAAQ,MAAM,OAAO,MAAQ,MAAM,OAAO,IAIzD,OAAO;EAAE,OAFK,MAAM,KAAM,MAAM,MAAM;EAEtB,QADD,MAAM,KAAM,MAAM,MAAM;CAChB;CAGzB,IACE,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,QAAQ,MACd,MAAM,QAAQ,IACd;EAEA,IAAI,MAAM,SAAS,IAGjB,OAAO;GAAE,QAFM,MAAM,MAAO,MAAM,OAAO,KAAM;GAE/B,SADA,MAAM,MAAO,MAAM,OAAO,KAAM;EACzB;EAEzB,OAAO;CACT;CAEA,OAAO;AACT;AAGA,SAAgB,6BACd,OACA,QACQ;CACR,OAAO,KAAK,IACV,4BACA,KAAK,KAAM,QAAQ,SAAU,uBAAuB,CACtD;AACF;AAGA,SAAgB,0BACd,OACA,QACA,SAAiB,QACT;CACR,IAAI,WAAW,OACb,OAAO;CAKT,OAAO,0BAFL,KAAK,KAAK,QAAQ,sBAAsB,IACxC,KAAK,KAAK,SAAS,sBAAsB,IACF;AAC3C;AAOA,SAAgB,yBACd,OACA,UACQ;CACR,IAAI;CAEJ,IACE,MAAM,SAAA,eACN,MAAM,SAAS,eACf,MAAM,SAAS,yBACf,MAAM,SAAS,eACf;EACA,MAAM,WAAW,MAAM;EACvB,MAAM,MAAM,OAAO,aAAa,WAAW,WAAW,UAAU;EAChE,IAAI,OAAO,QAAQ,YAAY,IAAI,WAAW,OAAO,GACnD,aAAa;OACR,IAAI,OAAO,MAAM,YAAY,YAAY,MAAM,YAAY,IAChE,OAAO,MAAM,WAAW,QACpB,0BACA;OAEJ,OAAO;CAEX,OAAO,IAAI,MAAM,SAAS,SAAS;EACjC,MAAM,SAAS,MAAM;EACrB,IAAI,QAAQ,SAAS,YAAY,OAAO,OAAO,SAAS,UACtD,aAAa,OAAO;OAEpB,OAAO;CAEX,OACE,OAAO;CAGT,MAAM,OAAO,uBAAuB,UAAU;CAC9C,IAAI,QAAQ,MACV,OAAO;CAGT,IAAI,aAAa,UACf,OAAO,6BAA6B,KAAK,OAAO,KAAK,MAAM;CAE7D,OAAO,0BAA0B,KAAK,OAAO,KAAK,MAAM;AAC1D;AAWA,SAAgB,4BACd,OACA,UACA,eACQ;CACR,MAAM,mBACJ,aAAa,WACT,gCACA;CAGN,MAAM,aAAa,MAAM;CACzB,IAAI,OAAO,eAAe,UAAU;EAClC,MAAM,YAAa,MAAM,aAAoC,GAAA,CAAI,MAC/D,GACF,CAAC,CAAC;EAEF,IAAI,eAAe,UAAU,OAAO,MAAM,SAAS,UACjD,OAAO,cAAc,MAAM,IAAc;EAG3C,IAAI,eAAe,YAAY,OAAO,MAAM,SAAS,UAAU;GAC7D,IAAI,aAAa,qBAAqB,aAAa,IAKjD,OAJqB,KAAK,IACxB,GACA,KAAK,KAAM,MAAM,KAAgB,SAAS,yBAAyB,CAEnD,IAAI;GAGxB,IAAI,SAAS,WAAW,QAAQ,GAC9B,OAAO,yBACL;IACE,GAAG;IACH,MAAM;IACN,QAAQ;KAAE,MAAM;KAAU,MAAM,MAAM;IAAK;GAC7C,GACA,QACF;GAEF,OAAO,cAAc,MAAM,IAAc;EAC3C;EAEA,IAAI,eAAe,OACjB,OAAO;EAGT,OAAO;CACT;CAGA,MAAM,SAAS,MAAM;CASrB,IAAI,UAAU,MACZ,OAAO;CAGT,IAAI,OAAO,SAAS,UAAU,OAAO,OAAO,SAAS,UACnD,OAAO,cAAc,OAAO,IAAI;CAGlC,IAAI,OAAO,SAAS,YAAY,OAAO,OAAO,SAAS,UAAU;EAC/D,MAAM,aAAa,OAAO,cAAc,GAAA,CAAI,MAAM,GAAG,CAAC,CAAC;EACvD,IAAI,cAAc,qBAAqB,cAAc,IAKnD,OAJqB,KAAK,IACxB,GACA,KAAK,KAAK,OAAO,KAAK,SAAS,yBAAyB,CAExC,IAAI;EAExB,IAAI,UAAU,WAAW,QAAQ,GAC/B,OAAO,yBACL;GAAE,MAAM;GAAS,QAAQ;IAAE,MAAM;IAAU,MAAM,OAAO;GAAK;EAAE,GAC/D,QACF;EAEF,OAAO,cAAc,OAAO,IAAI;CAClC;CAEA,IAAI,OAAO,SAAS,OAClB,OAAO;CAIT,IAAI,OAAO,SAAS,aAAa,MAAM,QAAQ,OAAO,OAAO,GAAG;EAC9D,IAAI,QAAQ;EACZ,KAAK,MAAM,SAAS,OAAO,SACzB,IAAI,SAAS,QAAQ,OAAO,UAAU,YAAY,UAAU,OAAO;GACjE,MAAM,aAAa;GACnB,IAAI,WAAW,SAAS,SACtB,SAAS,yBAAyB,YAAY,QAAQ;EAE1D;EAEF,OAAO,QAAQ,IAAI,QAAQ;CAC7B;CAEA,OAAO;AACT;AAKA,SAAS,iBAAiB,OAAmC;CAC3D,IAAI,OAAO,UAAU,YAAY,MAAM,WAAW,GAChD,OAAO;CAET,IAAI,MAAM,WAAW,OAAO,GAAG;EAC7B,MAAM,QAAQ,MAAM,QAAQ,GAAG;EAC/B,OAAO,QAAQ,IAAI,IAAI,KAAK,OAAQ,MAAM,SAAS,QAAQ,KAAK,IAAK,CAAC;CACxE;CACA,IAAI,uBAAuB,KAAK,KAAK,GACnC,OAAO;CAET,OAAO,KAAK,MAAO,MAAM,SAAS,IAAK,CAAC;AAC1C;AAEA,SAAS,iBACP,OACA,gBACA,iBACA,aACQ;CACR,IAAI,SAAS,GACX,OAAO;CAET,OAAO,KAAK,IACV,iBACA,KAAK,KAAM,QAAQ,iBAAkB,eAAe,CACtD;AACF;AAMA,SAAS,qBAAqB,OAAwC;CACpE,MAAM,OAAO,MAAM;CACnB,IAAI,OAAO,SAAS,UAClB,OAAO,iBAAiB,IAAI;CAE9B,IAAI,gBAAgB,YAClB,OAAO,KAAK;CAEd,IAAI,OAAO,MAAM,QAAQ,UACvB,OAAO,iBAAiB,MAAM,GAAG;CAEnC,MAAM,SAAU,MAAM,SAAS,MAAM;CAGrC,IAAI,UAAU,MAAM;EAClB,IAAI,OAAO,OAAO,SAAS,UACzB,OAAO,iBAAiB,OAAO,IAAI;EAErC,IAAI,OAAO,OAAO,QAAQ,UACxB,OAAO,iBAAiB,OAAO,GAAG;EAEpC,IAAI,OAAO,QAAQ,iBAAiB,YAClC,OAAO,OAAO,OAAO,MAAM;CAE/B;CACA,OAAO;AACT;AAMA,SAAS,eACP,OAC0B;CAC1B,MAAM,OACJ,OAAO,MAAM,SAAS,WAAW,uBAAuB,MAAM,IAAI,IAAI;CACxE,IAAI,SAAS,iBAAiB,SAAS,SACrC,OAAO;CAET,IAAI,SAAS,eAAe,SAAS,SACnC,OAAO;CAET,MAAM,YAAY,iBAAiB,KAAK;CACxC,MAAM,OAAO,SAAS,WAAW,aAAa,OAAO,YAAY;CACjE,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,OAAO;AACT;AAEA,SAAS,iBAAiB,OAAoD;CAC5E,IAAI,OAAO,MAAM,aAAa,UAC5B,OAAO,uBAAuB,MAAM,QAAQ;CAE9C,IAAI,OAAO,MAAM,cAAc,UAC7B,OAAO,uBAAuB,MAAM,SAAS;AAGjD;AAEA,SAAS,uBAAuB,UAA0B;CACxD,OAAO,SAAS,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,YAAY;AACnD;AAKA,SAAS,iBAAiB,MAAuB;CAC/C,MAAM,iBAAiB,uBAAuB,IAAI;CAClD,OACE,kBAAkB,IAAI,cAAc,KACpC,eAAe,WAAW,QAAQ,KAClC,eAAe,WAAW,QAAQ;AAEtC;AAGA,SAAS,gCAAgC,MAAuB;CAC9D,IAAI,OAAO,SAAS,UAAU;EAC5B,IAAI,iBAAiB,IAAI,KAAK,GAC5B,OAAO;EAET,OAAO,KAAK,IAAI,8BAA8B,KAAK,MAAM;CAC3D;CACA,IAAI,gBAAgB,eAAe,YAAY,OAAO,IAAI,GACxD,OAAO,KAAK,IACV,8BACA,KAAK,KAAM,KAAK,aAAa,IAAK,CAAC,CACrC;CAEF,OAAO;AACT;AAMA,SAAS,+BACP,OACA,UACA,eACoB;CACpB,MAAM,YAAY,iBAAiB,KAAK;CACxC,IAAI,MAAM,SAAS,WAAW,aAAa,MACzC;CAGF,MAAM,WAAW;CACjB,MAAM,cACJ,MAAM,SAAS,QAAQ,OAAO,MAAM,UAAU,WACzC,MAAM,QACP,KAAA;CACN,MAAM,OACJ,MAAM,QAAQ,MAAM,SAAS,aAAa,QAAQ,aAAa;CACjE,MAAM,YACJ,MAAM,OACN,MAAM,WACN,MAAM,UACN,aAAa,OACb,aAAa,WACb,aAAa;CAEf,IAAI,SAAS,WAAW,QAAQ,GAAG;EACjC,IAAI;EACJ,IAAI,OAAO,SAAS,UAClB,eAAe;OACV,IAAI,gBAAgB,eAAe,YAAY,OAAO,IAAI,GAAG;GAClE,MAAM,OACJ,gBAAgB,cACZ,IAAI,WAAW,IAAI,IACnB,IAAI,WAAW,KAAK,QAAQ,KAAK,YAAY,KAAK,UAAU;GAClE,eAAe,OAAO,KAAK,KAAK,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS,QAAQ;EACpE;EAEA,IAAI,gBAAgB,QAAQ,aAAa,SAAS,GAChD,OAAO,yBACL;GACE,MAAM;GACN,QAAQ;IAAE,MAAM;IAAU,MAAM;GAAa;EAC/C,GACA,QACF;EAEF,OAAO,yBACL;GACE,MAAM;GACN,WAAW,EAAE,KAAK,OAAO,cAAc,WAAW,YAAY,GAAG;EACnE,GACA,QACF;CACF;CAEA,IAAI,aAAa,mBAAmB;EAClC,IAAI,OAAO,SAAS,YAAY,KAAK,SAAS,GAAG;GAC/C,MAAM,QAAQ,KAAK,WAAW,OAAO,IAAI,KAAK,QAAQ,GAAG,IAAI;GAE7D,OAAO,4BACL;IACE,MAAM;IACN,aAAa;IACb,WAAW;IACX,MANe,SAAS,IAAI,KAAK,MAAM,QAAQ,CAAC,IAAI;GAOtD,GACA,UACA,aACF;EACF;EACA,IAAI,gBAAgB,eAAe,YAAY,OAAO,IAAI,GAAG;GAC3D,MAAM,uBAAuB,KAAK,KAAM,KAAK,aAAa,IAAK,CAAC;GAChE,MAAM,mBACJ,aAAa,WACT,gCACA;GACN,OACE,KAAK,IACH,GACA,KAAK,KAAK,uBAAuB,yBAAyB,CAC5D,IAAI;EAER;EACA,OAAO;CACT;CAEA,IAAI,SAAS,WAAW,QAAQ,KAAK,SAAS,WAAW,QAAQ,GAC/D;CAEF,OAAO,gCAAgC,IAAI;AAC7C;AAaA,SAAgB,8BACd,OACQ;CACR,MAAM,OAAO,eAAe,KAAK;CACjC,IAAI,QAAQ,MACV,OAAO;CAET,IAAI;CACJ,IAAI,MAAM,SAAS,eAAe;EAChC,MAAM,QAAQ,MAAM;EACpB,QAAQ,iBAAiB,OAAO,IAAI;CACtC,OAAO,IAAI,MAAM,SAAS,aAAa;EACrC,MAAM,QAAQ,MAAM;EACpB,QAAQ,iBAAiB,OAAO,UAAU,WAAW,QAAQ,OAAO,GAAG;CACzE,OACE,QAAQ,qBAAqB,KAAK;CAEpC,IAAI,SAAS,SACX,OAAO,iBACL,OACA,wBACA,yBACA,yBACF;CAEF,OAAO,iBACL,OACA,wBACA,yBACA,yBACF;AACF;AAEA,MAAM,aAAuD,CAAC;AAE9D,eAAe,aACb,WAAyB,cACL;CACpB,MAAM,SAAS,WAAW;CAC1B,IAAI,QACF,OAAO;CAMT,MAAM,WAAW,IAAIA,aAAAA,UAHnB,aAAa,WACT,MAAM,OAAO,kCACb,MAAM,OAAO,mCACgB;CACnC,WAAW,YAAY;CACvB,OAAO;AACT;AAEA,SAAgB,iBAAiB,OAA6B;CAC5D,IAAI,MAAM,YAAY,CAAC,CAAC,SAAS,QAAQ,GACvC,OAAO;CAET,OAAO;AACT;AAMA,SAAS,4BACP,YACS;CACT,IAAI,WAAW,aAAa,GAC1B,OAAO;CAET,WAAW;CACX,OAAO;AACT;AAEA,SAAS,gBACP,OACA,YACS;CACT,IAAI,UAAyB;CAC7B,MAAM,uBAAO,IAAI,IAAY;CAC7B,IAAI;EACF,OAAO,WAAW,MAAM;GACtB,KAAA,GAAA,gBAAA,QAAA,CAAY,OAAO,GACjB,OAAO;GAET,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,4BAA4B,UAAU,GAC9D,OAAO;GAET,KAAK,IAAI,OAAO;GAChB,MAAM,aAAa,OAAO,yBAAyB,SAAS,QAAQ;GACpE,IAAI,cAAc,MAChB,OACG,WAAW,cAAc,OAAO,WAAW,UAAU,cACtD,EAAE,WAAW;GAGjB,UAAU,OAAO,eAAe,OAAO;EACzC;CACF,QAAQ;EACN,OAAO;CACT;CACA,OAAO;AACT;AAEA,MAAM,8BAA8B,OAAO,yBACzC,YAAY,WACZ,YACF,CAAC,EAAE;AAMH,SAAS,oBAAoB,OAAwB;CACnD,IAAI;EACF,IAAI,YAAY,OAAO,KAAK,GAC1B,OAAO;CAEX,QAAQ;EACN,OAAO;CACT;CACA,IAAI,+BAA+B,MACjC,OAAO;CAET,IAAI;EACF,4BAA4B,KAAK,KAAK;EACtC,OAAO;CACT,QAAQ;EACN,OAAO;CACT;AACF;AAMA,SAAS,2BACP,OACA,KACA,YACS;CACT,MAAM,OAAO,IAAI,IAAY,CAAC,KAAK,CAAC;CACpC,IAAI;CACJ,IAAI;EACF,UAAU,OAAO,eAAe,KAAK;CACvC,QAAQ;EACN,OAAO;CACT;CAEA,OAAO,WAAW,MAAM;EACtB,KAAA,GAAA,gBAAA,QAAA,CAAY,OAAO,GACjB,OAAO;EAET,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,4BAA4B,UAAU,GAC9D,OAAO;EAET,KAAK,IAAI,OAAO;EAChB,IAAI;EACJ,IAAI;GACF,aAAa,OAAO,yBAAyB,SAAS,GAAG;EAC3D,QAAQ;GACN,OAAO;EACT;EACA,IAAI,cAAc,MAChB,OAAO,EAAE,WAAW;EAEtB,IAAI;GACF,UAAU,OAAO,eAAe,OAAO;EACzC,QAAQ;GACN,OAAO;EACT;CACF;CACA,OAAO;AACT;AAQA,SAAgB,iCAAiC,OAAyB;CACxE,IAAI,OAAO,UAAU,UACnB,OAAO;CAET,IAAI,SAAS,QAAQ,OAAO,UAAU,UACpC,OAAO;CAET,KAAA,GAAA,gBAAA,QAAA,CAAY,KAAK,GACf,OAAO;CAET,IAAI,oBAAoB,KAAK,GAC3B,OAAO;CAGT,MAAM,QAAkB,CAAC,KAAK;CAC9B,MAAM,uBAAO,IAAI,IAAY;CAC7B,MAAM,aAAyC,EAC7C,WAAW,sCACb;CACA,OAAO,MAAM,SAAS,GAAG;EACvB,MAAM,UAAU,MAAM,IAAI;EAC1B,KAAA,GAAA,gBAAA,QAAA,CAAY,OAAO,GACjB,OAAO;EAET,IAAI,KAAK,IAAI,OAAO,GAClB,OAAO;EAET,KAAK,IAAI,OAAO;EAChB,IAAI,CAAC,4BAA4B,UAAU,GACzC,OAAO;EAGT,IAAI;GACF,IAAI,gBAAgB,SAAS,UAAU,GACrC,OAAO;GAET,KAAK,MAAM,OAAO,SAAS;IACzB,IAAI,CAAC,4BAA4B,UAAU,GACzC,OAAO;IAET,MAAM,aAAa,OAAO,yBAAyB,SAAS,GAAG;IAC/D,IAAI,cAAc,MAAM;KACtB,IAAI,2BAA2B,SAAS,KAAK,UAAU,GACrD,OAAO;KAET;IACF;IACA,IAAI,WAAW,eAAe,MAC5B;IAEF,IAAI,EAAE,WAAW,aACf,OAAO;IAET,IAAI,WAAW,YAAY;KACzB,IAAI,OAAO,WAAW,UAAU,UAC9B,OAAO;KAET,IACE,WAAW,SAAS,QACpB,OAAO,WAAW,UAAU,YAC5B,oBAAoB,WAAW,KAAK,GAEpC,OAAO;KAET,IACE,WAAW,SAAS,QACpB,OAAO,WAAW,UAAU,UAE5B,MAAM,KAAK,WAAW,KAAK;IAE/B;GACF;EACF,QAAQ;GACN,OAAO;EACT;CACF;CACA,OAAO;AACT;AAEA,SAAS,yBACP,OACA,eACQ;CACR,IAAI,YAAY;CAChB,KACE,IAAI,SAAS,GACb,SAAS,MAAM,QACf,UAAU,6BACV;EACA,MAAM,cAAc,2BAClB,cAAc,MAAM,MAAM,QAAQ,SAAS,2BAA2B,CAAC,GACvE,WACF;EACA,MAAM,iBACJ,SAAS,IAAI,0CAA0C;EACzD,YAAY,KAAK,IACf,OAAO,kBACP,YAAY,cAAc,cAC5B;CACF;CACA,OAAO;AACT;AAEA,SAAS,+BACP,OACA,eACQ;CACR,MAAM,aAAaC,oBAAAA,gCACjB,OACA,iCACF;CACA,MAAM,gBAAgB,2BACpB,cAAc,WAAW,OAAO,GAChC,WACF;CACA,IAAI,CAAC,WAAW,WACd,OAAO;CAKT,IACE,CAAC,OAAO,cAAc,WAAW,aAAa,KAC9C,WAAW,iBAAiB,OAAO,kBAEnC,OAAO;CAGT,MAAM,eAAe,KAAK,IACxB,GACA,WAAW,gBAAgB,WAAW,QAAQ,MAChD;CACA,OAAO,KAAK,IACV,OAAO,kBACP,gBAAgB,eAAe,sCACjC;AACF;AAEA,SAAgB,wBACd,SACA,eACA,WAAyB,cACjB;CACR,MAAM,mBAAmB;CACzB,MAAM,aAAa,SACjB,yBAAyB,MAAM,aAAa;CAC9C,KAAA,GAAA,gBAAA,QAAA,CAAY,OAAO,GACjB,MAAM,IAAI,4BAA4B;EACpC,QAAQ;EACR,MAAM;CACR,CAAC;CAYH,MAAM,yCAAyB,IAAI,IAAY;CAE/C,MAAM,gBAAgB,OAAgB,SAAuB;EAC3D,IAAI,SAAS,QAAQ,OAAO,UAAU,aAAA,GAAA,gBAAA,QAAA,CAAoB,KAAK,GAC7D,MAAM,IAAI,4BAA4B;GACpC,QAAQ;GACR;EACF,CAAC;EAEH,IAAI,MAAM,QAAQ,KAAK,GACrB,KAAK,IAAI,QAAQ,GAAG,QAAQ,MAAM,QAAQ,SAAS;GACjD,MAAM,MAAM,MAAM;GAClB,IACE,OAAO,QAAQ,YACf,OAAO,QAAQ,YACf,OAAO,QAAQ,WACf;IACA,aAAa,KAAK,GAAG,KAAK,GAAG,MAAM,EAAE;IACrC;GACF;GACA,MAAM,OAAO;GACb,IAAI,QAAQ,QAAQ,OAAO,SAAS,UAClC;GAEF,KAAA,GAAA,gBAAA,QAAA,CAAY,IAAI,GACd,MAAM,IAAI,4BAA4B;IACpC,QAAQ;IACR,MAAM,GAAG,KAAK,GAAG,MAAM;GACzB,CAAC;GAEH,IAAI,OAAO,KAAK,SAAS,UAAU;IACjC,aAAa,+BAA+B,MAAM,SAAS;IAC3D;GACF;GACA,IAAI,KAAK,SAAA,eAAmC,KAAK,SAAS,YAAY;IACpE,MAAM,WACJ,OAAO,KAAK,OAAO,WAAW,KAAK,KAAK,KAAK,WAAW;IAC1D,IAAI,OAAO,aAAa,YAAY,SAAS,SAAS,GACpD,uBAAuB,IAAI,QAAQ;GAEvC;GACA,IAAI,KAAK,SAAA,SACP;GAGF,IACEC,oBAAAA,yBAAyB,IAAI,MAC5B,KAAK,SAAA,eACJ,KAAK,SAAS,eACd,KAAK,SAAS,WACd,KAAK,SAAS,yBACd,KAAK,SAAS,gBAChB;IACA,aAAa,KAAK,KAChB,yBAAyB,MAAM,QAAQ,IAAI,yBAC7C;IACA;GACF;GAEA,IACEA,oBAAAA,yBAAyB,IAAI,MAC5B,KAAK,SAAS,cACb,KAAK,SAAS,UACd,KAAK,SAAA,eACP;IACA,aAAa,KAAK,KAChB,4BAA4B,MAAM,UAAU,SAAS,IACnD,yBACJ;IACA;GACF;GAEA,IAAIA,oBAAAA,yBAAyB,IAAI,KAAK,KAAK,SAAS,SAAS;IAC3D,MAAM,oBAAoB,+BACxB,MACA,UACA,SACF;IACA,IAAI,qBAAqB,MAAM;KAC7B,aAAa,KAAK,KAChB,oBAAoB,yBACtB;KACA;IACF;GACF;GAEA,IAAIA,oBAAAA,yBAAyB,IAAI,KAAK,iBAAiB,KAAK,IAAI,GAAG;IACjE,MAAM,mBAAmB,8BAA8B,IAAI;IAC3D,IAAI,mBAAmB,GAAG;KACxB,aAAa,KAAK,KAChB,mBAAmB,yBACrB;KACA;IACF;GACF;GAEA,IAAI,KAAK,SAAA,eAAmC,KAAK,aAAa,MAAM;IAClE,MAAM,WAAW,KAAK,UAAU;IAChC,IAAI,OAAO,aAAa,YAAY,SAAS,SAAS,GACpD,aAAa,UAAU,QAAQ;IAEjC,MAAM,OAAO,KAAK,UAAU;IAC5B,IAAI,QAAQ,MACV,aACE,OAAO,SAAS,WACZ,UAAU,IAAI,IACd,+BAA+B,MAAM,SAAS;IAEtD,MAAM,SAAS,KAAK,UAAU;IAC9B,IAAI,UAAU,MACZ,aAAa,QAAQ,GAAG,KAAK,GAAG,MAAM,mBAAmB;IAE3D;GACF;GAEA,MAAM,cAAc,KAAK,KAAK;GAC9B,IAAI,eAAe,MAAM;IACvB,aAAa,+BAA+B,MAAM,SAAS;IAC3D;GACF;GAEA,aAAa,aAAa,GAAG,KAAK,GAAG,MAAM,IAAI,KAAK,MAAM;EAC5D;OACK,IAAI,OAAO,UAAU,UAC1B,aAAa,UAAU,KAAK;OACvB,IAAI,OAAO,UAAU,UAC1B,aAAa,UAAU,MAAM,SAAS,CAAC;OAClC,IAAI,OAAO,UAAU,WAC1B,aAAa,UAAU,MAAM,SAAS,CAAC;OAClC,IAAI,SAAS,QAAQ,OAAO,UAAU,UAC3C,aAAa,+BAA+B,OAAO,SAAS;CAEhE;CAEA,MAAM,sBAAsB,QAAQ;CACpC,MAAM,mBACJ,uBAAuB,QAAQ,OAAO,wBAAwB,WAC1D,sBACA,KAAA;CACN,IAAI,oBAAoB,SAAA,GAAA,gBAAA,QAAA,CAAgB,gBAAgB,GACtD,MAAM,IAAI,4BAA4B;EACpC,QAAQ;EACR,MAAM;CACR,CAAC;CAEH,IAAI;CACJ,IAAI;EACF,iBACE,oBAAoB,OAChB,OAAO,yBAAyB,kBAAkB,MAAM,IACxD,KAAA;CACR,QAAQ;EACN,MAAM,IAAI,4BAA4B;GACpC,QAAQ;GACR,MAAM;EACR,CAAC;CACH;CACA,IAAI,kBAAkB,QAAQ,EAAE,WAAW,iBACzC,MAAM,IAAI,4BAA4B;EACpC,QAAQ;EACR,MAAM;CACR,CAAC;CAGH,IAAI,YAAY;CAChB,MAAM,cAAc,QAAQ,QAAQ;CAKpC,IAHE,gBAAgB,UAChB,gBAAgB,UAAU,0BAC1BC,oBAAAA,4BAA4B,QAAQ,OAAO,KACjB,OAAO,QAAQ,YAAY,UACrD,aAAa,KAAK,KAChB,yBACE;EACE,MAAM;EACN,WAAW,QAAQ;CACrB,GACA,QACF,IAAI,yBACN;MAEA,aAAa,QAAQ,SAAS,SAAS;CAEzC,2BAA2B,WAAW,SAAS;CAC/C,MAAM,cAAe,QAA6C;CAClE,IAAI,gBAAgB,QAAQ,gBAAgB,aAAa;EACvD,MAAM,YAAa,QAAsB,cAAc,CAAC;EACxD,KAAA,GAAA,gBAAA,QAAA,CAAY,SAAS,GACnB,MAAM,IAAI,4BAA4B;GACpC,QAAQ;GACR,MAAM;EACR,CAAC;EAEH,KAAK,MAAM,YAAY,WAAW;GAChC,KAAA,GAAA,gBAAA,QAAA,CAAY,QAAQ,GAClB,MAAM,IAAI,4BAA4B;IACpC,QAAQ;IACR,MAAM;GACR,CAAC;GAEH,IACE,OAAO,SAAS,OAAO,YACvB,uBAAuB,IAAI,SAAS,EAAE,GAEtC;GAEF,IAAI,OAAO,SAAS,SAAS,YAAY,SAAS,KAAK,SAAS,GAC9D,aAAa,UAAU,SAAS,IAAI;GAEtC,MAAM,OAAgB,SAAS;GAC/B,IAAI,QAAQ,MACV,aACE,OAAO,SAAS,WACZ,UAAU,IAAI,IACd,+BAA+B,MAAM,SAAS;EAExD;EACA,IAAI;EACJ,IAAI;GACF,qBACE,oBAAoB,OAChB,OAAO,yBAAyB,kBAAkB,eAAe,IACjE,KAAA;EACR,QAAQ;GACN,MAAM,IAAI,4BAA4B;IACpC,QAAQ;IACR,MAAM;GACR,CAAC;EACH;EACA,IAAI,sBAAsB,MAAM;GAC9B,IAAI,EAAE,WAAW,qBACf,MAAM,IAAI,4BAA4B;IACpC,QAAQ;IACR,MAAM;GACR,CAAC;GAEH,IAAI,mBAAmB,SAAS,MAC9B,aAAa,+BACX,mBAAmB,OACnB,SACF;EAEJ;CACF;CACA,OAAO,2BAA2B,WAAW,SAAS;AACxD;AAQA,SAAgB,qBACd,WACA,YACA,aACwB;CACxB,MAAM,SAAiC,OAAO,OAAO,IAAI;CACzD,MAAM,aAAyD,CAAC;CAChE,IAAI,WAAW;CACf,KAAK,MAAM,CAAC,MAAM,aAAa,OAAO,QAAQ,SAAS,GAAG;EACxD,MAAM,SAAS,WAAW;EAC1B,MAAM,UAAU,KAAK,MAAM,MAAM;EACjC,OAAO,QAAQ;EACf,YAAY;EACZ,WAAW,KAAK;GAAE;GAAM,WAAW,SAAS;EAAQ,CAAC;CACvD;CACA,IAAI,WAAW,cAAc;CAC7B,IAAI,YAAY,KAAK,WAAW,WAAW,GACzC,OAAO;CAET,WAAW,MAAM,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;CACnD,KAAK,IAAI,IAAI,GAAG,WAAW,GAAG,KAAK,IAAI,KAAK,WAAW,QAAQ;EAC7D,OAAO,WAAW,EAAE,CAAC,SAAS;EAC9B;CACF;CACA,OAAO;AACT;AAOA,MAAM,0BAA0B;AAMhC,MAAa,qBAAqB,OAChC,WAAyB,iBACqB;CAC9C,MAAM,MAAM,MAAM,aAAa,QAAQ;CACvC,MAAM,eAAe,SAAyB,IAAI,MAAM,IAAI;CAC5D,MAAM,WAAW,aAAa;CAC9B,QAAQ,YAAiC;EACvC,MAAM,QAAQ,wBAAwB,SAAS,aAAa,QAAQ;EAIpE,OAAO,2BAHgB,WACnB,KAAK,KAAK,QAAQ,uBAAuB,IACzC,OAC8C,SAAS;CAC7D;AACF;AAGA,MAAa,sBAAsB;CACjC,MAAM,aAA4B,CAElC;CAEA,QAAc;EACZ,KAAK,MAAM,OAAO,OAAO,KAAK,UAAU,GACtC,OAAO,WAAW;CAEtB;CAEA,gBAAyB;EACvB,OAAO,OAAO,KAAK,UAAU,CAAC,CAAC,SAAS;CAC1C;AACF"}

package/dist/esm/utils/tokens.mjs CHANGED Viewed

@@ -51,6 +51,8 @@ const TIMED_MEDIA_TYPES = new Set([
 const MAX_STRUCTURED_TOKENIZATION_CHARS = 2e5;
 const MAX_TOKENS_PER_OMITTED_STRUCTURED_CHAR = 4;
 const MAX_STRUCTURED_SAFETY_INSPECTION_WORK = 1e4;
+const MAX_TEXT_TOKENIZATION_CHARS = 8192;
+const TEXT_TOKEN_CHUNK_BOUNDARY_SAFETY_TOKENS = 4;
 function extractImageDimensions(base64Data) {
 	const raw = base64Data.startsWith("data:") ? base64Data.slice(base64Data.indexOf(",") + 1) : base64Data;
 	if (raw.length < 32) return null;
@@ -383,11 +385,13 @@ function hasUnsafeStructuredSerialization(value) {
 	return false;
 }
 function getBoundedTextTokenCount(value, getTokenCount) {
-	const preview = value.length > MAX_STRUCTURED_TOKENIZATION_CHARS ? value.slice(0, MAX_STRUCTURED_TOKENIZATION_CHARS) : value;
-	const previewTokens = ensureSafeTokenMeasurement(getTokenCount(preview), "tokenizer");
-	const omittedChars = value.length - preview.length;
-	if (omittedChars <= 0) return previewTokens;
-	return Math.min(Number.MAX_SAFE_INTEGER, previewTokens + omittedChars * MAX_TOKENS_PER_OMITTED_STRUCTURED_CHAR);
+	let numTokens = 0;
+	for (let offset = 0; offset < value.length; offset += MAX_TEXT_TOKENIZATION_CHARS) {
+		const chunkTokens = ensureSafeTokenMeasurement(getTokenCount(value.slice(offset, offset + MAX_TEXT_TOKENIZATION_CHARS)), "tokenizer");
+		const boundaryTokens = offset > 0 ? TEXT_TOKEN_CHUNK_BOUNDARY_SAFETY_TOKENS : 0;
+		numTokens = Math.min(Number.MAX_SAFE_INTEGER, numTokens + chunkTokens + boundaryTokens);
+	}
+	return numTokens;
 }
 function getBoundedStructuredTokenCount(value, getTokenCount) {
 	const serialized = serializeStructuredValueBounded(value, MAX_STRUCTURED_TOKENIZATION_CHARS);

package/dist/esm/utils/tokens.mjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"tokens.mjs","names":[],"sources":["../../../src/utils/tokens.ts"],"mappings":";;;;;AAmBA,IAAa,8BAAb,cAAiD,MAAM;CACrD,OAAgB;CAChB;CACA;CAEA,YAAY,EACV,QACA,QAIC;EACD,MACE,KAAK,UAAU;GACb,MAAM;GACN;GACA;EACF,CAAC,CACH;EACA,KAAK,OAAO;EACZ,KAAK,SAAS;EACd,KAAK,OAAO;CACd;AACF;AAEA,SAAS,2BAA2B,OAAe,MAAsB;CACvE,IACE,CAAC,OAAO,cAAc,KAAK,KAC3B,QAAQ,KACR,SAAS,OAAO,kBAEhB,MAAM,IAAI,4BAA4B;EACpC,QAAQ;EACR;CACF,CAAC;CAEH,OAAO;AACT;AAGA,MAAM,6BAA6B;AAEnC,MAAM,0BAA0B;AAEhC,MAAM,0BAA0B;AAEhC,MAAM,yBAAyB;AAE/B,MAAM,+BAA+B;AAIrC,MAAa,4BAA4B;AAMzC,MAAM,gCAAgC;AAEtC,MAAM,6BAA6B;AAInC,MAAM,4BAA4B;AAElC,MAAM,+BAA+B;AAUrC,MAAM,0BAA0B;AAEhC,MAAM,0BAA0B;AAEhC,MAAM,yBAAyB;AAC/B,MAAM,yBAAyB;AAE/B,MAAM,4BAA4B;AAClC,MAAM,4BAA4B;AAGlC,MAAM,oBAAoB,IAAI,IAAI;CAChC;CACA;CACA;CACA;CACA;AACF,CAAC;AAMD,MAAM,oCAAoC;AAE1C,MAAM,yCAAyC;AAC/C,MAAM,wCAAwC;AAO9C,SAAgB,uBACd,YAC0C;CAC1C,MAAM,MAAM,WAAW,WAAW,OAAO,IACrC,WAAW,MAAM,WAAW,QAAQ,GAAG,IAAI,CAAC,IAC5C;CAEJ,IAAI,IAAI,SAAS,IACf,OAAO;CAGT,MAAM,QAAQ,IAAI,WAAW,OAAO,KAAK,IAAI,MAAM,GAAG,EAAE,GAAG,QAAQ,CAAC;CAEpE,IAAI,MAAM,OAAO,OAAQ,MAAM,OAAO,IAMpC,OAAO;EAAE,OAHN,MAAM,OAAO,KAAO,MAAM,OAAO,KAAO,MAAM,OAAO,IAAK,MAAM;EAGnD,QADb,MAAM,OAAO,KAAO,MAAM,OAAO,KAAO,MAAM,OAAO,IAAK,MAAM;CAC5C;CAGzB,IAAI,MAAM,OAAO,OAAQ,MAAM,OAAO,KAAM;EAE1C,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,SAAS,GAAG,KACpC,IACE,MAAM,OAAO,QACZ,MAAM,IAAI,OAAO,OAAQ,MAAM,IAAI,OAAO,MAC3C;GACA,MAAM,SAAU,MAAM,IAAI,MAAM,IAAK,MAAM,IAAI;GAE/C,OAAO;IAAE,OADM,MAAM,IAAI,MAAM,IAAK,MAAM,IAAI;IAC9B;GAAO;EACzB;EAEF,OAAO;CACT;CAEA,IAAI,MAAM,OAAO,MAAQ,MAAM,OAAO,MAAQ,MAAM,OAAO,IAIzD,OAAO;EAAE,OAFK,MAAM,KAAM,MAAM,MAAM;EAEtB,QADD,MAAM,KAAM,MAAM,MAAM;CAChB;CAGzB,IACE,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,QAAQ,MACd,MAAM,QAAQ,IACd;EAEA,IAAI,MAAM,SAAS,IAGjB,OAAO;GAAE,QAFM,MAAM,MAAO,MAAM,OAAO,KAAM;GAE/B,SADA,MAAM,MAAO,MAAM,OAAO,KAAM;EACzB;EAEzB,OAAO;CACT;CAEA,OAAO;AACT;AAGA,SAAgB,6BACd,OACA,QACQ;CACR,OAAO,KAAK,IACV,4BACA,KAAK,KAAM,QAAQ,SAAU,uBAAuB,CACtD;AACF;AAGA,SAAgB,0BACd,OACA,QACA,SAAiB,QACT;CACR,IAAI,WAAW,OACb,OAAO;CAKT,OAAO,0BAFL,KAAK,KAAK,QAAQ,sBAAsB,IACxC,KAAK,KAAK,SAAS,sBAAsB,IACF;AAC3C;AAOA,SAAgB,yBACd,OACA,UACQ;CACR,IAAI;CAEJ,IACE,MAAM,SAAA,eACN,MAAM,SAAS,eACf,MAAM,SAAS,yBACf,MAAM,SAAS,eACf;EACA,MAAM,WAAW,MAAM;EACvB,MAAM,MAAM,OAAO,aAAa,WAAW,WAAW,UAAU;EAChE,IAAI,OAAO,QAAQ,YAAY,IAAI,WAAW,OAAO,GACnD,aAAa;OACR,IAAI,OAAO,MAAM,YAAY,YAAY,MAAM,YAAY,IAChE,OAAO,MAAM,WAAW,QACpB,0BACA;OAEJ,OAAO;CAEX,OAAO,IAAI,MAAM,SAAS,SAAS;EACjC,MAAM,SAAS,MAAM;EACrB,IAAI,QAAQ,SAAS,YAAY,OAAO,OAAO,SAAS,UACtD,aAAa,OAAO;OAEpB,OAAO;CAEX,OACE,OAAO;CAGT,MAAM,OAAO,uBAAuB,UAAU;CAC9C,IAAI,QAAQ,MACV,OAAO;CAGT,IAAI,aAAa,UACf,OAAO,6BAA6B,KAAK,OAAO,KAAK,MAAM;CAE7D,OAAO,0BAA0B,KAAK,OAAO,KAAK,MAAM;AAC1D;AAWA,SAAgB,4BACd,OACA,UACA,eACQ;CACR,MAAM,mBACJ,aAAa,WACT,gCACA;CAGN,MAAM,aAAa,MAAM;CACzB,IAAI,OAAO,eAAe,UAAU;EAClC,MAAM,YAAa,MAAM,aAAoC,GAAA,CAAI,MAC/D,GACF,CAAC,CAAC;EAEF,IAAI,eAAe,UAAU,OAAO,MAAM,SAAS,UACjD,OAAO,cAAc,MAAM,IAAc;EAG3C,IAAI,eAAe,YAAY,OAAO,MAAM,SAAS,UAAU;GAC7D,IAAI,aAAa,qBAAqB,aAAa,IAKjD,OAJqB,KAAK,IACxB,GACA,KAAK,KAAM,MAAM,KAAgB,SAAS,yBAAyB,CAEnD,IAAI;GAGxB,IAAI,SAAS,WAAW,QAAQ,GAC9B,OAAO,yBACL;IACE,GAAG;IACH,MAAM;IACN,QAAQ;KAAE,MAAM;KAAU,MAAM,MAAM;IAAK;GAC7C,GACA,QACF;GAEF,OAAO,cAAc,MAAM,IAAc;EAC3C;EAEA,IAAI,eAAe,OACjB,OAAO;EAGT,OAAO;CACT;CAGA,MAAM,SAAS,MAAM;CASrB,IAAI,UAAU,MACZ,OAAO;CAGT,IAAI,OAAO,SAAS,UAAU,OAAO,OAAO,SAAS,UACnD,OAAO,cAAc,OAAO,IAAI;CAGlC,IAAI,OAAO,SAAS,YAAY,OAAO,OAAO,SAAS,UAAU;EAC/D,MAAM,aAAa,OAAO,cAAc,GAAA,CAAI,MAAM,GAAG,CAAC,CAAC;EACvD,IAAI,cAAc,qBAAqB,cAAc,IAKnD,OAJqB,KAAK,IACxB,GACA,KAAK,KAAK,OAAO,KAAK,SAAS,yBAAyB,CAExC,IAAI;EAExB,IAAI,UAAU,WAAW,QAAQ,GAC/B,OAAO,yBACL;GAAE,MAAM;GAAS,QAAQ;IAAE,MAAM;IAAU,MAAM,OAAO;GAAK;EAAE,GAC/D,QACF;EAEF,OAAO,cAAc,OAAO,IAAI;CAClC;CAEA,IAAI,OAAO,SAAS,OAClB,OAAO;CAIT,IAAI,OAAO,SAAS,aAAa,MAAM,QAAQ,OAAO,OAAO,GAAG;EAC9D,IAAI,QAAQ;EACZ,KAAK,MAAM,SAAS,OAAO,SACzB,IAAI,SAAS,QAAQ,OAAO,UAAU,YAAY,UAAU,OAAO;GACjE,MAAM,aAAa;GACnB,IAAI,WAAW,SAAS,SACtB,SAAS,yBAAyB,YAAY,QAAQ;EAE1D;EAEF,OAAO,QAAQ,IAAI,QAAQ;CAC7B;CAEA,OAAO;AACT;AAKA,SAAS,iBAAiB,OAAmC;CAC3D,IAAI,OAAO,UAAU,YAAY,MAAM,WAAW,GAChD,OAAO;CAET,IAAI,MAAM,WAAW,OAAO,GAAG;EAC7B,MAAM,QAAQ,MAAM,QAAQ,GAAG;EAC/B,OAAO,QAAQ,IAAI,IAAI,KAAK,OAAQ,MAAM,SAAS,QAAQ,KAAK,IAAK,CAAC;CACxE;CACA,IAAI,uBAAuB,KAAK,KAAK,GACnC,OAAO;CAET,OAAO,KAAK,MAAO,MAAM,SAAS,IAAK,CAAC;AAC1C;AAEA,SAAS,iBACP,OACA,gBACA,iBACA,aACQ;CACR,IAAI,SAAS,GACX,OAAO;CAET,OAAO,KAAK,IACV,iBACA,KAAK,KAAM,QAAQ,iBAAkB,eAAe,CACtD;AACF;AAMA,SAAS,qBAAqB,OAAwC;CACpE,MAAM,OAAO,MAAM;CACnB,IAAI,OAAO,SAAS,UAClB,OAAO,iBAAiB,IAAI;CAE9B,IAAI,gBAAgB,YAClB,OAAO,KAAK;CAEd,IAAI,OAAO,MAAM,QAAQ,UACvB,OAAO,iBAAiB,MAAM,GAAG;CAEnC,MAAM,SAAU,MAAM,SAAS,MAAM;CAGrC,IAAI,UAAU,MAAM;EAClB,IAAI,OAAO,OAAO,SAAS,UACzB,OAAO,iBAAiB,OAAO,IAAI;EAErC,IAAI,OAAO,OAAO,QAAQ,UACxB,OAAO,iBAAiB,OAAO,GAAG;EAEpC,IAAI,OAAO,QAAQ,iBAAiB,YAClC,OAAO,OAAO,OAAO,MAAM;CAE/B;CACA,OAAO;AACT;AAMA,SAAS,eACP,OAC0B;CAC1B,MAAM,OACJ,OAAO,MAAM,SAAS,WAAW,uBAAuB,MAAM,IAAI,IAAI;CACxE,IAAI,SAAS,iBAAiB,SAAS,SACrC,OAAO;CAET,IAAI,SAAS,eAAe,SAAS,SACnC,OAAO;CAET,MAAM,YAAY,iBAAiB,KAAK;CACxC,MAAM,OAAO,SAAS,WAAW,aAAa,OAAO,YAAY;CACjE,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,OAAO;AACT;AAEA,SAAS,iBAAiB,OAAoD;CAC5E,IAAI,OAAO,MAAM,aAAa,UAC5B,OAAO,uBAAuB,MAAM,QAAQ;CAE9C,IAAI,OAAO,MAAM,cAAc,UAC7B,OAAO,uBAAuB,MAAM,SAAS;AAGjD;AAEA,SAAS,uBAAuB,UAA0B;CACxD,OAAO,SAAS,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,YAAY;AACnD;AAKA,SAAS,iBAAiB,MAAuB;CAC/C,MAAM,iBAAiB,uBAAuB,IAAI;CAClD,OACE,kBAAkB,IAAI,cAAc,KACpC,eAAe,WAAW,QAAQ,KAClC,eAAe,WAAW,QAAQ;AAEtC;AAGA,SAAS,gCAAgC,MAAuB;CAC9D,IAAI,OAAO,SAAS,UAAU;EAC5B,IAAI,iBAAiB,IAAI,KAAK,GAC5B,OAAO;EAET,OAAO,KAAK,IAAI,8BAA8B,KAAK,MAAM;CAC3D;CACA,IAAI,gBAAgB,eAAe,YAAY,OAAO,IAAI,GACxD,OAAO,KAAK,IACV,8BACA,KAAK,KAAM,KAAK,aAAa,IAAK,CAAC,CACrC;CAEF,OAAO;AACT;AAMA,SAAS,+BACP,OACA,UACA,eACoB;CACpB,MAAM,YAAY,iBAAiB,KAAK;CACxC,IAAI,MAAM,SAAS,WAAW,aAAa,MACzC;CAGF,MAAM,WAAW;CACjB,MAAM,cACJ,MAAM,SAAS,QAAQ,OAAO,MAAM,UAAU,WACzC,MAAM,QACP,KAAA;CACN,MAAM,OACJ,MAAM,QAAQ,MAAM,SAAS,aAAa,QAAQ,aAAa;CACjE,MAAM,YACJ,MAAM,OACN,MAAM,WACN,MAAM,UACN,aAAa,OACb,aAAa,WACb,aAAa;CAEf,IAAI,SAAS,WAAW,QAAQ,GAAG;EACjC,IAAI;EACJ,IAAI,OAAO,SAAS,UAClB,eAAe;OACV,IAAI,gBAAgB,eAAe,YAAY,OAAO,IAAI,GAAG;GAClE,MAAM,OACJ,gBAAgB,cACZ,IAAI,WAAW,IAAI,IACnB,IAAI,WAAW,KAAK,QAAQ,KAAK,YAAY,KAAK,UAAU;GAClE,eAAe,OAAO,KAAK,KAAK,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS,QAAQ;EACpE;EAEA,IAAI,gBAAgB,QAAQ,aAAa,SAAS,GAChD,OAAO,yBACL;GACE,MAAM;GACN,QAAQ;IAAE,MAAM;IAAU,MAAM;GAAa;EAC/C,GACA,QACF;EAEF,OAAO,yBACL;GACE,MAAM;GACN,WAAW,EAAE,KAAK,OAAO,cAAc,WAAW,YAAY,GAAG;EACnE,GACA,QACF;CACF;CAEA,IAAI,aAAa,mBAAmB;EAClC,IAAI,OAAO,SAAS,YAAY,KAAK,SAAS,GAAG;GAC/C,MAAM,QAAQ,KAAK,WAAW,OAAO,IAAI,KAAK,QAAQ,GAAG,IAAI;GAE7D,OAAO,4BACL;IACE,MAAM;IACN,aAAa;IACb,WAAW;IACX,MANe,SAAS,IAAI,KAAK,MAAM,QAAQ,CAAC,IAAI;GAOtD,GACA,UACA,aACF;EACF;EACA,IAAI,gBAAgB,eAAe,YAAY,OAAO,IAAI,GAAG;GAC3D,MAAM,uBAAuB,KAAK,KAAM,KAAK,aAAa,IAAK,CAAC;GAChE,MAAM,mBACJ,aAAa,WACT,gCACA;GACN,OACE,KAAK,IACH,GACA,KAAK,KAAK,uBAAuB,yBAAyB,CAC5D,IAAI;EAER;EACA,OAAO;CACT;CAEA,IAAI,SAAS,WAAW,QAAQ,KAAK,SAAS,WAAW,QAAQ,GAC/D;CAEF,OAAO,gCAAgC,IAAI;AAC7C;AAaA,SAAgB,8BACd,OACQ;CACR,MAAM,OAAO,eAAe,KAAK;CACjC,IAAI,QAAQ,MACV,OAAO;CAET,IAAI;CACJ,IAAI,MAAM,SAAS,eAAe;EAChC,MAAM,QAAQ,MAAM;EACpB,QAAQ,iBAAiB,OAAO,IAAI;CACtC,OAAO,IAAI,MAAM,SAAS,aAAa;EACrC,MAAM,QAAQ,MAAM;EACpB,QAAQ,iBAAiB,OAAO,UAAU,WAAW,QAAQ,OAAO,GAAG;CACzE,OACE,QAAQ,qBAAqB,KAAK;CAEpC,IAAI,SAAS,SACX,OAAO,iBACL,OACA,wBACA,yBACA,yBACF;CAEF,OAAO,iBACL,OACA,wBACA,yBACA,yBACF;AACF;AAEA,MAAM,aAAuD,CAAC;AAE9D,eAAe,aACb,WAAyB,cACL;CACpB,MAAM,SAAS,WAAW;CAC1B,IAAI,QACF,OAAO;CAMT,MAAM,WAAW,IAAI,UAHnB,aAAa,WACT,MAAM,OAAO,kCACb,MAAM,OAAO,mCACgB;CACnC,WAAW,YAAY;CACvB,OAAO;AACT;AAEA,SAAgB,iBAAiB,OAA6B;CAC5D,IAAI,MAAM,YAAY,CAAC,CAAC,SAAS,QAAQ,GACvC,OAAO;CAET,OAAO;AACT;AAMA,SAAS,4BACP,YACS;CACT,IAAI,WAAW,aAAa,GAC1B,OAAO;CAET,WAAW;CACX,OAAO;AACT;AAEA,SAAS,gBACP,OACA,YACS;CACT,IAAI,UAAyB;CAC7B,MAAM,uBAAO,IAAI,IAAY;CAC7B,IAAI;EACF,OAAO,WAAW,MAAM;GACtB,IAAI,QAAQ,OAAO,GACjB,OAAO;GAET,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,4BAA4B,UAAU,GAC9D,OAAO;GAET,KAAK,IAAI,OAAO;GAChB,MAAM,aAAa,OAAO,yBAAyB,SAAS,QAAQ;GACpE,IAAI,cAAc,MAChB,OACG,WAAW,cAAc,OAAO,WAAW,UAAU,cACtD,EAAE,WAAW;GAGjB,UAAU,OAAO,eAAe,OAAO;EACzC;CACF,QAAQ;EACN,OAAO;CACT;CACA,OAAO;AACT;AAEA,MAAM,8BAA8B,OAAO,yBACzC,YAAY,WACZ,YACF,CAAC,EAAE;AAMH,SAAS,oBAAoB,OAAwB;CACnD,IAAI;EACF,IAAI,YAAY,OAAO,KAAK,GAC1B,OAAO;CAEX,QAAQ;EACN,OAAO;CACT;CACA,IAAI,+BAA+B,MACjC,OAAO;CAET,IAAI;EACF,4BAA4B,KAAK,KAAK;EACtC,OAAO;CACT,QAAQ;EACN,OAAO;CACT;AACF;AAMA,SAAS,2BACP,OACA,KACA,YACS;CACT,MAAM,OAAO,IAAI,IAAY,CAAC,KAAK,CAAC;CACpC,IAAI;CACJ,IAAI;EACF,UAAU,OAAO,eAAe,KAAK;CACvC,QAAQ;EACN,OAAO;CACT;CAEA,OAAO,WAAW,MAAM;EACtB,IAAI,QAAQ,OAAO,GACjB,OAAO;EAET,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,4BAA4B,UAAU,GAC9D,OAAO;EAET,KAAK,IAAI,OAAO;EAChB,IAAI;EACJ,IAAI;GACF,aAAa,OAAO,yBAAyB,SAAS,GAAG;EAC3D,QAAQ;GACN,OAAO;EACT;EACA,IAAI,cAAc,MAChB,OAAO,EAAE,WAAW;EAEtB,IAAI;GACF,UAAU,OAAO,eAAe,OAAO;EACzC,QAAQ;GACN,OAAO;EACT;CACF;CACA,OAAO;AACT;AAQA,SAAgB,iCAAiC,OAAyB;CACxE,IAAI,OAAO,UAAU,UACnB,OAAO;CAET,IAAI,SAAS,QAAQ,OAAO,UAAU,UACpC,OAAO;CAET,IAAI,QAAQ,KAAK,GACf,OAAO;CAET,IAAI,oBAAoB,KAAK,GAC3B,OAAO;CAGT,MAAM,QAAkB,CAAC,KAAK;CAC9B,MAAM,uBAAO,IAAI,IAAY;CAC7B,MAAM,aAAyC,EAC7C,WAAW,sCACb;CACA,OAAO,MAAM,SAAS,GAAG;EACvB,MAAM,UAAU,MAAM,IAAI;EAC1B,IAAI,QAAQ,OAAO,GACjB,OAAO;EAET,IAAI,KAAK,IAAI,OAAO,GAClB,OAAO;EAET,KAAK,IAAI,OAAO;EAChB,IAAI,CAAC,4BAA4B,UAAU,GACzC,OAAO;EAGT,IAAI;GACF,IAAI,gBAAgB,SAAS,UAAU,GACrC,OAAO;GAET,KAAK,MAAM,OAAO,SAAS;IACzB,IAAI,CAAC,4BAA4B,UAAU,GACzC,OAAO;IAET,MAAM,aAAa,OAAO,yBAAyB,SAAS,GAAG;IAC/D,IAAI,cAAc,MAAM;KACtB,IAAI,2BAA2B,SAAS,KAAK,UAAU,GACrD,OAAO;KAET;IACF;IACA,IAAI,WAAW,eAAe,MAC5B;IAEF,IAAI,EAAE,WAAW,aACf,OAAO;IAET,IAAI,WAAW,YAAY;KACzB,IAAI,OAAO,WAAW,UAAU,UAC9B,OAAO;KAET,IACE,WAAW,SAAS,QACpB,OAAO,WAAW,UAAU,YAC5B,oBAAoB,WAAW,KAAK,GAEpC,OAAO;KAET,IACE,WAAW,SAAS,QACpB,OAAO,WAAW,UAAU,UAE5B,MAAM,KAAK,WAAW,KAAK;IAE/B;GACF;EACF,QAAQ;GACN,OAAO;EACT;CACF;CACA,OAAO;AACT;AAEA,SAAS,yBACP,OACA,eACQ;CACR,MAAM,UACJ,MAAM,SAAS,oCACX,MAAM,MAAM,GAAG,iCAAiC,IAChD;CACN,MAAM,gBAAgB,2BACpB,cAAc,OAAO,GACrB,WACF;CACA,MAAM,eAAe,MAAM,SAAS,QAAQ;CAC5C,IAAI,gBAAgB,GAClB,OAAO;CAET,OAAO,KAAK,IACV,OAAO,kBACP,gBAAgB,eAAe,sCACjC;AACF;AAEA,SAAS,+BACP,OACA,eACQ;CACR,MAAM,aAAa,gCACjB,OACA,iCACF;CACA,MAAM,gBAAgB,2BACpB,cAAc,WAAW,OAAO,GAChC,WACF;CACA,IAAI,CAAC,WAAW,WACd,OAAO;CAKT,IACE,CAAC,OAAO,cAAc,WAAW,aAAa,KAC9C,WAAW,iBAAiB,OAAO,kBAEnC,OAAO;CAGT,MAAM,eAAe,KAAK,IACxB,GACA,WAAW,gBAAgB,WAAW,QAAQ,MAChD;CACA,OAAO,KAAK,IACV,OAAO,kBACP,gBAAgB,eAAe,sCACjC;AACF;AAEA,SAAgB,wBACd,SACA,eACA,WAAyB,cACjB;CACR,MAAM,mBAAmB;CACzB,MAAM,aAAa,SACjB,yBAAyB,MAAM,aAAa;CAC9C,IAAI,QAAQ,OAAO,GACjB,MAAM,IAAI,4BAA4B;EACpC,QAAQ;EACR,MAAM;CACR,CAAC;CAYH,MAAM,yCAAyB,IAAI,IAAY;CAE/C,MAAM,gBAAgB,OAAgB,SAAuB;EAC3D,IAAI,SAAS,QAAQ,OAAO,UAAU,YAAY,QAAQ,KAAK,GAC7D,MAAM,IAAI,4BAA4B;GACpC,QAAQ;GACR;EACF,CAAC;EAEH,IAAI,MAAM,QAAQ,KAAK,GACrB,KAAK,IAAI,QAAQ,GAAG,QAAQ,MAAM,QAAQ,SAAS;GACjD,MAAM,MAAM,MAAM;GAClB,IACE,OAAO,QAAQ,YACf,OAAO,QAAQ,YACf,OAAO,QAAQ,WACf;IACA,aAAa,KAAK,GAAG,KAAK,GAAG,MAAM,EAAE;IACrC;GACF;GACA,MAAM,OAAO;GACb,IAAI,QAAQ,QAAQ,OAAO,SAAS,UAClC;GAEF,IAAI,QAAQ,IAAI,GACd,MAAM,IAAI,4BAA4B;IACpC,QAAQ;IACR,MAAM,GAAG,KAAK,GAAG,MAAM;GACzB,CAAC;GAEH,IAAI,OAAO,KAAK,SAAS,UAAU;IACjC,aAAa,+BAA+B,MAAM,SAAS;IAC3D;GACF;GACA,IAAI,KAAK,SAAA,eAAmC,KAAK,SAAS,YAAY;IACpE,MAAM,WACJ,OAAO,KAAK,OAAO,WAAW,KAAK,KAAK,KAAK,WAAW;IAC1D,IAAI,OAAO,aAAa,YAAY,SAAS,SAAS,GACpD,uBAAuB,IAAI,QAAQ;GAEvC;GACA,IAAI,KAAK,SAAA,SACP;GAGF,IACE,yBAAyB,IAAI,MAC5B,KAAK,SAAA,eACJ,KAAK,SAAS,eACd,KAAK,SAAS,WACd,KAAK,SAAS,yBACd,KAAK,SAAS,gBAChB;IACA,aAAa,KAAK,KAChB,yBAAyB,MAAM,QAAQ,IAAI,yBAC7C;IACA;GACF;GAEA,IACE,yBAAyB,IAAI,MAC5B,KAAK,SAAS,cACb,KAAK,SAAS,UACd,KAAK,SAAA,eACP;IACA,aAAa,KAAK,KAChB,4BAA4B,MAAM,UAAU,SAAS,IACnD,yBACJ;IACA;GACF;GAEA,IAAI,yBAAyB,IAAI,KAAK,KAAK,SAAS,SAAS;IAC3D,MAAM,oBAAoB,+BACxB,MACA,UACA,SACF;IACA,IAAI,qBAAqB,MAAM;KAC7B,aAAa,KAAK,KAChB,oBAAoB,yBACtB;KACA;IACF;GACF;GAEA,IAAI,yBAAyB,IAAI,KAAK,iBAAiB,KAAK,IAAI,GAAG;IACjE,MAAM,mBAAmB,8BAA8B,IAAI;IAC3D,IAAI,mBAAmB,GAAG;KACxB,aAAa,KAAK,KAChB,mBAAmB,yBACrB;KACA;IACF;GACF;GAEA,IAAI,KAAK,SAAA,eAAmC,KAAK,aAAa,MAAM;IAClE,MAAM,WAAW,KAAK,UAAU;IAChC,IAAI,OAAO,aAAa,YAAY,SAAS,SAAS,GACpD,aAAa,UAAU,QAAQ;IAEjC,MAAM,OAAO,KAAK,UAAU;IAC5B,IAAI,QAAQ,MACV,aACE,OAAO,SAAS,WACZ,UAAU,IAAI,IACd,+BAA+B,MAAM,SAAS;IAEtD,MAAM,SAAS,KAAK,UAAU;IAC9B,IAAI,UAAU,MACZ,aAAa,QAAQ,GAAG,KAAK,GAAG,MAAM,mBAAmB;IAE3D;GACF;GAEA,MAAM,cAAc,KAAK,KAAK;GAC9B,IAAI,eAAe,MAAM;IACvB,aAAa,+BAA+B,MAAM,SAAS;IAC3D;GACF;GAEA,aAAa,aAAa,GAAG,KAAK,GAAG,MAAM,IAAI,KAAK,MAAM;EAC5D;OACK,IAAI,OAAO,UAAU,UAC1B,aAAa,UAAU,KAAK;OACvB,IAAI,OAAO,UAAU,UAC1B,aAAa,UAAU,MAAM,SAAS,CAAC;OAClC,IAAI,OAAO,UAAU,WAC1B,aAAa,UAAU,MAAM,SAAS,CAAC;OAClC,IAAI,SAAS,QAAQ,OAAO,UAAU,UAC3C,aAAa,+BAA+B,OAAO,SAAS;CAEhE;CAEA,MAAM,sBAAsB,QAAQ;CACpC,MAAM,mBACJ,uBAAuB,QAAQ,OAAO,wBAAwB,WAC1D,sBACA,KAAA;CACN,IAAI,oBAAoB,QAAQ,QAAQ,gBAAgB,GACtD,MAAM,IAAI,4BAA4B;EACpC,QAAQ;EACR,MAAM;CACR,CAAC;CAEH,IAAI;CACJ,IAAI;EACF,iBACE,oBAAoB,OAChB,OAAO,yBAAyB,kBAAkB,MAAM,IACxD,KAAA;CACR,QAAQ;EACN,MAAM,IAAI,4BAA4B;GACpC,QAAQ;GACR,MAAM;EACR,CAAC;CACH;CACA,IAAI,kBAAkB,QAAQ,EAAE,WAAW,iBACzC,MAAM,IAAI,4BAA4B;EACpC,QAAQ;EACR,MAAM;CACR,CAAC;CAGH,IAAI,YAAY;CAChB,MAAM,cAAc,QAAQ,QAAQ;CAKpC,IAHE,gBAAgB,UAChB,gBAAgB,UAAU,0BAC1B,4BAA4B,QAAQ,OAAO,KACjB,OAAO,QAAQ,YAAY,UACrD,aAAa,KAAK,KAChB,yBACE;EACE,MAAM;EACN,WAAW,QAAQ;CACrB,GACA,QACF,IAAI,yBACN;MAEA,aAAa,QAAQ,SAAS,SAAS;CAEzC,2BAA2B,WAAW,SAAS;CAC/C,MAAM,cAAe,QAA6C;CAClE,IAAI,gBAAgB,QAAQ,gBAAgB,aAAa;EACvD,MAAM,YAAa,QAAsB,cAAc,CAAC;EACxD,IAAI,QAAQ,SAAS,GACnB,MAAM,IAAI,4BAA4B;GACpC,QAAQ;GACR,MAAM;EACR,CAAC;EAEH,KAAK,MAAM,YAAY,WAAW;GAChC,IAAI,QAAQ,QAAQ,GAClB,MAAM,IAAI,4BAA4B;IACpC,QAAQ;IACR,MAAM;GACR,CAAC;GAEH,IACE,OAAO,SAAS,OAAO,YACvB,uBAAuB,IAAI,SAAS,EAAE,GAEtC;GAEF,IAAI,OAAO,SAAS,SAAS,YAAY,SAAS,KAAK,SAAS,GAC9D,aAAa,UAAU,SAAS,IAAI;GAEtC,MAAM,OAAgB,SAAS;GAC/B,IAAI,QAAQ,MACV,aACE,OAAO,SAAS,WACZ,UAAU,IAAI,IACd,+BAA+B,MAAM,SAAS;EAExD;EACA,IAAI;EACJ,IAAI;GACF,qBACE,oBAAoB,OAChB,OAAO,yBAAyB,kBAAkB,eAAe,IACjE,KAAA;EACR,QAAQ;GACN,MAAM,IAAI,4BAA4B;IACpC,QAAQ;IACR,MAAM;GACR,CAAC;EACH;EACA,IAAI,sBAAsB,MAAM;GAC9B,IAAI,EAAE,WAAW,qBACf,MAAM,IAAI,4BAA4B;IACpC,QAAQ;IACR,MAAM;GACR,CAAC;GAEH,IAAI,mBAAmB,SAAS,MAC9B,aAAa,+BACX,mBAAmB,OACnB,SACF;EAEJ;CACF;CACA,OAAO,2BAA2B,WAAW,SAAS;AACxD;AAQA,SAAgB,qBACd,WACA,YACA,aACwB;CACxB,MAAM,SAAiC,OAAO,OAAO,IAAI;CACzD,MAAM,aAAyD,CAAC;CAChE,IAAI,WAAW;CACf,KAAK,MAAM,CAAC,MAAM,aAAa,OAAO,QAAQ,SAAS,GAAG;EACxD,MAAM,SAAS,WAAW;EAC1B,MAAM,UAAU,KAAK,MAAM,MAAM;EACjC,OAAO,QAAQ;EACf,YAAY;EACZ,WAAW,KAAK;GAAE;GAAM,WAAW,SAAS;EAAQ,CAAC;CACvD;CACA,IAAI,WAAW,cAAc;CAC7B,IAAI,YAAY,KAAK,WAAW,WAAW,GACzC,OAAO;CAET,WAAW,MAAM,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;CACnD,KAAK,IAAI,IAAI,GAAG,WAAW,GAAG,KAAK,IAAI,KAAK,WAAW,QAAQ;EAC7D,OAAO,WAAW,EAAE,CAAC,SAAS;EAC9B;CACF;CACA,OAAO;AACT;AAOA,MAAM,0BAA0B;AAMhC,MAAa,qBAAqB,OAChC,WAAyB,iBACqB;CAC9C,MAAM,MAAM,MAAM,aAAa,QAAQ;CACvC,MAAM,eAAe,SAAyB,IAAI,MAAM,IAAI;CAC5D,MAAM,WAAW,aAAa;CAC9B,QAAQ,YAAiC;EACvC,MAAM,QAAQ,wBAAwB,SAAS,aAAa,QAAQ;EAIpE,OAAO,2BAHgB,WACnB,KAAK,KAAK,QAAQ,uBAAuB,IACzC,OAC8C,SAAS;CAC7D;AACF;AAGA,MAAa,sBAAsB;CACjC,MAAM,aAA4B,CAElC;CAEA,QAAc;EACZ,KAAK,MAAM,OAAO,OAAO,KAAK,UAAU,GACtC,OAAO,WAAW;CAEtB;CAEA,gBAAyB;EACvB,OAAO,OAAO,KAAK,UAAU,CAAC,CAAC,SAAS;CAC1C;AACF"}
1	+ {"version":3,"file":"tokens.mjs","names":[],"sources":["../../../src/utils/tokens.ts"],"mappings":";;;;;AAmBA,IAAa,8BAAb,cAAiD,MAAM;CACrD,OAAgB;CAChB;CACA;CAEA,YAAY,EACV,QACA,QAIC;EACD,MACE,KAAK,UAAU;GACb,MAAM;GACN;GACA;EACF,CAAC,CACH;EACA,KAAK,OAAO;EACZ,KAAK,SAAS;EACd,KAAK,OAAO;CACd;AACF;AAEA,SAAS,2BAA2B,OAAe,MAAsB;CACvE,IACE,CAAC,OAAO,cAAc,KAAK,KAC3B,QAAQ,KACR,SAAS,OAAO,kBAEhB,MAAM,IAAI,4BAA4B;EACpC,QAAQ;EACR;CACF,CAAC;CAEH,OAAO;AACT;AAGA,MAAM,6BAA6B;AAEnC,MAAM,0BAA0B;AAEhC,MAAM,0BAA0B;AAEhC,MAAM,yBAAyB;AAE/B,MAAM,+BAA+B;AAIrC,MAAa,4BAA4B;AAMzC,MAAM,gCAAgC;AAEtC,MAAM,6BAA6B;AAInC,MAAM,4BAA4B;AAElC,MAAM,+BAA+B;AAUrC,MAAM,0BAA0B;AAEhC,MAAM,0BAA0B;AAEhC,MAAM,yBAAyB;AAC/B,MAAM,yBAAyB;AAE/B,MAAM,4BAA4B;AAClC,MAAM,4BAA4B;AAGlC,MAAM,oBAAoB,IAAI,IAAI;CAChC;CACA;CACA;CACA;CACA;AACF,CAAC;AAMD,MAAM,oCAAoC;AAE1C,MAAM,yCAAyC;AAC/C,MAAM,wCAAwC;AAE9C,MAAM,8BAA8B;AAEpC,MAAM,0CAA0C;AAOhD,SAAgB,uBACd,YAC0C;CAC1C,MAAM,MAAM,WAAW,WAAW,OAAO,IACrC,WAAW,MAAM,WAAW,QAAQ,GAAG,IAAI,CAAC,IAC5C;CAEJ,IAAI,IAAI,SAAS,IACf,OAAO;CAGT,MAAM,QAAQ,IAAI,WAAW,OAAO,KAAK,IAAI,MAAM,GAAG,EAAE,GAAG,QAAQ,CAAC;CAEpE,IAAI,MAAM,OAAO,OAAQ,MAAM,OAAO,IAMpC,OAAO;EAAE,OAHN,MAAM,OAAO,KAAO,MAAM,OAAO,KAAO,MAAM,OAAO,IAAK,MAAM;EAGnD,QADb,MAAM,OAAO,KAAO,MAAM,OAAO,KAAO,MAAM,OAAO,IAAK,MAAM;CAC5C;CAGzB,IAAI,MAAM,OAAO,OAAQ,MAAM,OAAO,KAAM;EAE1C,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,SAAS,GAAG,KACpC,IACE,MAAM,OAAO,QACZ,MAAM,IAAI,OAAO,OAAQ,MAAM,IAAI,OAAO,MAC3C;GACA,MAAM,SAAU,MAAM,IAAI,MAAM,IAAK,MAAM,IAAI;GAE/C,OAAO;IAAE,OADM,MAAM,IAAI,MAAM,IAAK,MAAM,IAAI;IAC9B;GAAO;EACzB;EAEF,OAAO;CACT;CAEA,IAAI,MAAM,OAAO,MAAQ,MAAM,OAAO,MAAQ,MAAM,OAAO,IAIzD,OAAO;EAAE,OAFK,MAAM,KAAM,MAAM,MAAM;EAEtB,QADD,MAAM,KAAM,MAAM,MAAM;CAChB;CAGzB,IACE,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,QAAQ,MACd,MAAM,QAAQ,IACd;EAEA,IAAI,MAAM,SAAS,IAGjB,OAAO;GAAE,QAFM,MAAM,MAAO,MAAM,OAAO,KAAM;GAE/B,SADA,MAAM,MAAO,MAAM,OAAO,KAAM;EACzB;EAEzB,OAAO;CACT;CAEA,OAAO;AACT;AAGA,SAAgB,6BACd,OACA,QACQ;CACR,OAAO,KAAK,IACV,4BACA,KAAK,KAAM,QAAQ,SAAU,uBAAuB,CACtD;AACF;AAGA,SAAgB,0BACd,OACA,QACA,SAAiB,QACT;CACR,IAAI,WAAW,OACb,OAAO;CAKT,OAAO,0BAFL,KAAK,KAAK,QAAQ,sBAAsB,IACxC,KAAK,KAAK,SAAS,sBAAsB,IACF;AAC3C;AAOA,SAAgB,yBACd,OACA,UACQ;CACR,IAAI;CAEJ,IACE,MAAM,SAAA,eACN,MAAM,SAAS,eACf,MAAM,SAAS,yBACf,MAAM,SAAS,eACf;EACA,MAAM,WAAW,MAAM;EACvB,MAAM,MAAM,OAAO,aAAa,WAAW,WAAW,UAAU;EAChE,IAAI,OAAO,QAAQ,YAAY,IAAI,WAAW,OAAO,GACnD,aAAa;OACR,IAAI,OAAO,MAAM,YAAY,YAAY,MAAM,YAAY,IAChE,OAAO,MAAM,WAAW,QACpB,0BACA;OAEJ,OAAO;CAEX,OAAO,IAAI,MAAM,SAAS,SAAS;EACjC,MAAM,SAAS,MAAM;EACrB,IAAI,QAAQ,SAAS,YAAY,OAAO,OAAO,SAAS,UACtD,aAAa,OAAO;OAEpB,OAAO;CAEX,OACE,OAAO;CAGT,MAAM,OAAO,uBAAuB,UAAU;CAC9C,IAAI,QAAQ,MACV,OAAO;CAGT,IAAI,aAAa,UACf,OAAO,6BAA6B,KAAK,OAAO,KAAK,MAAM;CAE7D,OAAO,0BAA0B,KAAK,OAAO,KAAK,MAAM;AAC1D;AAWA,SAAgB,4BACd,OACA,UACA,eACQ;CACR,MAAM,mBACJ,aAAa,WACT,gCACA;CAGN,MAAM,aAAa,MAAM;CACzB,IAAI,OAAO,eAAe,UAAU;EAClC,MAAM,YAAa,MAAM,aAAoC,GAAA,CAAI,MAC/D,GACF,CAAC,CAAC;EAEF,IAAI,eAAe,UAAU,OAAO,MAAM,SAAS,UACjD,OAAO,cAAc,MAAM,IAAc;EAG3C,IAAI,eAAe,YAAY,OAAO,MAAM,SAAS,UAAU;GAC7D,IAAI,aAAa,qBAAqB,aAAa,IAKjD,OAJqB,KAAK,IACxB,GACA,KAAK,KAAM,MAAM,KAAgB,SAAS,yBAAyB,CAEnD,IAAI;GAGxB,IAAI,SAAS,WAAW,QAAQ,GAC9B,OAAO,yBACL;IACE,GAAG;IACH,MAAM;IACN,QAAQ;KAAE,MAAM;KAAU,MAAM,MAAM;IAAK;GAC7C,GACA,QACF;GAEF,OAAO,cAAc,MAAM,IAAc;EAC3C;EAEA,IAAI,eAAe,OACjB,OAAO;EAGT,OAAO;CACT;CAGA,MAAM,SAAS,MAAM;CASrB,IAAI,UAAU,MACZ,OAAO;CAGT,IAAI,OAAO,SAAS,UAAU,OAAO,OAAO,SAAS,UACnD,OAAO,cAAc,OAAO,IAAI;CAGlC,IAAI,OAAO,SAAS,YAAY,OAAO,OAAO,SAAS,UAAU;EAC/D,MAAM,aAAa,OAAO,cAAc,GAAA,CAAI,MAAM,GAAG,CAAC,CAAC;EACvD,IAAI,cAAc,qBAAqB,cAAc,IAKnD,OAJqB,KAAK,IACxB,GACA,KAAK,KAAK,OAAO,KAAK,SAAS,yBAAyB,CAExC,IAAI;EAExB,IAAI,UAAU,WAAW,QAAQ,GAC/B,OAAO,yBACL;GAAE,MAAM;GAAS,QAAQ;IAAE,MAAM;IAAU,MAAM,OAAO;GAAK;EAAE,GAC/D,QACF;EAEF,OAAO,cAAc,OAAO,IAAI;CAClC;CAEA,IAAI,OAAO,SAAS,OAClB,OAAO;CAIT,IAAI,OAAO,SAAS,aAAa,MAAM,QAAQ,OAAO,OAAO,GAAG;EAC9D,IAAI,QAAQ;EACZ,KAAK,MAAM,SAAS,OAAO,SACzB,IAAI,SAAS,QAAQ,OAAO,UAAU,YAAY,UAAU,OAAO;GACjE,MAAM,aAAa;GACnB,IAAI,WAAW,SAAS,SACtB,SAAS,yBAAyB,YAAY,QAAQ;EAE1D;EAEF,OAAO,QAAQ,IAAI,QAAQ;CAC7B;CAEA,OAAO;AACT;AAKA,SAAS,iBAAiB,OAAmC;CAC3D,IAAI,OAAO,UAAU,YAAY,MAAM,WAAW,GAChD,OAAO;CAET,IAAI,MAAM,WAAW,OAAO,GAAG;EAC7B,MAAM,QAAQ,MAAM,QAAQ,GAAG;EAC/B,OAAO,QAAQ,IAAI,IAAI,KAAK,OAAQ,MAAM,SAAS,QAAQ,KAAK,IAAK,CAAC;CACxE;CACA,IAAI,uBAAuB,KAAK,KAAK,GACnC,OAAO;CAET,OAAO,KAAK,MAAO,MAAM,SAAS,IAAK,CAAC;AAC1C;AAEA,SAAS,iBACP,OACA,gBACA,iBACA,aACQ;CACR,IAAI,SAAS,GACX,OAAO;CAET,OAAO,KAAK,IACV,iBACA,KAAK,KAAM,QAAQ,iBAAkB,eAAe,CACtD;AACF;AAMA,SAAS,qBAAqB,OAAwC;CACpE,MAAM,OAAO,MAAM;CACnB,IAAI,OAAO,SAAS,UAClB,OAAO,iBAAiB,IAAI;CAE9B,IAAI,gBAAgB,YAClB,OAAO,KAAK;CAEd,IAAI,OAAO,MAAM,QAAQ,UACvB,OAAO,iBAAiB,MAAM,GAAG;CAEnC,MAAM,SAAU,MAAM,SAAS,MAAM;CAGrC,IAAI,UAAU,MAAM;EAClB,IAAI,OAAO,OAAO,SAAS,UACzB,OAAO,iBAAiB,OAAO,IAAI;EAErC,IAAI,OAAO,OAAO,QAAQ,UACxB,OAAO,iBAAiB,OAAO,GAAG;EAEpC,IAAI,OAAO,QAAQ,iBAAiB,YAClC,OAAO,OAAO,OAAO,MAAM;CAE/B;CACA,OAAO;AACT;AAMA,SAAS,eACP,OAC0B;CAC1B,MAAM,OACJ,OAAO,MAAM,SAAS,WAAW,uBAAuB,MAAM,IAAI,IAAI;CACxE,IAAI,SAAS,iBAAiB,SAAS,SACrC,OAAO;CAET,IAAI,SAAS,eAAe,SAAS,SACnC,OAAO;CAET,MAAM,YAAY,iBAAiB,KAAK;CACxC,MAAM,OAAO,SAAS,WAAW,aAAa,OAAO,YAAY;CACjE,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,OAAO;AACT;AAEA,SAAS,iBAAiB,OAAoD;CAC5E,IAAI,OAAO,MAAM,aAAa,UAC5B,OAAO,uBAAuB,MAAM,QAAQ;CAE9C,IAAI,OAAO,MAAM,cAAc,UAC7B,OAAO,uBAAuB,MAAM,SAAS;AAGjD;AAEA,SAAS,uBAAuB,UAA0B;CACxD,OAAO,SAAS,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,YAAY;AACnD;AAKA,SAAS,iBAAiB,MAAuB;CAC/C,MAAM,iBAAiB,uBAAuB,IAAI;CAClD,OACE,kBAAkB,IAAI,cAAc,KACpC,eAAe,WAAW,QAAQ,KAClC,eAAe,WAAW,QAAQ;AAEtC;AAGA,SAAS,gCAAgC,MAAuB;CAC9D,IAAI,OAAO,SAAS,UAAU;EAC5B,IAAI,iBAAiB,IAAI,KAAK,GAC5B,OAAO;EAET,OAAO,KAAK,IAAI,8BAA8B,KAAK,MAAM;CAC3D;CACA,IAAI,gBAAgB,eAAe,YAAY,OAAO,IAAI,GACxD,OAAO,KAAK,IACV,8BACA,KAAK,KAAM,KAAK,aAAa,IAAK,CAAC,CACrC;CAEF,OAAO;AACT;AAMA,SAAS,+BACP,OACA,UACA,eACoB;CACpB,MAAM,YAAY,iBAAiB,KAAK;CACxC,IAAI,MAAM,SAAS,WAAW,aAAa,MACzC;CAGF,MAAM,WAAW;CACjB,MAAM,cACJ,MAAM,SAAS,QAAQ,OAAO,MAAM,UAAU,WACzC,MAAM,QACP,KAAA;CACN,MAAM,OACJ,MAAM,QAAQ,MAAM,SAAS,aAAa,QAAQ,aAAa;CACjE,MAAM,YACJ,MAAM,OACN,MAAM,WACN,MAAM,UACN,aAAa,OACb,aAAa,WACb,aAAa;CAEf,IAAI,SAAS,WAAW,QAAQ,GAAG;EACjC,IAAI;EACJ,IAAI,OAAO,SAAS,UAClB,eAAe;OACV,IAAI,gBAAgB,eAAe,YAAY,OAAO,IAAI,GAAG;GAClE,MAAM,OACJ,gBAAgB,cACZ,IAAI,WAAW,IAAI,IACnB,IAAI,WAAW,KAAK,QAAQ,KAAK,YAAY,KAAK,UAAU;GAClE,eAAe,OAAO,KAAK,KAAK,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS,QAAQ;EACpE;EAEA,IAAI,gBAAgB,QAAQ,aAAa,SAAS,GAChD,OAAO,yBACL;GACE,MAAM;GACN,QAAQ;IAAE,MAAM;IAAU,MAAM;GAAa;EAC/C,GACA,QACF;EAEF,OAAO,yBACL;GACE,MAAM;GACN,WAAW,EAAE,KAAK,OAAO,cAAc,WAAW,YAAY,GAAG;EACnE,GACA,QACF;CACF;CAEA,IAAI,aAAa,mBAAmB;EAClC,IAAI,OAAO,SAAS,YAAY,KAAK,SAAS,GAAG;GAC/C,MAAM,QAAQ,KAAK,WAAW,OAAO,IAAI,KAAK,QAAQ,GAAG,IAAI;GAE7D,OAAO,4BACL;IACE,MAAM;IACN,aAAa;IACb,WAAW;IACX,MANe,SAAS,IAAI,KAAK,MAAM,QAAQ,CAAC,IAAI;GAOtD,GACA,UACA,aACF;EACF;EACA,IAAI,gBAAgB,eAAe,YAAY,OAAO,IAAI,GAAG;GAC3D,MAAM,uBAAuB,KAAK,KAAM,KAAK,aAAa,IAAK,CAAC;GAChE,MAAM,mBACJ,aAAa,WACT,gCACA;GACN,OACE,KAAK,IACH,GACA,KAAK,KAAK,uBAAuB,yBAAyB,CAC5D,IAAI;EAER;EACA,OAAO;CACT;CAEA,IAAI,SAAS,WAAW,QAAQ,KAAK,SAAS,WAAW,QAAQ,GAC/D;CAEF,OAAO,gCAAgC,IAAI;AAC7C;AAaA,SAAgB,8BACd,OACQ;CACR,MAAM,OAAO,eAAe,KAAK;CACjC,IAAI,QAAQ,MACV,OAAO;CAET,IAAI;CACJ,IAAI,MAAM,SAAS,eAAe;EAChC,MAAM,QAAQ,MAAM;EACpB,QAAQ,iBAAiB,OAAO,IAAI;CACtC,OAAO,IAAI,MAAM,SAAS,aAAa;EACrC,MAAM,QAAQ,MAAM;EACpB,QAAQ,iBAAiB,OAAO,UAAU,WAAW,QAAQ,OAAO,GAAG;CACzE,OACE,QAAQ,qBAAqB,KAAK;CAEpC,IAAI,SAAS,SACX,OAAO,iBACL,OACA,wBACA,yBACA,yBACF;CAEF,OAAO,iBACL,OACA,wBACA,yBACA,yBACF;AACF;AAEA,MAAM,aAAuD,CAAC;AAE9D,eAAe,aACb,WAAyB,cACL;CACpB,MAAM,SAAS,WAAW;CAC1B,IAAI,QACF,OAAO;CAMT,MAAM,WAAW,IAAI,UAHnB,aAAa,WACT,MAAM,OAAO,kCACb,MAAM,OAAO,mCACgB;CACnC,WAAW,YAAY;CACvB,OAAO;AACT;AAEA,SAAgB,iBAAiB,OAA6B;CAC5D,IAAI,MAAM,YAAY,CAAC,CAAC,SAAS,QAAQ,GACvC,OAAO;CAET,OAAO;AACT;AAMA,SAAS,4BACP,YACS;CACT,IAAI,WAAW,aAAa,GAC1B,OAAO;CAET,WAAW;CACX,OAAO;AACT;AAEA,SAAS,gBACP,OACA,YACS;CACT,IAAI,UAAyB;CAC7B,MAAM,uBAAO,IAAI,IAAY;CAC7B,IAAI;EACF,OAAO,WAAW,MAAM;GACtB,IAAI,QAAQ,OAAO,GACjB,OAAO;GAET,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,4BAA4B,UAAU,GAC9D,OAAO;GAET,KAAK,IAAI,OAAO;GAChB,MAAM,aAAa,OAAO,yBAAyB,SAAS,QAAQ;GACpE,IAAI,cAAc,MAChB,OACG,WAAW,cAAc,OAAO,WAAW,UAAU,cACtD,EAAE,WAAW;GAGjB,UAAU,OAAO,eAAe,OAAO;EACzC;CACF,QAAQ;EACN,OAAO;CACT;CACA,OAAO;AACT;AAEA,MAAM,8BAA8B,OAAO,yBACzC,YAAY,WACZ,YACF,CAAC,EAAE;AAMH,SAAS,oBAAoB,OAAwB;CACnD,IAAI;EACF,IAAI,YAAY,OAAO,KAAK,GAC1B,OAAO;CAEX,QAAQ;EACN,OAAO;CACT;CACA,IAAI,+BAA+B,MACjC,OAAO;CAET,IAAI;EACF,4BAA4B,KAAK,KAAK;EACtC,OAAO;CACT,QAAQ;EACN,OAAO;CACT;AACF;AAMA,SAAS,2BACP,OACA,KACA,YACS;CACT,MAAM,OAAO,IAAI,IAAY,CAAC,KAAK,CAAC;CACpC,IAAI;CACJ,IAAI;EACF,UAAU,OAAO,eAAe,KAAK;CACvC,QAAQ;EACN,OAAO;CACT;CAEA,OAAO,WAAW,MAAM;EACtB,IAAI,QAAQ,OAAO,GACjB,OAAO;EAET,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,4BAA4B,UAAU,GAC9D,OAAO;EAET,KAAK,IAAI,OAAO;EAChB,IAAI;EACJ,IAAI;GACF,aAAa,OAAO,yBAAyB,SAAS,GAAG;EAC3D,QAAQ;GACN,OAAO;EACT;EACA,IAAI,cAAc,MAChB,OAAO,EAAE,WAAW;EAEtB,IAAI;GACF,UAAU,OAAO,eAAe,OAAO;EACzC,QAAQ;GACN,OAAO;EACT;CACF;CACA,OAAO;AACT;AAQA,SAAgB,iCAAiC,OAAyB;CACxE,IAAI,OAAO,UAAU,UACnB,OAAO;CAET,IAAI,SAAS,QAAQ,OAAO,UAAU,UACpC,OAAO;CAET,IAAI,QAAQ,KAAK,GACf,OAAO;CAET,IAAI,oBAAoB,KAAK,GAC3B,OAAO;CAGT,MAAM,QAAkB,CAAC,KAAK;CAC9B,MAAM,uBAAO,IAAI,IAAY;CAC7B,MAAM,aAAyC,EAC7C,WAAW,sCACb;CACA,OAAO,MAAM,SAAS,GAAG;EACvB,MAAM,UAAU,MAAM,IAAI;EAC1B,IAAI,QAAQ,OAAO,GACjB,OAAO;EAET,IAAI,KAAK,IAAI,OAAO,GAClB,OAAO;EAET,KAAK,IAAI,OAAO;EAChB,IAAI,CAAC,4BAA4B,UAAU,GACzC,OAAO;EAGT,IAAI;GACF,IAAI,gBAAgB,SAAS,UAAU,GACrC,OAAO;GAET,KAAK,MAAM,OAAO,SAAS;IACzB,IAAI,CAAC,4BAA4B,UAAU,GACzC,OAAO;IAET,MAAM,aAAa,OAAO,yBAAyB,SAAS,GAAG;IAC/D,IAAI,cAAc,MAAM;KACtB,IAAI,2BAA2B,SAAS,KAAK,UAAU,GACrD,OAAO;KAET;IACF;IACA,IAAI,WAAW,eAAe,MAC5B;IAEF,IAAI,EAAE,WAAW,aACf,OAAO;IAET,IAAI,WAAW,YAAY;KACzB,IAAI,OAAO,WAAW,UAAU,UAC9B,OAAO;KAET,IACE,WAAW,SAAS,QACpB,OAAO,WAAW,UAAU,YAC5B,oBAAoB,WAAW,KAAK,GAEpC,OAAO;KAET,IACE,WAAW,SAAS,QACpB,OAAO,WAAW,UAAU,UAE5B,MAAM,KAAK,WAAW,KAAK;IAE/B;GACF;EACF,QAAQ;GACN,OAAO;EACT;CACF;CACA,OAAO;AACT;AAEA,SAAS,yBACP,OACA,eACQ;CACR,IAAI,YAAY;CAChB,KACE,IAAI,SAAS,GACb,SAAS,MAAM,QACf,UAAU,6BACV;EACA,MAAM,cAAc,2BAClB,cAAc,MAAM,MAAM,QAAQ,SAAS,2BAA2B,CAAC,GACvE,WACF;EACA,MAAM,iBACJ,SAAS,IAAI,0CAA0C;EACzD,YAAY,KAAK,IACf,OAAO,kBACP,YAAY,cAAc,cAC5B;CACF;CACA,OAAO;AACT;AAEA,SAAS,+BACP,OACA,eACQ;CACR,MAAM,aAAa,gCACjB,OACA,iCACF;CACA,MAAM,gBAAgB,2BACpB,cAAc,WAAW,OAAO,GAChC,WACF;CACA,IAAI,CAAC,WAAW,WACd,OAAO;CAKT,IACE,CAAC,OAAO,cAAc,WAAW,aAAa,KAC9C,WAAW,iBAAiB,OAAO,kBAEnC,OAAO;CAGT,MAAM,eAAe,KAAK,IACxB,GACA,WAAW,gBAAgB,WAAW,QAAQ,MAChD;CACA,OAAO,KAAK,IACV,OAAO,kBACP,gBAAgB,eAAe,sCACjC;AACF;AAEA,SAAgB,wBACd,SACA,eACA,WAAyB,cACjB;CACR,MAAM,mBAAmB;CACzB,MAAM,aAAa,SACjB,yBAAyB,MAAM,aAAa;CAC9C,IAAI,QAAQ,OAAO,GACjB,MAAM,IAAI,4BAA4B;EACpC,QAAQ;EACR,MAAM;CACR,CAAC;CAYH,MAAM,yCAAyB,IAAI,IAAY;CAE/C,MAAM,gBAAgB,OAAgB,SAAuB;EAC3D,IAAI,SAAS,QAAQ,OAAO,UAAU,YAAY,QAAQ,KAAK,GAC7D,MAAM,IAAI,4BAA4B;GACpC,QAAQ;GACR;EACF,CAAC;EAEH,IAAI,MAAM,QAAQ,KAAK,GACrB,KAAK,IAAI,QAAQ,GAAG,QAAQ,MAAM,QAAQ,SAAS;GACjD,MAAM,MAAM,MAAM;GAClB,IACE,OAAO,QAAQ,YACf,OAAO,QAAQ,YACf,OAAO,QAAQ,WACf;IACA,aAAa,KAAK,GAAG,KAAK,GAAG,MAAM,EAAE;IACrC;GACF;GACA,MAAM,OAAO;GACb,IAAI,QAAQ,QAAQ,OAAO,SAAS,UAClC;GAEF,IAAI,QAAQ,IAAI,GACd,MAAM,IAAI,4BAA4B;IACpC,QAAQ;IACR,MAAM,GAAG,KAAK,GAAG,MAAM;GACzB,CAAC;GAEH,IAAI,OAAO,KAAK,SAAS,UAAU;IACjC,aAAa,+BAA+B,MAAM,SAAS;IAC3D;GACF;GACA,IAAI,KAAK,SAAA,eAAmC,KAAK,SAAS,YAAY;IACpE,MAAM,WACJ,OAAO,KAAK,OAAO,WAAW,KAAK,KAAK,KAAK,WAAW;IAC1D,IAAI,OAAO,aAAa,YAAY,SAAS,SAAS,GACpD,uBAAuB,IAAI,QAAQ;GAEvC;GACA,IAAI,KAAK,SAAA,SACP;GAGF,IACE,yBAAyB,IAAI,MAC5B,KAAK,SAAA,eACJ,KAAK,SAAS,eACd,KAAK,SAAS,WACd,KAAK,SAAS,yBACd,KAAK,SAAS,gBAChB;IACA,aAAa,KAAK,KAChB,yBAAyB,MAAM,QAAQ,IAAI,yBAC7C;IACA;GACF;GAEA,IACE,yBAAyB,IAAI,MAC5B,KAAK,SAAS,cACb,KAAK,SAAS,UACd,KAAK,SAAA,eACP;IACA,aAAa,KAAK,KAChB,4BAA4B,MAAM,UAAU,SAAS,IACnD,yBACJ;IACA;GACF;GAEA,IAAI,yBAAyB,IAAI,KAAK,KAAK,SAAS,SAAS;IAC3D,MAAM,oBAAoB,+BACxB,MACA,UACA,SACF;IACA,IAAI,qBAAqB,MAAM;KAC7B,aAAa,KAAK,KAChB,oBAAoB,yBACtB;KACA;IACF;GACF;GAEA,IAAI,yBAAyB,IAAI,KAAK,iBAAiB,KAAK,IAAI,GAAG;IACjE,MAAM,mBAAmB,8BAA8B,IAAI;IAC3D,IAAI,mBAAmB,GAAG;KACxB,aAAa,KAAK,KAChB,mBAAmB,yBACrB;KACA;IACF;GACF;GAEA,IAAI,KAAK,SAAA,eAAmC,KAAK,aAAa,MAAM;IAClE,MAAM,WAAW,KAAK,UAAU;IAChC,IAAI,OAAO,aAAa,YAAY,SAAS,SAAS,GACpD,aAAa,UAAU,QAAQ;IAEjC,MAAM,OAAO,KAAK,UAAU;IAC5B,IAAI,QAAQ,MACV,aACE,OAAO,SAAS,WACZ,UAAU,IAAI,IACd,+BAA+B,MAAM,SAAS;IAEtD,MAAM,SAAS,KAAK,UAAU;IAC9B,IAAI,UAAU,MACZ,aAAa,QAAQ,GAAG,KAAK,GAAG,MAAM,mBAAmB;IAE3D;GACF;GAEA,MAAM,cAAc,KAAK,KAAK;GAC9B,IAAI,eAAe,MAAM;IACvB,aAAa,+BAA+B,MAAM,SAAS;IAC3D;GACF;GAEA,aAAa,aAAa,GAAG,KAAK,GAAG,MAAM,IAAI,KAAK,MAAM;EAC5D;OACK,IAAI,OAAO,UAAU,UAC1B,aAAa,UAAU,KAAK;OACvB,IAAI,OAAO,UAAU,UAC1B,aAAa,UAAU,MAAM,SAAS,CAAC;OAClC,IAAI,OAAO,UAAU,WAC1B,aAAa,UAAU,MAAM,SAAS,CAAC;OAClC,IAAI,SAAS,QAAQ,OAAO,UAAU,UAC3C,aAAa,+BAA+B,OAAO,SAAS;CAEhE;CAEA,MAAM,sBAAsB,QAAQ;CACpC,MAAM,mBACJ,uBAAuB,QAAQ,OAAO,wBAAwB,WAC1D,sBACA,KAAA;CACN,IAAI,oBAAoB,QAAQ,QAAQ,gBAAgB,GACtD,MAAM,IAAI,4BAA4B;EACpC,QAAQ;EACR,MAAM;CACR,CAAC;CAEH,IAAI;CACJ,IAAI;EACF,iBACE,oBAAoB,OAChB,OAAO,yBAAyB,kBAAkB,MAAM,IACxD,KAAA;CACR,QAAQ;EACN,MAAM,IAAI,4BAA4B;GACpC,QAAQ;GACR,MAAM;EACR,CAAC;CACH;CACA,IAAI,kBAAkB,QAAQ,EAAE,WAAW,iBACzC,MAAM,IAAI,4BAA4B;EACpC,QAAQ;EACR,MAAM;CACR,CAAC;CAGH,IAAI,YAAY;CAChB,MAAM,cAAc,QAAQ,QAAQ;CAKpC,IAHE,gBAAgB,UAChB,gBAAgB,UAAU,0BAC1B,4BAA4B,QAAQ,OAAO,KACjB,OAAO,QAAQ,YAAY,UACrD,aAAa,KAAK,KAChB,yBACE;EACE,MAAM;EACN,WAAW,QAAQ;CACrB,GACA,QACF,IAAI,yBACN;MAEA,aAAa,QAAQ,SAAS,SAAS;CAEzC,2BAA2B,WAAW,SAAS;CAC/C,MAAM,cAAe,QAA6C;CAClE,IAAI,gBAAgB,QAAQ,gBAAgB,aAAa;EACvD,MAAM,YAAa,QAAsB,cAAc,CAAC;EACxD,IAAI,QAAQ,SAAS,GACnB,MAAM,IAAI,4BAA4B;GACpC,QAAQ;GACR,MAAM;EACR,CAAC;EAEH,KAAK,MAAM,YAAY,WAAW;GAChC,IAAI,QAAQ,QAAQ,GAClB,MAAM,IAAI,4BAA4B;IACpC,QAAQ;IACR,MAAM;GACR,CAAC;GAEH,IACE,OAAO,SAAS,OAAO,YACvB,uBAAuB,IAAI,SAAS,EAAE,GAEtC;GAEF,IAAI,OAAO,SAAS,SAAS,YAAY,SAAS,KAAK,SAAS,GAC9D,aAAa,UAAU,SAAS,IAAI;GAEtC,MAAM,OAAgB,SAAS;GAC/B,IAAI,QAAQ,MACV,aACE,OAAO,SAAS,WACZ,UAAU,IAAI,IACd,+BAA+B,MAAM,SAAS;EAExD;EACA,IAAI;EACJ,IAAI;GACF,qBACE,oBAAoB,OAChB,OAAO,yBAAyB,kBAAkB,eAAe,IACjE,KAAA;EACR,QAAQ;GACN,MAAM,IAAI,4BAA4B;IACpC,QAAQ;IACR,MAAM;GACR,CAAC;EACH;EACA,IAAI,sBAAsB,MAAM;GAC9B,IAAI,EAAE,WAAW,qBACf,MAAM,IAAI,4BAA4B;IACpC,QAAQ;IACR,MAAM;GACR,CAAC;GAEH,IAAI,mBAAmB,SAAS,MAC9B,aAAa,+BACX,mBAAmB,OACnB,SACF;EAEJ;CACF;CACA,OAAO,2BAA2B,WAAW,SAAS;AACxD;AAQA,SAAgB,qBACd,WACA,YACA,aACwB;CACxB,MAAM,SAAiC,OAAO,OAAO,IAAI;CACzD,MAAM,aAAyD,CAAC;CAChE,IAAI,WAAW;CACf,KAAK,MAAM,CAAC,MAAM,aAAa,OAAO,QAAQ,SAAS,GAAG;EACxD,MAAM,SAAS,WAAW;EAC1B,MAAM,UAAU,KAAK,MAAM,MAAM;EACjC,OAAO,QAAQ;EACf,YAAY;EACZ,WAAW,KAAK;GAAE;GAAM,WAAW,SAAS;EAAQ,CAAC;CACvD;CACA,IAAI,WAAW,cAAc;CAC7B,IAAI,YAAY,KAAK,WAAW,WAAW,GACzC,OAAO;CAET,WAAW,MAAM,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;CACnD,KAAK,IAAI,IAAI,GAAG,WAAW,GAAG,KAAK,IAAI,KAAK,WAAW,QAAQ;EAC7D,OAAO,WAAW,EAAE,CAAC,SAAS;EAC9B;CACF;CACA,OAAO;AACT;AAOA,MAAM,0BAA0B;AAMhC,MAAa,qBAAqB,OAChC,WAAyB,iBACqB;CAC9C,MAAM,MAAM,MAAM,aAAa,QAAQ;CACvC,MAAM,eAAe,SAAyB,IAAI,MAAM,IAAI;CAC5D,MAAM,WAAW,aAAa;CAC9B,QAAQ,YAAiC;EACvC,MAAM,QAAQ,wBAAwB,SAAS,aAAa,QAAQ;EAIpE,OAAO,2BAHgB,WACnB,KAAK,KAAK,QAAQ,uBAAuB,IACzC,OAC8C,SAAS;CAC7D;AACF;AAGA,MAAa,sBAAsB;CACjC,MAAM,aAA4B,CAElC;CAEA,QAAc;EACZ,KAAK,MAAM,OAAO,OAAO,KAAK,UAAU,GACtC,OAAO,WAAW;CAEtB;CAEA,gBAAyB;EACvB,OAAO,OAAO,KAAK,UAAU,CAAC,CAAC,SAAS;CAC1C;AACF"}

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "@librechat/agents",
-  "version": "3.6.6",
+  "version": "3.6.7",
   "reova": {
     "enabled": true,
     "endpoint": "https://telemetry.reo.dev/data"

package/src/utils/tokens.ts CHANGED Viewed

@@ -120,6 +120,10 @@ const MAX_STRUCTURED_TOKENIZATION_CHARS = 200_000;
 /** One UTF-16 character can encode to several tokenizer pieces. */
 const MAX_TOKENS_PER_OMITTED_STRUCTURED_CHAR = 4;
 const MAX_STRUCTURED_SAFETY_INSPECTION_WORK = 10_000;
+/** Bounds BPE work per plain-text segment without omitting any text. */
+const MAX_TEXT_TOKENIZATION_CHARS = 8_192;
+/** Covers tokenizer segmentation changes on both sides of a chunk boundary. */
+const TEXT_TOKEN_CHUNK_BOUNDARY_SAFETY_TOKENS = 4;
 /**
  * Extracts image dimensions from the first bytes of a base64-encoded
@@ -886,22 +890,24 @@ function getBoundedTextTokenCount(
   value: string,
   getTokenCount: (text: string) => number
 ): number {
-  const preview =
-    value.length > MAX_STRUCTURED_TOKENIZATION_CHARS
-      ? value.slice(0, MAX_STRUCTURED_TOKENIZATION_CHARS)
-      : value;
-  const previewTokens = ensureSafeTokenMeasurement(
-    getTokenCount(preview),
-    'tokenizer'
-  );
-  const omittedChars = value.length - preview.length;
-  if (omittedChars <= 0) {
-    return previewTokens;
+  let numTokens = 0;
+  for (
+    let offset = 0;
+    offset < value.length;
+    offset += MAX_TEXT_TOKENIZATION_CHARS
+  ) {
+    const chunkTokens = ensureSafeTokenMeasurement(
+      getTokenCount(value.slice(offset, offset + MAX_TEXT_TOKENIZATION_CHARS)),
+      'tokenizer'
+    );
+    const boundaryTokens =
+      offset > 0 ? TEXT_TOKEN_CHUNK_BOUNDARY_SAFETY_TOKENS : 0;
+    numTokens = Math.min(
+      Number.MAX_SAFE_INTEGER,
+      numTokens + chunkTokens + boundaryTokens
+    );
   }
-  return Math.min(
-    Number.MAX_SAFE_INTEGER,
-    previewTokens + omittedChars * MAX_TOKENS_PER_OMITTED_STRUCTURED_CHAR
-  );
+  return numTokens;
 }
 function getBoundedStructuredTokenCount(