@simosphere/tokenizer-speedup 0.10.0 → 0.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/dist/cli/analyze.js +1 -1
  2. package/dist/cli/config-cmd.js +1 -1
  3. package/dist/cli/index.js +1 -1
  4. package/dist/cli/init-cmd.js +1 -1
  5. package/dist/cli/setup-cmd.js +1 -1
  6. package/dist/cloud/stripe-checkout.js +1 -1
  7. package/dist/cloud/stripe-config.js +1 -1
  8. package/dist/cloud/stripe-usage.js +1 -1
  9. package/dist/cloud/stripe-webhooks.js +1 -1
  10. package/dist/cloud/subscription-store.js +1 -1
  11. package/dist/manifest.json +49 -49
  12. package/dist/mcp/index.js +1 -1
  13. package/dist/mcp/resources.js +1 -1
  14. package/dist/mcp/session.js +1 -1
  15. package/dist/mcp/tools.js +1 -1
  16. package/dist/plugin/cli.js +1 -1
  17. package/dist/plugin/compression/anonymize.js +1 -1
  18. package/dist/plugin/compression/cache.js +1 -1
  19. package/dist/plugin/compression/chunked.js +1 -1
  20. package/dist/plugin/compression/dedup.js +1 -1
  21. package/dist/plugin/compression/format-optimizer.js +1 -1
  22. package/dist/plugin/compression/index.js +1 -1
  23. package/dist/plugin/compression/pipeline.js +1 -1
  24. package/dist/plugin/compression/quality.js +1 -1
  25. package/dist/plugin/compression/rule-based.js +1 -1
  26. package/dist/plugin/compression/structural.js +1 -1
  27. package/dist/plugin/compression/text-utils.js +1 -1
  28. package/dist/plugin/compression/token-budget.js +1 -1
  29. package/dist/plugin/config-loader.js +1 -1
  30. package/dist/plugin/context-compressor.js +1 -1
  31. package/dist/plugin/hook-dispatcher.js +1 -1
  32. package/dist/plugin/metering-client.js +1 -1
  33. package/dist/plugin/metrics-collector.js +1 -1
  34. package/dist/plugin/onboarding.js +1 -1
  35. package/dist/plugin/precompact-handler.js +1 -1
  36. package/dist/plugin/report.js +1 -1
  37. package/dist/plugin/rule-analytics-store.js +1 -1
  38. package/dist/plugin/rule-analytics.js +1 -1
  39. package/dist/plugin/rule-engine.js +1 -1
  40. package/dist/plugin/rule-sharing.js +1 -1
  41. package/dist/plugin/rule-suggest.js +1 -1
  42. package/dist/plugin/session-start.js +1 -1
  43. package/dist/plugin/smart-context.js +1 -1
  44. package/dist/shared/license.js +1 -1
  45. package/dist/shared/providers/claude-provider.js +1 -1
  46. package/dist/shared/providers/openai-provider.js +1 -1
  47. package/dist/shared/token-counter.js +1 -1
  48. package/dist/shared/token-provider.js +1 -1
  49. package/package.json +1 -1
package/dist/plugin/compression/anonymize.js CHANGED
@@ -1 +1 @@
1
- const _0x2b0754=_0x3d07;function _0x54a6(){const _0x3de042=['charCodeAt','694368uTHyND','211818HfSomf','3504834IlqTfc','dateOfBirth','53OPRyQI','919224TcWIlL','set','Hiwok','1255905eyEbHN','OpRId','padStart','ZGcma','1966HUJgpm','join','type','regex','DE00-ANON-','flags','replacements','customerId','length','0.0.0.','ipv4','jkRBV','BGFHd','KBbdz','sort','550904ZuWnNA','01.01.1970-','14pEGWSI','SIhwL','entries'];_0x54a6=function(){return _0x3de042;};return _0x54a6();}(function(_0x2ce119,_0x4cc73f){const _0x5d04a4=_0x3d07,_0x3b5019=_0x2ce119();while(!![]){try{const _0x57ce3f=-parseInt(_0x5d04a4(0x136))/0x1*(parseInt(_0x5d04a4(0x13e))/0x2)+parseInt(_0x5d04a4(0x133))/0x3+parseInt(_0x5d04a4(0x137))/0x4+-parseInt(_0x5d04a4(0x13a))/0x5+-parseInt(_0x5d04a4(0x132))/0x6+-parseInt(_0x5d04a4(0x12e))/0x7*(parseInt(_0x5d04a4(0x12c))/0x8)+parseInt(_0x5d04a4(0x134))/0x9;if(_0x57ce3f===_0x4cc73f)break;else _0x3b5019['push'](_0x3b5019['shift']());}catch(_0x56abb1){_0x3b5019['push'](_0x3b5019['shift']());}}}(_0x54a6,0x207f0));import{countTokens}from'../../shared/token-counter.js';function _0x3d07(_0x2c9500,_0x2b39e1){_0x2c9500=_0x2c9500-0x127;const _0x54a605=_0x54a6();let _0x3d07c6=_0x54a605[_0x2c9500];return _0x3d07c6;}function simpleHash(_0x42b01b){const _0x291f08=_0x3d07,_0x306eb7={'Hiwok':function(_0x4e235d,_0x39f3c0){return _0x4e235d<_0x39f3c0;},'KBbdz':function(_0x2b4af6,_0x539609){return _0x2b4af6-_0x539609;},'BGFHd':function(_0x5925cb,_0x3b5a90){return _0x5925cb<<_0x3b5a90;}};let _0x11038b=0x0;for(let _0x8eb720=0x0;_0x306eb7[_0x291f08(0x139)](_0x8eb720,_0x42b01b['length']);_0x8eb720++)_0x11038b=_0x306eb7[_0x291f08(0x12a)](_0x306eb7[_0x291f08(0x129)](_0x11038b,0x5),_0x11038b)+_0x42b01b[_0x291f08(0x131)](_0x8eb720)|0x0;return Math['abs'](_0x11038b)['toString'](0x10)['slice'](0x0,0x4)['toUpperCase']()[_0x291f08(0x13c)](0x4,'0');}const PII_PATTERNS=[{'type':'email','regex':/[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}/g,'buildPseudonym':_0x30fe5d=>'user_'+_0x30fe5d+'@example.redacted'},{'type':'iban','regex':/\bDE\d{2}\s?\d{4}\s?\d{4}\s?\d{4}\s?\d{4}\s?\d{2}\b/g,'buildPseudonym':_0x4b83f3=>_0x2b0754(0x142)+_0x4b83f3},{'type':'phone','regex':/(?:\+\d{1,3}[\s\-]?)?(?:\(?\d{2,5}\)?[\s\-]?)?\d{3,}[\s\-]?\d{2,}/g,'buildPseudonym':_0x31e33c=>'+49-ANON-'+_0x31e33c},{'type':_0x2b0754(0x145),'regex':/\b(?:KD|KDN|ORD)-[A-Z0-9\-]{4,12}\b/g,'buildPseudonym':_0x52a5c0=>'KD-ANON-'+_0x52a5c0},{'type':_0x2b0754(0x135),'regex':/\b(?:\d{2}\.\d{2}\.\d{4}|\d{4}-\d{2}-\d{2})\b/g,'buildPseudonym':_0x5c6391=>_0x2b0754(0x12d)+_0x5c6391},{'type':_0x2b0754(0x127),'regex':/\b(?:\d{1,3}\.){3}\d{1,3}\b/g,'buildPseudonym':_0x147b6a=>_0x2b0754(0x147)+_0x147b6a},{'type':'address','regex':/[A-ZÄÖÜ][a-zäöüß]+(?:str\.|straße|stra(?:ss|ß)e|weg|gasse|platz|allee|ring|damm)\s+\d{1,4}\s*[a-z]?\s*,\s*\d{5}\s+[A-ZÄÖÜ][a-zäöüß\-]+/g,'buildPseudonym':_0x24a89b=>'Anonymstr.\x201,\x2000000\x20Anon-'+_0x24a89b}];export function anonymizeText(_0x422431){const _0x105f8c=_0x2b0754,_0x28b981={'ZGcma':function(_0x1e65fc,_0x40cc78){return _0x1e65fc(_0x40cc78);}},_0x3e540b=new Map(),_0xee313a=new Set(),_0x5840f4=new Map();let _0x23c566=_0x422431;for(const _0x34d789 of PII_PATTERNS){const _0x204830=new RegExp(_0x34d789['regex']['source'],_0x34d789[_0x105f8c(0x141)][_0x105f8c(0x143)]),_0x454a83=_0x23c566['match'](_0x204830);if(!_0x454a83)continue;const _0xbc4f66=[...new Set(_0x454a83)]['sort']((_0x48f0d5,_0x315a36)=>_0x315a36[_0x105f8c(0x146)]-_0x48f0d5[_0x105f8c(0x146)]);for(const _0x3707e6 of _0xbc4f66){if(_0x5840f4['has'](_0x3707e6))continue;const _0x2e25df=_0x28b981[_0x105f8c(0x13d)](simpleHash,_0x3707e6),_0x224be3=_0x34d789['buildPseudonym'](_0x2e25df);_0x5840f4[_0x105f8c(0x138)](_0x3707e6,_0x224be3),_0x3e540b[_0x105f8c(0x138)](_0x224be3,_0x3707e6),_0xee313a['add'](_0x34d789[_0x105f8c(0x140)]),_0x23c566=_0x23c566['split'](_0x3707e6)['join'](_0x224be3);}}return{'text':_0x23c566,'map':{'replacements':_0x3e540b,'piiTypesFound':[..._0xee313a]}};}export function deanonymizeText(_0x339c40,_0x599873){const _0x1a2fb7=_0x2b0754;let _0x5c713e=_0x339c40;const _0x5bcefc=[..._0x599873[_0x1a2fb7(0x144)][_0x1a2fb7(0x130)]()][_0x1a2fb7(0x12b)](([_0x411419],[_0x24a635])=>_0x24a635[_0x1a2fb7(0x146)]-_0x411419['length']);for(const [_0x2c77ca,_0x1e5bb9]of _0x5bcefc)_0x5c713e=_0x5c713e['split'](_0x2c77ca)['join'](_0x1e5bb9);return _0x5c713e;}export const anonymizeStrategy={'name':'pii-anonymize','compress'(_0x3aa80c,_0x416799){const _0x553cd1=_0x2b0754,_0x517608={'jkRBV':function(_0x53cf6b,_0x248838){return _0x53cf6b(_0x248838);},'OpRId':function(_0x1f11eb,_0x23038f){return _0x1f11eb(_0x23038f);},'SIhwL':function(_0x112b47,_0x5394e2){return _0x112b47>_0x5394e2;},'UPxkl':function(_0x3db449,_0xabea84){return _0x3db449-_0xabea84;}},_0x28696e=_0x517608[_0x553cd1(0x128)](countTokens,_0x3aa80c),{text:_0x4db566,map:_0x316eaf}=_0x517608[_0x553cd1(0x13b)](anonymizeText,_0x3aa80c),_0x3ee1c9=countTokens(_0x4db566);return{'text':_0x4db566,'originalTokens':_0x28696e,'compressedTokens':_0x3ee1c9,'compressionRatio':_0x517608[_0x553cd1(0x12f)](_0x28696e,0x0)?_0x517608['UPxkl'](0x1,_0x3ee1c9/_0x28696e):0x0,'strategyUsed':'pii-anonymize\x20('+_0x316eaf['piiTypesFound'][_0x553cd1(0x13f)](',\x20')+')','qualityEstimate':0x1,'wasTruncated':!0x1};}};
1
+ function _0x5f12(_0x44f92c,_0x4e3a00){_0x44f92c=_0x44f92c-0x1e2;const _0x4c1f3=_0x4c1f();let _0x5f1253=_0x4c1f3[_0x44f92c];return _0x5f1253;}const _0x3c87e1=_0x5f12;(function(_0x372a8f,_0x2d2aef){const _0x10976f=_0x5f12,_0x2dc375=_0x372a8f();while(!![]){try{const _0x661e69=-parseInt(_0x10976f(0x1ef))/0x1*(-parseInt(_0x10976f(0x1ff))/0x2)+-parseInt(_0x10976f(0x1e3))/0x3+-parseInt(_0x10976f(0x1ee))/0x4*(parseInt(_0x10976f(0x1fc))/0x5)+parseInt(_0x10976f(0x1fd))/0x6*(parseInt(_0x10976f(0x1f4))/0x7)+parseInt(_0x10976f(0x1f1))/0x8*(parseInt(_0x10976f(0x1f5))/0x9)+-parseInt(_0x10976f(0x1fb))/0xa+parseInt(_0x10976f(0x1fe))/0xb*(-parseInt(_0x10976f(0x1e4))/0xc);if(_0x661e69===_0x2d2aef)break;else _0x2dc375['push'](_0x2dc375['shift']());}catch(_0x244c38){_0x2dc375['push'](_0x2dc375['shift']());}}}(_0x4c1f,0xdb34e));import{countTokens}from'../../shared/token-counter.js';function _0x4c1f(){const _0x31c1fd=['pii-anonymize','sort','10974866aRXwps','8307FhDIyY','set','Anonymstr.\x201,\x2000000\x20Anon-','0.0.0.','customerId','add','1167430viyTBa','2113620pSsIjG','6kSaAXJ','121GzsBdu','64AZQcEy','match','jjXiW','join','fVseP','split','4893072RRcuZq','1603164ULrZWS','+49-ANON-','01.01.1970-','iban','pii-anonymize\x20(','slice','length','XMLaA','address','regex','4bVlxIZ','49492EeeUXf','phone','12016KECyVU'];_0x4c1f=function(){return _0x31c1fd;};return _0x4c1f();}function simpleHash(_0x2e7892){const _0x4f2315=_0x5f12,_0x455795={'vvVIw':function(_0xd86d0c,_0x2778f8){return _0xd86d0c+_0x2778f8;},'uKwmJ':function(_0x2ee96b,_0x39163f){return _0x2ee96b-_0x39163f;}};let _0x11dedd=0x0;for(let _0x3aef43=0x0;_0x3aef43<_0x2e7892[_0x4f2315(0x1ea)];_0x3aef43++)_0x11dedd=_0x455795['vvVIw'](_0x455795['uKwmJ'](_0x11dedd<<0x5,_0x11dedd),_0x2e7892['charCodeAt'](_0x3aef43))|0x0;return Math['abs'](_0x11dedd)['toString'](0x10)[_0x4f2315(0x1e9)](0x0,0x4)['toUpperCase']()['padStart'](0x4,'0');}const PII_PATTERNS=[{'type':'email','regex':/[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}/g,'buildPseudonym':_0x3d2290=>'user_'+_0x3d2290+'@example.redacted'},{'type':_0x3c87e1(0x1e7),'regex':/\bDE\d{2}\s?\d{4}\s?\d{4}\s?\d{4}\s?\d{4}\s?\d{2}\b/g,'buildPseudonym':_0x1c3e67=>'DE00-ANON-'+_0x1c3e67},{'type':_0x3c87e1(0x1f0),'regex':/(?:\+\d{1,3}[\s\-]?)?(?:\(?\d{2,5}\)?[\s\-]?)?\d{3,}[\s\-]?\d{2,}/g,'buildPseudonym':_0x59ff75=>_0x3c87e1(0x1e5)+_0x59ff75},{'type':_0x3c87e1(0x1f9),'regex':/\b(?:KD|KDN|ORD)-[A-Z0-9\-]{4,12}\b/g,'buildPseudonym':_0x2ab46d=>'KD-ANON-'+_0x2ab46d},{'type':'dateOfBirth','regex':/\b(?:\d{2}\.\d{2}\.\d{4}|\d{4}-\d{2}-\d{2})\b/g,'buildPseudonym':_0x5b9fd9=>_0x3c87e1(0x1e6)+_0x5b9fd9},{'type':'ipv4','regex':/\b(?:\d{1,3}\.){3}\d{1,3}\b/g,'buildPseudonym':_0x4235d7=>_0x3c87e1(0x1f8)+_0x4235d7},{'type':_0x3c87e1(0x1ec),'regex':/[A-ZÄÖÜ][a-zäöüß]+(?:str\.|straße|stra(?:ss|ß)e|weg|gasse|platz|allee|ring|damm)\s+\d{1,4}\s*[a-z]?\s*,\s*\d{5}\s+[A-ZÄÖÜ][a-zäöüß\-]+/g,'buildPseudonym':_0xba9b04=>_0x3c87e1(0x1f7)+_0xba9b04}];export function anonymizeText(_0x6f9afb){const _0x16f36a=_0x3c87e1,_0x1da833={'XMLaA':function(_0x199915,_0x584b5e){return _0x199915(_0x584b5e);}},_0x361366=new Map(),_0x1961d9=new Set(),_0x297a9b=new Map();let _0x4bd8b7=_0x6f9afb;for(const _0x2dcfd of PII_PATTERNS){const _0x2ece19=new RegExp(_0x2dcfd[_0x16f36a(0x1ed)]['source'],_0x2dcfd['regex']['flags']),_0x54cbcd=_0x4bd8b7[_0x16f36a(0x200)](_0x2ece19);if(!_0x54cbcd)continue;const _0x2f5484=[...new Set(_0x54cbcd)][_0x16f36a(0x1f3)]((_0x1d7820,_0x295e2a)=>_0x295e2a[_0x16f36a(0x1ea)]-_0x1d7820['length']);for(const _0x5e4d0d of _0x2f5484){if(_0x297a9b['has'](_0x5e4d0d))continue;const _0x3d67e8=_0x1da833[_0x16f36a(0x1eb)](simpleHash,_0x5e4d0d),_0x3bc411=_0x2dcfd['buildPseudonym'](_0x3d67e8);_0x297a9b[_0x16f36a(0x1f6)](_0x5e4d0d,_0x3bc411),_0x361366[_0x16f36a(0x1f6)](_0x3bc411,_0x5e4d0d),_0x1961d9[_0x16f36a(0x1fa)](_0x2dcfd['type']),_0x4bd8b7=_0x4bd8b7[_0x16f36a(0x1e2)](_0x5e4d0d)[_0x16f36a(0x202)](_0x3bc411);}}return{'text':_0x4bd8b7,'map':{'replacements':_0x361366,'piiTypesFound':[..._0x1961d9]}};}export function deanonymizeText(_0x384f84,_0x15b43b){const _0x39c915=_0x3c87e1;let _0xa52861=_0x384f84;const _0x2816fc=[..._0x15b43b['replacements']['entries']()]['sort'](([_0x500bbc],[_0x563ee6])=>_0x563ee6[_0x39c915(0x1ea)]-_0x500bbc['length']);for(const [_0x163dfa,_0x12a877]of _0x2816fc)_0xa52861=_0xa52861['split'](_0x163dfa)['join'](_0x12a877);return _0xa52861;}export const anonymizeStrategy={'name':_0x3c87e1(0x1f2),'compress'(_0x58fc8b,_0x4b5e7e){const _0x50055d=_0x3c87e1,_0x3e560e={'fVseP':function(_0x1b45ae,_0x83edae){return _0x1b45ae(_0x83edae);},'QDrdU':function(_0x2859b5,_0x459851){return _0x2859b5(_0x459851);},'jjXiW':function(_0x32d173,_0x12bbb3){return _0x32d173-_0x12bbb3;},'BRbLO':function(_0x4319ff,_0x493ca9){return _0x4319ff/_0x493ca9;}},_0x1525e5=_0x3e560e[_0x50055d(0x203)](countTokens,_0x58fc8b),{text:_0x320326,map:_0x243450}=_0x3e560e['QDrdU'](anonymizeText,_0x58fc8b),_0x4b72e3=countTokens(_0x320326);return{'text':_0x320326,'originalTokens':_0x1525e5,'compressedTokens':_0x4b72e3,'compressionRatio':_0x1525e5>0x0?_0x3e560e[_0x50055d(0x201)](0x1,_0x3e560e['BRbLO'](_0x4b72e3,_0x1525e5)):0x0,'strategyUsed':_0x50055d(0x1e8)+_0x243450['piiTypesFound'][_0x50055d(0x202)](',\x20')+')','qualityEstimate':0x1,'wasTruncated':!0x1};}};
package/dist/plugin/compression/cache.js CHANGED
@@ -1 +1 @@
1
- const _0x160256=_0x113f;function _0x44f2(){const _0x560bb5=['wijzB','stats','size','set','3220956GSuDTh','HlWcu','2438697iNEzJw','clear','32VmyAru','missCount','next','has','150wbxkld','result','euTOg','egbai','22gXtXGm','delete','expiresAt','maxEntries','NQXnE','81980XEyeFs','HLKrF','entries','abs','2182311VxHbbn','now','328363gdtOlk','get','hitCount','compress','ttlMs','4928780SKuORi','237730ggZetQ'];_0x44f2=function(){return _0x560bb5;};return _0x44f2();}(function(_0x8e3fb6,_0x18b567){const _0x5603a2=_0x113f,_0x1ff68b=_0x8e3fb6();while(!![]){try{const _0x1d0174=-parseInt(_0x5603a2(0x1b9))/0x1*(parseInt(_0x5603a2(0x1b4))/0x2)+parseInt(_0x5603a2(0x1cc))/0x3+-parseInt(_0x5603a2(0x1ca))/0x4+-parseInt(_0x5603a2(0x1c5))/0x5*(-parseInt(_0x5603a2(0x1d2))/0x6)+parseInt(_0x5603a2(0x1bf))/0x7+parseInt(_0x5603a2(0x1ce))/0x8*(parseInt(_0x5603a2(0x1bd))/0x9)+-parseInt(_0x5603a2(0x1c4))/0xa;if(_0x1d0174===_0x18b567)break;else _0x1ff68b['push'](_0x1ff68b['shift']());}catch(_0x526b97){_0x1ff68b['push'](_0x1ff68b['shift']());}}}(_0x44f2,0xc7d2d));function contentHash(_0x7e73ea){const _0x3e1d40=_0x113f,_0x554ee2={'egbai':function(_0x40f66c,_0xbdf59c){return _0x40f66c+_0xbdf59c;},'NQXnE':function(_0x516e3e,_0x14abf7){return _0x516e3e-_0x14abf7;},'MZsaN':function(_0x4befbb,_0x206258){return _0x4befbb<<_0x206258;}},_0x2b2880=_0x7e73ea['replace'](/\s+/g,'\x20')['trim']()['toLowerCase']();let _0x1038ef=0x0;for(let _0x1d527e=0x0;_0x1d527e<_0x2b2880['length'];_0x1d527e++)_0x1038ef=_0x554ee2[_0x3e1d40(0x1b3)](_0x554ee2[_0x3e1d40(0x1b8)](_0x554ee2['MZsaN'](_0x1038ef,0x5),_0x1038ef),_0x2b2880['charCodeAt'](_0x1d527e))|0x0;return Math[_0x3e1d40(0x1bc)](_0x1038ef)['toString'](0x24);}export class CompressionCache{[_0x160256(0x1b7)];['ttlMs'];['entries'];[_0x160256(0x1c1)]=0x0;['missCount']=0x0;constructor(_0x416707){const _0x17e797=_0x160256;this[_0x17e797(0x1b7)]=_0x416707?.[_0x17e797(0x1b7)]??0x64,this[_0x17e797(0x1c3)]=_0x416707?.['ttlMs']??0x493e0,this[_0x17e797(0x1bb)]=new Map();}['get'](_0x409d50){const _0x32f1d6=_0x160256,_0x22bd66=contentHash(_0x409d50),_0x4c2f10=this['entries'][_0x32f1d6(0x1c0)](_0x22bd66);if(_0x4c2f10)return Date[_0x32f1d6(0x1be)]()>_0x4c2f10['expiresAt']?(this['entries']['delete'](_0x22bd66),void this['missCount']++):(this['entries'][_0x32f1d6(0x1b5)](_0x22bd66),this[_0x32f1d6(0x1bb)]['set'](_0x22bd66,_0x4c2f10),this['hitCount']++,_0x4c2f10[_0x32f1d6(0x1d3)]);this['missCount']++;}[_0x160256(0x1c9)](_0x44fe5c,_0x1fefca){const _0x529889=_0x160256,_0x4f6e43={'euTOg':function(_0x417dda,_0x3fa4d6){return _0x417dda(_0x3fa4d6);},'piCzH':function(_0x1e59f4,_0x223110){return _0x1e59f4+_0x223110;}},_0x271c93=_0x4f6e43[_0x529889(0x1b2)](contentHash,_0x44fe5c);if(this[_0x529889(0x1bb)]['has'](_0x271c93)&&this[_0x529889(0x1bb)]['delete'](_0x271c93),this[_0x529889(0x1bb)][_0x529889(0x1c8)]>=this[_0x529889(0x1b7)]){const _0x577247=this['entries']['keys']()[_0x529889(0x1d0)]()['value'];void 0x0!==_0x577247&&this[_0x529889(0x1bb)]['delete'](_0x577247);}this[_0x529889(0x1bb)]['set'](_0x271c93,{'result':_0x1fefca,'expiresAt':_0x4f6e43['piCzH'](Date['now'](),this[_0x529889(0x1c3)])});}[_0x160256(0x1d1)](_0x44e4f0){const _0x5d3ad4=_0x160256,_0x501070={'QsOvb':function(_0x3a0eb1,_0x4a74e2){return _0x3a0eb1>_0x4a74e2;}},_0x254a70=contentHash(_0x44e4f0),_0x2e45f6=this['entries']['get'](_0x254a70);return!!_0x2e45f6&&(!_0x501070['QsOvb'](Date[_0x5d3ad4(0x1be)](),_0x2e45f6[_0x5d3ad4(0x1b6)])||(this[_0x5d3ad4(0x1bb)][_0x5d3ad4(0x1b5)](_0x254a70),!0x1));}['clear'](){const _0x25fd56=_0x160256;this['entries'][_0x25fd56(0x1cd)](),this[_0x25fd56(0x1c1)]=0x0,this['missCount']=0x0;}get[_0x160256(0x1c8)](){const _0x3e615e=_0x160256;return this[_0x3e615e(0x1bb)][_0x3e615e(0x1c8)];}get[_0x160256(0x1c7)](){const _0x18e306=_0x160256,_0x1837bc={'HLKrF':function(_0x32cdb8,_0x4f0d61){return _0x32cdb8+_0x4f0d61;},'wijzB':function(_0x3a982e,_0x35beda){return _0x3a982e===_0x35beda;},'HlWcu':function(_0x39646c,_0x530976){return _0x39646c/_0x530976;}},_0x5acca9=_0x1837bc[_0x18e306(0x1ba)](this[_0x18e306(0x1c1)],this[_0x18e306(0x1cf)]);return{'hits':this[_0x18e306(0x1c1)],'misses':this[_0x18e306(0x1cf)],'entries':this['entries'][_0x18e306(0x1c8)],'hitRate':_0x1837bc[_0x18e306(0x1c6)](0x0,_0x5acca9)?0x0:_0x1837bc[_0x18e306(0x1cb)](this['hitCount'],_0x5acca9)};}}function _0x113f(_0x5f3f67,_0xc70dce){_0x5f3f67=_0x5f3f67-0x1b2;const _0x44f2cc=_0x44f2();let _0x113f2a=_0x44f2cc[_0x5f3f67];return _0x113f2a;}export function createCachedPipeline(_0x5670a9,_0x228356){const _0x4b69e7=new CompressionCache(_0x228356);return{'compress':function(_0x7dce2e,_0x557a88){const _0x30ff7b=_0x113f,_0x282d28=_0x4b69e7['get'](_0x7dce2e);if(void 0x0!==_0x282d28)return _0x282d28;const _0x5165ca=_0x5670a9[_0x30ff7b(0x1c2)](_0x7dce2e,_0x557a88);return _0x4b69e7[_0x30ff7b(0x1c9)](_0x7dce2e,_0x5165ca),_0x5165ca;},'cache':_0x4b69e7};}
1
+ function _0x3d00(){const _0x3d4da7=['996JaIpKH','entries','stats','22vkHmcz','ttlMs','keys','get','Hkwvf','jjBrH','size','5521929dMzKKz','OWSwj','now','181959wSecJE','toString','74076QLgtSO','8IJIHgA','678572FovnMr','76wUfjzs','9415gsizJo','value','wmEEj','5426930UJeMQJ','missCount','toLowerCase','AxDlF','clear','has','thDNS','iksRf','expiresAt','compress','abs','set','NgQTM','result','safrE','delete','maxEntries','11099196JnXlUO','hitCount'];_0x3d00=function(){return _0x3d4da7;};return _0x3d00();}const _0x2729c8=_0xa12d;(function(_0x15e1b8,_0x39d07c){const _0xba2d2f=_0xa12d,_0x22141c=_0x15e1b8();while(!![]){try{const _0x38c539=parseInt(_0xba2d2f(0x100))/0x1+parseInt(_0xba2d2f(0x105))/0x2*(-parseInt(_0xba2d2f(0x102))/0x3)+parseInt(_0xba2d2f(0x104))/0x4+-parseInt(_0xba2d2f(0x106))/0x5*(parseInt(_0xba2d2f(0x11c))/0x6)+-parseInt(_0xba2d2f(0x126))/0x7*(parseInt(_0xba2d2f(0x103))/0x8)+parseInt(_0xba2d2f(0x11a))/0x9+parseInt(_0xba2d2f(0x109))/0xa*(parseInt(_0xba2d2f(0x11f))/0xb);if(_0x38c539===_0x39d07c)break;else _0x22141c['push'](_0x22141c['shift']());}catch(_0x423984){_0x22141c['push'](_0x22141c['shift']());}}}(_0x3d00,0x99eef));function contentHash(_0x21269e){const _0x47ba62=_0xa12d,_0x2fc507={'iksRf':function(_0x37fc49,_0x4fa95a){return _0x37fc49+_0x4fa95a;},'jjBrH':function(_0x55a8c1,_0x533144){return _0x55a8c1-_0x533144;},'thDNS':function(_0x445ed8,_0x46ffd2){return _0x445ed8<<_0x46ffd2;}},_0x739445=_0x21269e['replace'](/\s+/g,'\x20')['trim']()[_0x47ba62(0x10b)]();let _0x1deb58=0x0;for(let _0x3cef34=0x0;_0x3cef34<_0x739445['length'];_0x3cef34++)_0x1deb58=_0x2fc507[_0x47ba62(0x110)](_0x2fc507[_0x47ba62(0x124)](_0x2fc507[_0x47ba62(0x10f)](_0x1deb58,0x5),_0x1deb58),_0x739445['charCodeAt'](_0x3cef34))|0x0;return Math[_0x47ba62(0x113)](_0x1deb58)[_0x47ba62(0x101)](0x24);}export class CompressionCache{[_0x2729c8(0x119)];['ttlMs'];['entries'];[_0x2729c8(0x11b)]=0x0;['missCount']=0x0;constructor(_0x493a7b){const _0x4b71ae=_0x2729c8;this['maxEntries']=_0x493a7b?.['maxEntries']??0x64,this[_0x4b71ae(0x120)]=_0x493a7b?.['ttlMs']??0x493e0,this['entries']=new Map();}[_0x2729c8(0x122)](_0x1a5df1){const _0x1a2c02=_0x2729c8,_0x26ff6d={'wmEEj':function(_0x4cd15c,_0x37f893){return _0x4cd15c>_0x37f893;}},_0x11a12c=contentHash(_0x1a5df1),_0x2a9c5c=this['entries'][_0x1a2c02(0x122)](_0x11a12c);if(_0x2a9c5c)return _0x26ff6d[_0x1a2c02(0x108)](Date['now'](),_0x2a9c5c['expiresAt'])?(this['entries'][_0x1a2c02(0x118)](_0x11a12c),void this['missCount']++):(this[_0x1a2c02(0x11d)][_0x1a2c02(0x118)](_0x11a12c),this[_0x1a2c02(0x11d)][_0x1a2c02(0x114)](_0x11a12c,_0x2a9c5c),this['hitCount']++,_0x2a9c5c[_0x1a2c02(0x116)]);this['missCount']++;}[_0x2729c8(0x114)](_0x245756,_0xdef437){const _0x4e6d51=_0x2729c8,_0x29f43d={'Hkwvf':function(_0xcb32ba,_0x17e64e){return _0xcb32ba(_0x17e64e);},'AxDlF':function(_0x19607e,_0x2ab34b){return _0x19607e!==_0x2ab34b;}},_0x4f5302=_0x29f43d[_0x4e6d51(0x123)](contentHash,_0x245756);if(this['entries'][_0x4e6d51(0x10e)](_0x4f5302)&&this[_0x4e6d51(0x11d)][_0x4e6d51(0x118)](_0x4f5302),this[_0x4e6d51(0x11d)]['size']>=this[_0x4e6d51(0x119)]){const _0x39c68e=this[_0x4e6d51(0x11d)][_0x4e6d51(0x121)]()['next']()[_0x4e6d51(0x107)];_0x29f43d[_0x4e6d51(0x10c)](void 0x0,_0x39c68e)&&this[_0x4e6d51(0x11d)]['delete'](_0x39c68e);}this[_0x4e6d51(0x11d)]['set'](_0x4f5302,{'result':_0xdef437,'expiresAt':Date['now']()+this['ttlMs']});}['has'](_0x5d3345){const _0x1bbafa=_0x2729c8,_0xdc36a1={'NgQTM':function(_0x5e8f9d,_0x4655ce){return _0x5e8f9d>_0x4655ce;}},_0x574da1=contentHash(_0x5d3345),_0x1f8ad3=this['entries'][_0x1bbafa(0x122)](_0x574da1);return!!_0x1f8ad3&&(!_0xdc36a1[_0x1bbafa(0x115)](Date[_0x1bbafa(0xff)](),_0x1f8ad3[_0x1bbafa(0x111)])||(this['entries']['delete'](_0x574da1),!0x1));}['clear'](){const _0x42f0ce=_0x2729c8;this[_0x42f0ce(0x11d)][_0x42f0ce(0x10d)](),this['hitCount']=0x0,this['missCount']=0x0;}get['size'](){const _0x30872a=_0x2729c8;return this[_0x30872a(0x11d)][_0x30872a(0x125)];}get[_0x2729c8(0x11e)](){const _0x58bf08=_0x2729c8,_0xbc8f5c={'OWSwj':function(_0x8d3844,_0x36765b){return _0x8d3844===_0x36765b;},'safrE':function(_0x5a595d,_0xb2f639){return _0x5a595d/_0xb2f639;}},_0x1f12bc=this['hitCount']+this['missCount'];return{'hits':this['hitCount'],'misses':this[_0x58bf08(0x10a)],'entries':this[_0x58bf08(0x11d)]['size'],'hitRate':_0xbc8f5c[_0x58bf08(0x127)](0x0,_0x1f12bc)?0x0:_0xbc8f5c[_0x58bf08(0x117)](this['hitCount'],_0x1f12bc)};}}function _0xa12d(_0x57ae99,_0x50f5f3){_0x57ae99=_0x57ae99-0xff;const _0x3d00e9=_0x3d00();let _0xa12d59=_0x3d00e9[_0x57ae99];return _0xa12d59;}export function createCachedPipeline(_0x586f84,_0x1df07e){const _0x4ad3ea=new CompressionCache(_0x1df07e);return{'compress':function(_0x3f9891,_0xbbd14d){const _0x41d30d=_0xa12d,_0x50ae00=_0x4ad3ea[_0x41d30d(0x122)](_0x3f9891);if(void 0x0!==_0x50ae00)return _0x50ae00;const _0x23e982=_0x586f84[_0x41d30d(0x112)](_0x3f9891,_0xbbd14d);return _0x4ad3ea['set'](_0x3f9891,_0x23e982),_0x23e982;},'cache':_0x4ad3ea};}
package/dist/plugin/compression/chunked.js CHANGED
@@ -1 +1 @@
1
- function _0x1207(_0x2c6af4,_0x34ba55){_0x2c6af4=_0x2c6af4-0x14d;const _0x508268=_0x5082();let _0x120751=_0x508268[_0x2c6af4];return _0x120751;}(function(_0x16eb46,_0x406023){const _0x494c5f=_0x1207,_0x737de0=_0x16eb46();while(!![]){try{const _0x446c0c=-parseInt(_0x494c5f(0x153))/0x1+parseInt(_0x494c5f(0x14d))/0x2*(-parseInt(_0x494c5f(0x163))/0x3)+parseInt(_0x494c5f(0x168))/0x4+-parseInt(_0x494c5f(0x152))/0x5+parseInt(_0x494c5f(0x165))/0x6*(parseInt(_0x494c5f(0x155))/0x7)+-parseInt(_0x494c5f(0x15f))/0x8*(-parseInt(_0x494c5f(0x160))/0x9)+-parseInt(_0x494c5f(0x162))/0xa*(-parseInt(_0x494c5f(0x159))/0xb);if(_0x446c0c===_0x406023)break;else _0x737de0['push'](_0x737de0['shift']());}catch(_0x45ef6f){_0x737de0['push'](_0x737de0['shift']());}}}(_0x5082,0xd6c37));import{encode,decode}from'gpt-tokenizer';import{countTokens}from'../../shared/token-counter.js';import{splitSentences}from'./text-utils.js';const DEFAULT_CHUNK_SIZE=0xc8,DEFAULT_OVERLAP=0x1e,DEFAULT_SENTENCES_PER_CHUNK=0x3,BASE_QUALITY_ESTIMATE=0.88,QUALITY_VARIANCE_WEIGHT=0.12;function splitIntoOverlappingChunks(_0x1be304,_0x2edc5c,_0x1f8623){const _0x362b5a=_0x1207,_0x22ef0a={'LvRCP':function(_0x2d82e0,_0x47a941){return _0x2d82e0-_0x47a941;},'tywIG':function(_0x419469,_0x52f5fc){return _0x419469<_0x52f5fc;},'dOFvc':function(_0x2cfecc,_0x5ccf59){return _0x2cfecc(_0x5ccf59);},'PbFkp':function(_0x3722bd,_0x4ad7b4){return _0x3722bd===_0x4ad7b4;}},_0x18b775=Math['min'](_0x1f8623,_0x22ef0a[_0x362b5a(0x157)](_0x2edc5c,0x1)),_0x4be528=Math['max'](0x1,_0x2edc5c-_0x18b775),_0x596294=[];let _0x3db8bb=0x0;for(;_0x22ef0a[_0x362b5a(0x15d)](_0x3db8bb,_0x1be304['length']);){const _0x59809b=Math[_0x362b5a(0x161)](_0x3db8bb+_0x2edc5c,_0x1be304[_0x362b5a(0x15e)]),_0x557df4=_0x1be304['slice'](_0x3db8bb,_0x59809b);if(_0x596294[_0x362b5a(0x151)](_0x22ef0a[_0x362b5a(0x167)](decode,_0x557df4)),_0x22ef0a['PbFkp'](_0x59809b,_0x1be304['length']))break;_0x3db8bb+=_0x4be528;}return _0x596294;}function summarizeChunk(_0x5b2ecc,_0x208ac5){const _0x5ed023=_0x1207,_0x5305b0={'DDQfT':function(_0x4452da,_0x30ba33){return _0x4452da(_0x30ba33);},'viJhP':function(_0x1e2ab5,_0x594ee0){return _0x1e2ab5===_0x594ee0;},'lqVFl':function(_0x4c0115,_0x20cd61){return _0x4c0115-_0x20cd61;}},_0x1d7600=_0x5305b0[_0x5ed023(0x166)](splitSentences,_0x5b2ecc);if(_0x1d7600['length']<=_0x208ac5)return _0x5b2ecc;if(_0x5305b0['viJhP'](0x1,_0x208ac5))return _0x1d7600[0x0];return[..._0x1d7600['slice'](0x0,0x1),..._0x1d7600[_0x5ed023(0x154)](-_0x5305b0['lqVFl'](_0x208ac5,0x1))][_0x5ed023(0x164)]('.\x20');}export function compressChunked(_0x1fbc86,_0x49cf7c={}){const _0x567750=_0x1207,_0x42a569={'TNPnr':function(_0xf899b6,_0x283390){return _0xf899b6+_0x283390;},'lvXHo':_0x567750(0x158),'QlQok':function(_0x1e36a0,_0x49eb65){return _0x1e36a0(_0x49eb65);},'iLAhq':function(_0x5d6197,_0x133189){return _0x5d6197<=_0x133189;},'TmjoE':function(_0x59b0a9,_0x777117){return _0x59b0a9>_0x777117;},'RUUhM':function(_0x2c2f05,_0x7aa7a4){return _0x2c2f05/_0x7aa7a4;},'LDdbG':function(_0x5ad344,_0x3c981e){return _0x5ad344/_0x3c981e;}};if(!_0x1fbc86||!_0x1fbc86[_0x567750(0x15b)]())return{'text':'','originalTokens':0x0,'compressedTokens':0x0,'compressionRatio':0x1,'strategyUsed':_0x42a569['lvXHo'],'qualityEstimate':0x1,'wasTruncated':!0x1};const _0x54e93c=_0x42a569[_0x567750(0x15c)](countTokens,_0x1fbc86),_0x2716d8=_0x49cf7c[_0x567750(0x15a)]??0xc8,_0x4595f4=_0x49cf7c['overlap']??0x1e,_0x5794b4=_0x49cf7c[_0x567750(0x150)]??0x3;if(_0x42a569[_0x567750(0x14f)](_0x54e93c,_0x2716d8))return{'text':_0x1fbc86,'originalTokens':_0x54e93c,'compressedTokens':_0x54e93c,'compressionRatio':0x1,'strategyUsed':_0x42a569['lvXHo'],'qualityEstimate':0x1,'wasTruncated':!0x1};const _0x42bf4c=splitIntoOverlappingChunks(_0x42a569[_0x567750(0x15c)](encode,_0x1fbc86),_0x2716d8,_0x4595f4),_0x1d0dbd=_0x42bf4c['map']((_0xb42696,_0x5c0f7f)=>{const _0x54eaf5=_0x567750,_0x41122a=summarizeChunk(_0xb42696,_0x5794b4);return'['+_0x42a569[_0x54eaf5(0x14e)](_0x5c0f7f,0x1)+'/'+_0x42bf4c[_0x54eaf5(0x15e)]+']\x20'+_0x41122a;})['join']('\x0a\x0a'),_0x275d05=countTokens(_0x1d0dbd);return{'text':_0x1d0dbd,'originalTokens':_0x54e93c,'compressedTokens':_0x275d05,'compressionRatio':_0x42a569[_0x567750(0x156)](_0x275d05,0x0)?_0x42a569['RUUhM'](_0x54e93c,_0x275d05):0x1,'strategyUsed':_0x567750(0x158),'qualityEstimate':Math['min'](0x1,_0x42a569['TNPnr'](0.88,_0x42a569['LDdbG'](_0x275d05,_0x54e93c)*0.12)),'wasTruncated':!0x0};}export const chunkedStrategy={'name':'chunked','compress':(_0x3c6a2f,_0x332e1c)=>compressChunked(_0x3c6a2f,_0x332e1c)};function _0x5082(){const _0x410d36=['71421WhhuZo','TmjoE','LvRCP','chunked','37213FmQcNL','chunkSize','trim','QlQok','tywIG','length','8dlJxDn','7263441OhKmJN','min','4850hzHLrD','33VtkCiQ','join','468vBouTo','DDQfT','dOFvc','174952kppiCd','57398OXzAnX','TNPnr','iLAhq','sentencesPerChunk','push','5928465JOKIcZ','906323Ktxkzx','slice'];_0x5082=function(){return _0x410d36;};return _0x5082();}
1
+ (function(_0x59d4ad,_0x3e5ce9){const _0x761dd4=_0x1ffc,_0x436ee0=_0x59d4ad();while(!![]){try{const _0x21423c=parseInt(_0x761dd4(0x1cc))/0x1+parseInt(_0x761dd4(0x1da))/0x2+parseInt(_0x761dd4(0x1d2))/0x3*(parseInt(_0x761dd4(0x1c9))/0x4)+-parseInt(_0x761dd4(0x1dd))/0x5*(-parseInt(_0x761dd4(0x1d4))/0x6)+-parseInt(_0x761dd4(0x1d3))/0x7*(-parseInt(_0x761dd4(0x1c8))/0x8)+-parseInt(_0x761dd4(0x1d9))/0x9*(-parseInt(_0x761dd4(0x1db))/0xa)+-parseInt(_0x761dd4(0x1c7))/0xb;if(_0x21423c===_0x3e5ce9)break;else _0x436ee0['push'](_0x436ee0['shift']());}catch(_0x17ffce){_0x436ee0['push'](_0x436ee0['shift']());}}}(_0x4ec8,0xf130e));import{encode,decode}from'gpt-tokenizer';import{countTokens}from'../../shared/token-counter.js';import{splitSentences}from'./text-utils.js';const DEFAULT_CHUNK_SIZE=0xc8,DEFAULT_OVERLAP=0x1e,DEFAULT_SENTENCES_PER_CHUNK=0x3,BASE_QUALITY_ESTIMATE=0.88,QUALITY_VARIANCE_WEIGHT=0.12;function splitIntoOverlappingChunks(_0x3f4850,_0x2505b4,_0x6ce262){const _0x207bf6=_0x1ffc,_0x13e4ab={'hLhYr':function(_0x7c0f75,_0x458cfb){return _0x7c0f75-_0x458cfb;},'thtJI':function(_0x1a4119,_0x2478e1){return _0x1a4119<_0x2478e1;},'wWIDI':function(_0x52a80a,_0x5b9854){return _0x52a80a===_0x5b9854;}},_0x566a2e=Math[_0x207bf6(0x1cb)](_0x6ce262,_0x2505b4-0x1),_0x3f966e=Math[_0x207bf6(0x1d8)](0x1,_0x13e4ab[_0x207bf6(0x1d0)](_0x2505b4,_0x566a2e)),_0x2f416a=[];let _0x18ebf4=0x0;for(;_0x13e4ab['thtJI'](_0x18ebf4,_0x3f4850['length']);){const _0x2a5052=Math[_0x207bf6(0x1cb)](_0x18ebf4+_0x2505b4,_0x3f4850['length']),_0x688aca=_0x3f4850[_0x207bf6(0x1c5)](_0x18ebf4,_0x2a5052);if(_0x2f416a['push'](decode(_0x688aca)),_0x13e4ab[_0x207bf6(0x1de)](_0x2a5052,_0x3f4850[_0x207bf6(0x1d5)]))break;_0x18ebf4+=_0x3f966e;}return _0x2f416a;}function _0x1ffc(_0x3a2e8e,_0x396600){_0x3a2e8e=_0x3a2e8e-0x1c5;const _0x4ec8fc=_0x4ec8();let _0x1ffcd8=_0x4ec8fc[_0x3a2e8e];return _0x1ffcd8;}function summarizeChunk(_0x34f7a2,_0x1ddfc){const _0x316d5e=_0x1ffc,_0x553e0d={'olbTw':function(_0x52e9e2,_0x38e2e5){return _0x52e9e2<=_0x38e2e5;},'ohuqu':function(_0x5ef1c2,_0x39b23f){return _0x5ef1c2-_0x39b23f;}},_0x533c64=splitSentences(_0x34f7a2);if(_0x553e0d[_0x316d5e(0x1ce)](_0x533c64['length'],_0x1ddfc))return _0x34f7a2;if(0x1===_0x1ddfc)return _0x533c64[0x0];return[..._0x533c64['slice'](0x0,0x1),..._0x533c64['slice'](-_0x553e0d[_0x316d5e(0x1cd)](_0x1ddfc,0x1))]['join']('.\x20');}export function compressChunked(_0x3eeefa,_0x580db4={}){const _0x20f595=_0x1ffc,_0x500343={'abVlK':function(_0x1d8e2c,_0x28a4ab,_0x4e606a){return _0x1d8e2c(_0x28a4ab,_0x4e606a);},'pCJvR':'chunked','IUnlE':function(_0x3267d0,_0x1a5721){return _0x3267d0(_0x1a5721);},'JriLY':function(_0x304239,_0xb35aa1,_0x157f81,_0x141555){return _0x304239(_0xb35aa1,_0x157f81,_0x141555);},'gcZfj':function(_0x2961e1,_0x50c683){return _0x2961e1>_0x50c683;},'HWBrE':function(_0x2dfff5,_0x56dfa6){return _0x2dfff5+_0x56dfa6;},'lWvfj':function(_0x1722c7,_0x15be11){return _0x1722c7*_0x15be11;},'weQLW':function(_0x17f025,_0x307ffd){return _0x17f025/_0x307ffd;}};if(!_0x3eeefa||!_0x3eeefa['trim']())return{'text':'','originalTokens':0x0,'compressedTokens':0x0,'compressionRatio':0x1,'strategyUsed':_0x500343['pCJvR'],'qualityEstimate':0x1,'wasTruncated':!0x1};const _0x436c10=_0x500343['IUnlE'](countTokens,_0x3eeefa),_0x445b75=_0x580db4[_0x20f595(0x1dc)]??0xc8,_0x502e16=_0x580db4['overlap']??0x1e,_0x55bc41=_0x580db4['sentencesPerChunk']??0x3;if(_0x436c10<=_0x445b75)return{'text':_0x3eeefa,'originalTokens':_0x436c10,'compressedTokens':_0x436c10,'compressionRatio':0x1,'strategyUsed':_0x20f595(0x1d1),'qualityEstimate':0x1,'wasTruncated':!0x1};const _0x351537=_0x500343[_0x20f595(0x1cf)](splitIntoOverlappingChunks,encode(_0x3eeefa),_0x445b75,_0x502e16),_0x59717c=_0x351537['map']((_0x4c5846,_0x20c87b)=>{const _0x585ff7=_0x20f595,_0x2ffd43=_0x500343[_0x585ff7(0x1d7)](summarizeChunk,_0x4c5846,_0x55bc41);return'['+(_0x20c87b+0x1)+'/'+_0x351537[_0x585ff7(0x1d5)]+']\x20'+_0x2ffd43;})[_0x20f595(0x1d6)]('\x0a\x0a'),_0x549bc7=countTokens(_0x59717c);return{'text':_0x59717c,'originalTokens':_0x436c10,'compressedTokens':_0x549bc7,'compressionRatio':_0x500343['gcZfj'](_0x549bc7,0x0)?_0x436c10/_0x549bc7:0x1,'strategyUsed':_0x500343['pCJvR'],'qualityEstimate':Math['min'](0x1,_0x500343[_0x20f595(0x1ca)](0.88,_0x500343['lWvfj'](_0x500343[_0x20f595(0x1c6)](_0x549bc7,_0x436c10),0.12))),'wasTruncated':!0x0};}export const chunkedStrategy={'name':'chunked','compress':(_0x49e94c,_0x3de92e)=>compressChunked(_0x49e94c,_0x3de92e)};function _0x4ec8(){const _0x77f3fb=['JriLY','hLhYr','chunked','433263nZZmXU','289793YEwBVL','12jpCqan','length','join','abVlK','max','9uhhunB','2297884lubwqU','15835270LAVDTb','chunkSize','1691930JiowGw','wWIDI','slice','weQLW','60605380GpKXTo','72WdAGrp','52fBgfCE','HWBrE','min','838193DdyYsr','ohuqu','olbTw'];_0x4ec8=function(){return _0x77f3fb;};return _0x4ec8();}
package/dist/plugin/compression/dedup.js CHANGED
@@ -1 +1 @@
1
- const _0x1b0f5b=_0x168c;(function(_0x4422ef,_0x4b4a88){const _0x3fd5db=_0x168c,_0x949efa=_0x4422ef();while(!![]){try{const _0x2f620a=parseInt(_0x3fd5db(0x15d))/0x1*(-parseInt(_0x3fd5db(0x165))/0x2)+parseInt(_0x3fd5db(0x16d))/0x3*(parseInt(_0x3fd5db(0x164))/0x4)+-parseInt(_0x3fd5db(0x158))/0x5+-parseInt(_0x3fd5db(0x171))/0x6*(parseInt(_0x3fd5db(0x15e))/0x7)+-parseInt(_0x3fd5db(0x160))/0x8*(parseInt(_0x3fd5db(0x163))/0x9)+-parseInt(_0x3fd5db(0x15b))/0xa*(parseInt(_0x3fd5db(0x156))/0xb)+parseInt(_0x3fd5db(0x16a))/0xc*(parseInt(_0x3fd5db(0x181))/0xd);if(_0x2f620a===_0x4b4a88)break;else _0x949efa['push'](_0x949efa['shift']());}catch(_0x493c5b){_0x949efa['push'](_0x949efa['shift']());}}}(_0xe36d,0xe8834));import{countTokens}from'../../shared/token-counter.js';import{extractContentWords,STOPWORDS}from'./text-utils.js';function _0xe36d(){const _0x207c8b=['bRZdf','map','EIxky','VuPdm','split','9359467Ylrhxc','test','isProtected','110bScSKt','maxTokens','1216810XSFyzW','KBJav','HFpDE','20040ZFphff','size','898553sSKfEY','28EXcSHc','trim','195016CfZjep','uByDd','VgQiN','126IRfRXE','1164972ouDSWA','2ScDoWs','text','IsnWX','XYKAG','MtezO','36fLaLRk','length','filter','12KFhRQT','has','fingerprint','wTarZ','1303866XRPcOU','sort','index','FspKp','dedup','OVOii','add','lYTpH','thWfz','oqNJy','FOBtK'];_0xe36d=function(){return _0x207c8b;};return _0xe36d();}const DEFAULT_SIMILARITY_THRESHOLD=0.7,MIN_WORDS_FOR_DEDUP=0x5,CODE_FENCE_PATTERN=/^```/,HEADING_PATTERN=/^#/,BASE_QUALITY_ESTIMATE=0.95;function isInsideCodeFence(_0x420839,_0x24a22b){const _0x515f41=_0x168c,_0x1b88b0={'uByDd':function(_0x2238e9,_0x460f5e){return _0x2238e9<_0x460f5e;}};let _0x4a8fb1=!0x1;for(let _0x4e06cb=0x0;_0x1b88b0[_0x515f41(0x161)](_0x4e06cb,_0x24a22b);_0x4e06cb++){const _0x4990ab=_0x420839[_0x4e06cb]['split']('\x0a');for(const _0x263793 of _0x4990ab)CODE_FENCE_PATTERN[_0x515f41(0x182)](_0x263793[_0x515f41(0x15f)]())&&(_0x4a8fb1=!_0x4a8fb1);}return _0x4a8fb1;}function hasCodeFence(_0x2c03c5){const _0x58c33f=_0x168c;return _0x2c03c5['split']('\x0a')['some'](_0x1a903e=>CODE_FENCE_PATTERN[_0x58c33f(0x182)](_0x1a903e['trim']()));}function isHeading(_0x1f2804){const _0x5486a1=_0x168c;return HEADING_PATTERN['test'](_0x1f2804[_0x5486a1(0x15f)]());}function isTooShort(_0x42dd71){const _0x141f4a=_0x168c,_0x9ed01c={'FOBtK':function(_0x1b042e,_0x4afc7c){return _0x1b042e<_0x4afc7c;}};return _0x9ed01c[_0x141f4a(0x17b)](_0x42dd71['trim']()['split'](/\s+/)[_0x141f4a(0x16b)],0x5);}function computeFingerprint(_0x37d0e4){const _0x50c6fc=_0x168c,_0x1472fe=extractContentWords(_0x37d0e4)['filter'](_0x44c3b4=>!STOPWORDS['has'](_0x44c3b4));return new Set(_0x1472fe[_0x50c6fc(0x172)]());}function computeJaccardSimilarity(_0x4236cc,_0x42ff75){const _0x9955d=_0x168c,_0xf34f27={'lYTpH':function(_0x12983e,_0x52a920){return _0x12983e===_0x52a920;},'cuCiA':function(_0x37d3bc,_0x3e0732){return _0x37d3bc<=_0x3e0732;},'oBFTY':function(_0x3def5b,_0x2d1f9d){return _0x3def5b-_0x2d1f9d;},'HFpDE':function(_0x35e42b,_0x10adff){return _0x35e42b+_0x10adff;},'bRZdf':function(_0x215dc4,_0x212da8){return _0x215dc4>_0x212da8;},'KBJav':function(_0x26e2a3,_0x33551b){return _0x26e2a3/_0x33551b;}};if(_0xf34f27[_0x9955d(0x178)](0x0,_0x4236cc['size'])&&0x0===_0x42ff75[_0x9955d(0x15c)])return 0x0;let _0x165220=0x0;const _0x32d207=_0xf34f27['cuCiA'](_0x4236cc['size'],_0x42ff75[_0x9955d(0x15c)])?_0x4236cc:_0x42ff75,_0x546048=_0x4236cc[_0x9955d(0x15c)]<=_0x42ff75['size']?_0x42ff75:_0x4236cc;for(const _0x44fa2e of _0x32d207)_0x546048[_0x9955d(0x16e)](_0x44fa2e)&&_0x165220++;const _0x2c2920=_0xf34f27['oBFTY'](_0xf34f27[_0x9955d(0x15a)](_0x4236cc['size'],_0x42ff75[_0x9955d(0x15c)]),_0x165220);return _0xf34f27[_0x9955d(0x17c)](_0x2c2920,0x0)?_0xf34f27[_0x9955d(0x159)](_0x165220,_0x2c2920):0x0;}function splitParagraphs(_0x215436){const _0x238fd6=_0x168c;return _0x215436[_0x238fd6(0x180)](/\n\n+/);}function buildParagraphEntries(_0x1fa82c){const _0x22ff77=_0x168c,_0x57db47={'oqNJy':function(_0x2a362c,_0x53d269){return _0x2a362c(_0x53d269);},'fRJlf':function(_0x519b36,_0x54ec7c){return _0x519b36(_0x54ec7c);},'IsnWX':function(_0x1fbdd2,_0x23540c,_0x209c63){return _0x1fbdd2(_0x23540c,_0x209c63);}};return _0x1fa82c[_0x22ff77(0x17d)]((_0x25551e,_0x412e57)=>{const _0x14e105=_0x22ff77,_0x2e3aa2=isTooShort(_0x25551e)||_0x57db47[_0x14e105(0x17a)](isHeading,_0x25551e)||_0x57db47['fRJlf'](hasCodeFence,_0x25551e)||_0x57db47[_0x14e105(0x167)](isInsideCodeFence,_0x1fa82c,_0x412e57);return{'index':_0x412e57,'text':_0x25551e,'fingerprint':_0x2e3aa2?new Set():computeFingerprint(_0x25551e),'isProtected':_0x2e3aa2};});}function findDuplicates(_0x108b78,_0xf4f015){const _0x166b93=_0x168c,_0x1609f8={'thWfz':function(_0x33c563,_0x1ab9e6){return _0x33c563<_0x1ab9e6;},'VuPdm':function(_0x827e34,_0x52df76){return _0x827e34>=_0x52df76;},'njDQa':function(_0x23d8ba,_0x308f7d,_0x19b37c){return _0x23d8ba(_0x308f7d,_0x19b37c);}},_0x166601=new Set();let _0x44a550=0x0;for(let _0x479b7e=0x0;_0x1609f8[_0x166b93(0x179)](_0x479b7e,_0x108b78[_0x166b93(0x16b)]);_0x479b7e++){const _0x465a3a=_0x108b78[_0x479b7e];if(_0x465a3a[_0x166b93(0x155)]||_0x166601['has'](_0x479b7e))continue;let _0x245b3f=!0x1;for(let _0x1ec09=_0x479b7e+0x1;_0x1609f8[_0x166b93(0x179)](_0x1ec09,_0x108b78['length']);_0x1ec09++){const _0x59fb50=_0x108b78[_0x1ec09];if(_0x59fb50[_0x166b93(0x155)]||_0x166601[_0x166b93(0x16e)](_0x1ec09))continue;_0x1609f8[_0x166b93(0x17f)](_0x1609f8['njDQa'](computeJaccardSimilarity,_0x465a3a['fingerprint'],_0x59fb50[_0x166b93(0x16f)]),_0xf4f015)&&(_0x166601[_0x166b93(0x177)](_0x1ec09),_0x245b3f=!0x0);}_0x245b3f&&_0x44a550++;}const _0x449567=new Set();for(let _0x5c6523=0x0;_0x1609f8[_0x166b93(0x179)](_0x5c6523,_0x108b78[_0x166b93(0x16b)]);_0x5c6523++)_0x166601['has'](_0x5c6523)||_0x449567['add'](_0x5c6523);return{'keptIndices':_0x449567,'removedIndices':_0x166601,'groupCount':_0x44a550};}function _0x168c(_0x268531,_0x1be3bf){_0x268531=_0x268531-0x155;const _0xe36dfb=_0xe36d();let _0x168c0c=_0xe36dfb[_0x268531];return _0x168c0c;}export function deduplicateText(_0x557d3d,_0x437c81){const _0x2e6a86=_0x168c,_0x132691={'MtezO':function(_0x4affa6,_0x3a8f06){return _0x4affa6(_0x3a8f06);},'WcHHA':function(_0x3232a1,_0x207334,_0x2cee15){return _0x3232a1(_0x207334,_0x2cee15);}};if(!_0x557d3d||!_0x557d3d[_0x2e6a86(0x15f)]())return{'text':'','originalParagraphs':0x0,'keptParagraphs':0x0,'removedParagraphs':0x0,'duplicateGroups':0x0};const _0x17f718=_0x437c81?.['similarityThreshold']??0.7,_0x347536=splitParagraphs(_0x557d3d),_0x3acccc=_0x132691[_0x2e6a86(0x169)](buildParagraphEntries,_0x347536),{keptIndices:_0x4cb585,removedIndices:_0x1ff09a,groupCount:_0x3b7ff3}=_0x132691['WcHHA'](findDuplicates,_0x3acccc,_0x17f718),_0x44d83b=_0x3acccc[_0x2e6a86(0x16c)](_0x192d0b=>_0x4cb585['has'](_0x192d0b[_0x2e6a86(0x173)]))[_0x2e6a86(0x17d)](_0xcde292=>_0xcde292['text']);return{'text':_0x44d83b['join']('\x0a\x0a'),'originalParagraphs':_0x347536[_0x2e6a86(0x16b)],'keptParagraphs':_0x44d83b[_0x2e6a86(0x16b)],'removedParagraphs':_0x1ff09a[_0x2e6a86(0x15c)],'duplicateGroups':_0x3b7ff3};}export function compressDedup(_0x5c3585,_0x8e0bb6={}){const _0x450db6=_0x168c,_0x1f818d={'EIxky':_0x450db6(0x175),'XYKAG':function(_0x2e850f,_0x2e936b){return _0x2e850f(_0x2e936b);},'VgQiN':function(_0x5e7567,_0x5b2d72){return _0x5e7567(_0x5b2d72);},'PYhIv':function(_0x20b285,_0x2e2695){return _0x20b285(_0x2e2695);},'FspKp':function(_0x100e74,_0x1da102){return _0x100e74>_0x1da102;},'OVOii':function(_0x337a61,_0x2b251b){return _0x337a61(_0x2b251b);},'wTarZ':function(_0xecab3e,_0xb1d1e9){return _0xecab3e<=_0xb1d1e9;},'hnQaV':function(_0x26658c,_0x27b604){return _0x26658c+_0x27b604;},'CAkrZ':function(_0x2a591e,_0x5f18cb){return _0x2a591e/_0x5f18cb;}};if(!_0x5c3585||!_0x5c3585[_0x450db6(0x15f)]())return{'text':'','originalTokens':0x0,'compressedTokens':0x0,'compressionRatio':0x1,'strategyUsed':_0x1f818d[_0x450db6(0x17e)],'qualityEstimate':0x1,'wasTruncated':!0x1};const _0x31a25b=_0x1f818d[_0x450db6(0x168)](countTokens,_0x5c3585),_0x21b5b7=_0x1f818d[_0x450db6(0x162)](deduplicateText,_0x5c3585),_0x27a5c8=_0x1f818d['PYhIv'](countTokens,_0x21b5b7[_0x450db6(0x166)]),_0x406011=_0x21b5b7['removedParagraphs']>0x0?Math['min'](0x1,0.95+_0x27a5c8/_0x31a25b*(0x1-0.95)):0x1;let _0x5c74c3=_0x21b5b7[_0x450db6(0x166)],_0x1656d1=!0x1;if(_0x8e0bb6[_0x450db6(0x157)]&&_0x1f818d[_0x450db6(0x174)](_0x27a5c8,_0x8e0bb6['maxTokens'])){const _0x45888c=splitParagraphs(_0x5c74c3),_0x7b3ff9=[];let _0x265165=0x0;for(const _0x3d9617 of _0x45888c){const _0x2d0c2a=_0x1f818d[_0x450db6(0x176)](countTokens,_0x3d9617);if(!_0x1f818d[_0x450db6(0x170)](_0x1f818d['hnQaV'](_0x265165,_0x2d0c2a),_0x8e0bb6[_0x450db6(0x157)])){_0x1656d1=!0x0;break;}_0x7b3ff9['push'](_0x3d9617),_0x265165+=_0x2d0c2a;}_0x5c74c3=_0x7b3ff9['join']('\x0a\x0a');}const _0x45e648=_0x1656d1?countTokens(_0x5c74c3):_0x27a5c8;return{'text':_0x5c74c3,'originalTokens':_0x31a25b,'compressedTokens':_0x45e648,'compressionRatio':_0x45e648>0x0?_0x1f818d['CAkrZ'](_0x31a25b,_0x45e648):0x1,'strategyUsed':_0x1f818d['EIxky'],'qualityEstimate':_0x406011,'wasTruncated':_0x1656d1};}export const dedupStrategy={'name':_0x1b0f5b(0x175),'compress':compressDedup};
1
+ const _0x4f16e7=_0x314c;(function(_0x4abf48,_0x13ef61){const _0x42c632=_0x314c,_0x4dc6cb=_0x4abf48();while(!![]){try{const _0x1e36ab=-parseInt(_0x42c632(0xd5))/0x1+parseInt(_0x42c632(0xca))/0x2*(-parseInt(_0x42c632(0xd2))/0x3)+-parseInt(_0x42c632(0xcd))/0x4+-parseInt(_0x42c632(0xcc))/0x5*(-parseInt(_0x42c632(0xd4))/0x6)+-parseInt(_0x42c632(0xbe))/0x7*(parseInt(_0x42c632(0xc0))/0x8)+-parseInt(_0x42c632(0xbf))/0x9+-parseInt(_0x42c632(0xbc))/0xa*(-parseInt(_0x42c632(0xcf))/0xb);if(_0x1e36ab===_0x13ef61)break;else _0x4dc6cb['push'](_0x4dc6cb['shift']());}catch(_0x14e5fb){_0x4dc6cb['push'](_0x4dc6cb['shift']());}}}(_0x46a1,0x31946));import{countTokens}from'../../shared/token-counter.js';import{extractContentWords,STOPWORDS}from'./text-utils.js';const DEFAULT_SIMILARITY_THRESHOLD=0.7,MIN_WORDS_FOR_DEDUP=0x5,CODE_FENCE_PATTERN=/^```/,HEADING_PATTERN=/^#/,BASE_QUALITY_ESTIMATE=0.95;function _0x314c(_0x1292a9,_0xd60322){_0x1292a9=_0x1292a9-0xba;const _0x46a170=_0x46a1();let _0x314c28=_0x46a170[_0x1292a9];return _0x314c28;}function isInsideCodeFence(_0x31da9a,_0x19fcf0){const _0x3d8f0d=_0x314c;let _0x1e0188=!0x1;for(let _0x4d7edf=0x0;_0x4d7edf<_0x19fcf0;_0x4d7edf++){const _0x41a232=_0x31da9a[_0x4d7edf]['split']('\x0a');for(const _0x682f01 of _0x41a232)CODE_FENCE_PATTERN['test'](_0x682f01[_0x3d8f0d(0xd1)]())&&(_0x1e0188=!_0x1e0188);}return _0x1e0188;}function hasCodeFence(_0x4e7cf3){const _0x5e980e=_0x314c;return _0x4e7cf3['split']('\x0a')['some'](_0x556472=>CODE_FENCE_PATTERN['test'](_0x556472[_0x5e980e(0xd1)]()));}function isHeading(_0x3e20bb){const _0x27d904=_0x314c;return HEADING_PATTERN['test'](_0x3e20bb[_0x27d904(0xd1)]());}function isTooShort(_0x6a8bec){const _0x49fcf1={'oGsRy':function(_0x2e8ff8,_0x478ce2){return _0x2e8ff8<_0x478ce2;}};return _0x49fcf1['oGsRy'](_0x6a8bec['trim']()['split'](/\s+/)['length'],0x5);}function computeFingerprint(_0x20e572){const _0x5e6613=_0x314c,_0x107e83=extractContentWords(_0x20e572)[_0x5e6613(0xd3)](_0x2019ae=>!STOPWORDS['has'](_0x2019ae));return new Set(_0x107e83['sort']());}function computeJaccardSimilarity(_0xaad06b,_0xcd4d1f){const _0x2cb14d=_0x314c,_0x29fd98={'RDtBW':function(_0x3cfbff,_0x3351d0){return _0x3cfbff===_0x3351d0;},'WjDwQ':function(_0x2c0edc,_0x24a6ca){return _0x2c0edc===_0x24a6ca;},'MhpAR':function(_0x4567ed,_0x428ac8){return _0x4567ed<=_0x428ac8;},'jVZhi':function(_0x18c932,_0x453c5e){return _0x18c932-_0x453c5e;},'uGhqX':function(_0x9597c1,_0x4f1ef8){return _0x9597c1+_0x4f1ef8;}};if(_0x29fd98['RDtBW'](0x0,_0xaad06b[_0x2cb14d(0xcb)])&&_0x29fd98['WjDwQ'](0x0,_0xcd4d1f[_0x2cb14d(0xcb)]))return 0x0;let _0xa1a151=0x0;const _0xaed88c=_0x29fd98[_0x2cb14d(0xc2)](_0xaad06b[_0x2cb14d(0xcb)],_0xcd4d1f[_0x2cb14d(0xcb)])?_0xaad06b:_0xcd4d1f,_0x511459=_0x29fd98['MhpAR'](_0xaad06b['size'],_0xcd4d1f[_0x2cb14d(0xcb)])?_0xcd4d1f:_0xaad06b;for(const _0xb3a17f of _0xaed88c)_0x511459['has'](_0xb3a17f)&&_0xa1a151++;const _0x3a2ea5=_0x29fd98['jVZhi'](_0x29fd98[_0x2cb14d(0xc8)](_0xaad06b[_0x2cb14d(0xcb)],_0xcd4d1f['size']),_0xa1a151);return _0x3a2ea5>0x0?_0xa1a151/_0x3a2ea5:0x0;}function _0x46a1(){const _0x52c395=['1720JLVaxD','similarityThreshold','1393BdpdyQ','3385026kKKCkw','3728uFANlh','maxTokens','MhpAR','length','min','qAHcE','fingerprint','dedup','uGhqX','MbDFO','35852nagJlK','size','95swtInd','1337640jswhOk','split','96217MNPevP','rHVBe','trim','60rQRxRq','filter','35886PvllGv','253267eefDNt','add','text'];_0x46a1=function(){return _0x52c395;};return _0x46a1();}function splitParagraphs(_0x212a99){const _0x3a28f8=_0x314c;return _0x212a99[_0x3a28f8(0xce)](/\n\n+/);}function buildParagraphEntries(_0x300814){const _0x2cc66a={'cZdEe':function(_0x14f44b,_0x13255e){return _0x14f44b(_0x13255e);}};return _0x300814['map']((_0x1c4ebf,_0x3a159f)=>{const _0x17d885=isTooShort(_0x1c4ebf)||isHeading(_0x1c4ebf)||hasCodeFence(_0x1c4ebf)||isInsideCodeFence(_0x300814,_0x3a159f);return{'index':_0x3a159f,'text':_0x1c4ebf,'fingerprint':_0x17d885?new Set():_0x2cc66a['cZdEe'](computeFingerprint,_0x1c4ebf),'isProtected':_0x17d885};});}function findDuplicates(_0x4d929d,_0x1e41fc){const _0xdb8031=_0x314c,_0x9dd6c0={'MbDFO':function(_0x301ac7,_0x47b39b){return _0x301ac7+_0x47b39b;},'qAHcE':function(_0x59b199,_0x1682ea){return _0x59b199<_0x1682ea;},'ogNgf':function(_0x51a5b6,_0x337097){return _0x51a5b6>=_0x337097;}},_0x2a2bf7=new Set();let _0x284745=0x0;for(let _0x34c192=0x0;_0x34c192<_0x4d929d[_0xdb8031(0xc3)];_0x34c192++){const _0x3b5478=_0x4d929d[_0x34c192];if(_0x3b5478['isProtected']||_0x2a2bf7['has'](_0x34c192))continue;let _0x267ff3=!0x1;for(let _0x4785c6=_0x9dd6c0[_0xdb8031(0xc9)](_0x34c192,0x1);_0x9dd6c0[_0xdb8031(0xc5)](_0x4785c6,_0x4d929d['length']);_0x4785c6++){const _0x7f3047=_0x4d929d[_0x4785c6];if(_0x7f3047['isProtected']||_0x2a2bf7['has'](_0x4785c6))continue;_0x9dd6c0['ogNgf'](computeJaccardSimilarity(_0x3b5478[_0xdb8031(0xc6)],_0x7f3047[_0xdb8031(0xc6)]),_0x1e41fc)&&(_0x2a2bf7[_0xdb8031(0xba)](_0x4785c6),_0x267ff3=!0x0);}_0x267ff3&&_0x284745++;}const _0x50e715=new Set();for(let _0x4ea424=0x0;_0x4ea424<_0x4d929d[_0xdb8031(0xc3)];_0x4ea424++)_0x2a2bf7['has'](_0x4ea424)||_0x50e715['add'](_0x4ea424);return{'keptIndices':_0x50e715,'removedIndices':_0x2a2bf7,'groupCount':_0x284745};}export function deduplicateText(_0x2d9f14,_0x5e39c9){const _0x5a83fe=_0x314c;if(!_0x2d9f14||!_0x2d9f14[_0x5a83fe(0xd1)]())return{'text':'','originalParagraphs':0x0,'keptParagraphs':0x0,'removedParagraphs':0x0,'duplicateGroups':0x0};const _0x102438=_0x5e39c9?.[_0x5a83fe(0xbd)]??0.7,_0x257563=splitParagraphs(_0x2d9f14),_0x4e3a92=buildParagraphEntries(_0x257563),{keptIndices:_0x3feb22,removedIndices:_0x32447d,groupCount:_0x37f5ff}=findDuplicates(_0x4e3a92,_0x102438),_0x3a39a2=_0x4e3a92[_0x5a83fe(0xd3)](_0xa11682=>_0x3feb22['has'](_0xa11682['index']))['map'](_0x1abf73=>_0x1abf73[_0x5a83fe(0xbb)]);return{'text':_0x3a39a2['join']('\x0a\x0a'),'originalParagraphs':_0x257563[_0x5a83fe(0xc3)],'keptParagraphs':_0x3a39a2['length'],'removedParagraphs':_0x32447d['size'],'duplicateGroups':_0x37f5ff};}export function compressDedup(_0x21261a,_0x3c63ef={}){const _0x399ac8=_0x314c,_0x2b9091={'Krgyn':_0x399ac8(0xc7),'ZoVrb':function(_0x5f2c93,_0x415d16){return _0x5f2c93(_0x415d16);},'rHVBe':function(_0x312e45,_0x440d53){return _0x312e45(_0x440d53);},'QMzgB':function(_0x4478a3,_0x56eca0){return _0x4478a3/_0x56eca0;},'xWOTS':function(_0x1e2652,_0x139ea1){return _0x1e2652>_0x139ea1;},'wgExI':function(_0x3a382f,_0x30513e){return _0x3a382f<=_0x30513e;}};if(!_0x21261a||!_0x21261a['trim']())return{'text':'','originalTokens':0x0,'compressedTokens':0x0,'compressionRatio':0x1,'strategyUsed':_0x2b9091['Krgyn'],'qualityEstimate':0x1,'wasTruncated':!0x1};const _0x563c3e=_0x2b9091['ZoVrb'](countTokens,_0x21261a),_0xbeab31=_0x2b9091[_0x399ac8(0xd0)](deduplicateText,_0x21261a),_0x447111=countTokens(_0xbeab31[_0x399ac8(0xbb)]),_0x4a5e9c=_0xbeab31['removedParagraphs']>0x0?Math[_0x399ac8(0xc4)](0x1,0.95+_0x2b9091['QMzgB'](_0x447111,_0x563c3e)*(0x1-0.95)):0x1;let _0x878f3e=_0xbeab31[_0x399ac8(0xbb)],_0x509764=!0x1;if(_0x3c63ef[_0x399ac8(0xc1)]&&_0x2b9091['xWOTS'](_0x447111,_0x3c63ef['maxTokens'])){const _0x421ac9=splitParagraphs(_0x878f3e),_0x265a7d=[];let _0x16bf10=0x0;for(const _0x5a33ec of _0x421ac9){const _0x519a45=countTokens(_0x5a33ec);if(!_0x2b9091['wgExI'](_0x16bf10+_0x519a45,_0x3c63ef['maxTokens'])){_0x509764=!0x0;break;}_0x265a7d['push'](_0x5a33ec),_0x16bf10+=_0x519a45;}_0x878f3e=_0x265a7d['join']('\x0a\x0a');}const _0x5b1737=_0x509764?_0x2b9091['rHVBe'](countTokens,_0x878f3e):_0x447111;return{'text':_0x878f3e,'originalTokens':_0x563c3e,'compressedTokens':_0x5b1737,'compressionRatio':_0x5b1737>0x0?_0x563c3e/_0x5b1737:0x1,'strategyUsed':_0x2b9091['Krgyn'],'qualityEstimate':_0x4a5e9c,'wasTruncated':_0x509764};}export const dedupStrategy={'name':_0x4f16e7(0xc7),'compress':compressDedup};
package/dist/plugin/compression/format-optimizer.js CHANGED
@@ -1 +1 @@
1
- (function(_0x17004a,_0x31a11a){const _0x3230b9=_0x27c0,_0x15a9ee=_0x17004a();while(!![]){try{const _0x7921c1=parseInt(_0x3230b9(0xd5))/0x1*(-parseInt(_0x3230b9(0xcd))/0x2)+-parseInt(_0x3230b9(0xe6))/0x3*(parseInt(_0x3230b9(0xec))/0x4)+parseInt(_0x3230b9(0xd6))/0x5*(parseInt(_0x3230b9(0xe5))/0x6)+-parseInt(_0x3230b9(0xca))/0x7*(parseInt(_0x3230b9(0xd7))/0x8)+-parseInt(_0x3230b9(0xea))/0x9+-parseInt(_0x3230b9(0xe7))/0xa+parseInt(_0x3230b9(0xdc))/0xb;if(_0x7921c1===_0x31a11a)break;else _0x15a9ee['push'](_0x15a9ee['shift']());}catch(_0x26f45f){_0x15a9ee['push'](_0x15a9ee['shift']());}}}(_0x1220,0xd7fcf));import{countTokens}from'../../shared/token-counter.js';const FENCED_JSON_BLOCK=/```(?:json)?\s*\n([\s\S]*?)```/g,INLINE_JSON_OBJECT=/(?<![\\:a-zA-Z0-9_"'])(\{[\s\S]*?\})(?![\\a-zA-Z0-9_])/g,INLINE_JSON_ARRAY=/(?<![\\:a-zA-Z0-9_"'])(\[[\s\S]*?\])(?![\\a-zA-Z0-9_])/g,CONSECUTIVE_BLANK_LINES=/\n{3,}/g,EXCESSIVE_LIST_INDENT=/^( {4,}|\t{2,})([*\-+]|\d+\.)\s/gm,BOLD_WHOLE_PARAGRAPH=/^\*\*([^*\n]+)\*\*$/gm,ITALIC_WHOLE_PARAGRAPH=/^_([^_\n]+)_$/gm,MAX_FLATTEN_DEPTH=0x2;function isJsonLike(_0x468098){const _0x48b95c=_0x27c0,_0x3cb83a=_0x468098['trim']();return _0x3cb83a[_0x48b95c(0xc5)]('{')&&_0x3cb83a['endsWith']('}')||_0x3cb83a[_0x48b95c(0xc5)]('[')&&_0x3cb83a[_0x48b95c(0xc6)](']');}function tryParseJson(_0x47b79d){try{return JSON['parse'](_0x47b79d);}catch{return null;}}function flattenObject(_0x49d860,_0x521c29,_0x5af7ea){const _0x1f3e7b=_0x27c0,_0x7d4888={'dtnkC':function(_0x25ae4e,_0x441fe7){return _0x25ae4e==_0x441fe7;},'PrqPF':function(_0x23741b,_0x37d69f){return _0x23741b+_0x37d69f;}},_0x781464=[];for(const [_0x31b490,_0x25b445]of Object['entries'](_0x49d860)){const _0x16493b=_0x521c29?_0x521c29+'.'+_0x31b490:_0x31b490;null!==_0x25b445&&_0x7d4888[_0x1f3e7b(0xd9)]('object',typeof _0x25b445)&&!Array['isArray'](_0x25b445)&&_0x5af7ea<0x2?_0x781464[_0x1f3e7b(0xce)](...flattenObject(_0x25b445,_0x16493b,_0x7d4888[_0x1f3e7b(0xe3)](_0x5af7ea,0x1))):_0x781464['push']([_0x16493b,_0x25b445]);}return _0x781464;}function formatValue(_0x2ee06c){const _0x3adc6c=_0x27c0,_0x591ab1={'FDhHx':function(_0x1895cb,_0x2bec77){return _0x1895cb===_0x2bec77;},'WjfRm':_0x3adc6c(0xe0),'bWXXz':function(_0x26a6ed,_0x5d72d7){return _0x26a6ed===_0x5d72d7;},'KBapf':function(_0x461888,_0x3057cc){return _0x461888==_0x3057cc;}};if(_0x591ab1['FDhHx'](null,_0x2ee06c))return _0x591ab1['WjfRm'];if(_0x591ab1[_0x3adc6c(0xdb)](void 0x0,_0x2ee06c))return'undefined';if(_0x3adc6c(0xcb)==typeof _0x2ee06c)return _0x2ee06c;if(_0x591ab1['KBapf'](_0x3adc6c(0xc4),typeof _0x2ee06c)||_0x3adc6c(0xcc)==typeof _0x2ee06c)return String(_0x2ee06c);if(Array[_0x3adc6c(0xe9)](_0x2ee06c))return'['+_0x2ee06c[_0x3adc6c(0xe1)](_0x13958f=>'object'==typeof _0x13958f&&null!==_0x13958f?JSON['stringify'](_0x13958f):formatValue(_0x13958f))[_0x3adc6c(0xd3)](',\x20')+']';return JSON[_0x3adc6c(0xdd)](_0x2ee06c);}function hasDeepNesting(_0x21dff8,_0x180212){const _0x8e86e2=_0x27c0,_0x43263f={'opzeX':'object'};return _0x180212>0x2||_0x43263f[_0x8e86e2(0xc9)]==typeof _0x21dff8&&null!==_0x21dff8&&(Array[_0x8e86e2(0xe9)](_0x21dff8)?_0x21dff8['some'](_0x2389d9=>hasDeepNesting(_0x2389d9,_0x180212+0x1)):Object[_0x8e86e2(0xe2)](_0x21dff8)[_0x8e86e2(0xe8)](_0x392f4f=>hasDeepNesting(_0x392f4f,_0x180212+0x1)));}function convertJsonToCompact(_0x22a7f3){const _0x4fee2f=_0x27c0,_0x48af26={'oKQYy':function(_0x5dfd66,_0x2d9dda,_0x1002ca){return _0x5dfd66(_0x2d9dda,_0x1002ca);},'dPMTH':function(_0x39e679,_0x5966e2){return _0x39e679==_0x5966e2;},'AcQwb':'object','WwnUy':function(_0x4c9dbc,_0x4b867f){return _0x4c9dbc!==_0x4b867f;},'Llrpi':function(_0x117b9a,_0x28b812,_0x4a4f3c){return _0x117b9a(_0x28b812,_0x4a4f3c);},'qHInI':function(_0x46caa7,_0x18bd84){return _0x46caa7(_0x18bd84);}};if(Array[_0x4fee2f(0xe9)](_0x22a7f3))return _0x22a7f3['every'](_0x1705bf=>'object'!=typeof _0x1705bf||null===_0x1705bf)?'['+_0x22a7f3[_0x4fee2f(0xe1)](formatValue)['join'](',\x20')+']':_0x48af26['oKQYy'](hasDeepNesting,_0x22a7f3,0x0)?JSON[_0x4fee2f(0xdd)](_0x22a7f3):_0x22a7f3['map'](_0x5c4cc2=>_0x4fee2f(0xed)!=typeof _0x5c4cc2||null===_0x5c4cc2||Array['isArray'](_0x5c4cc2)?formatValue(_0x5c4cc2):convertObjectToCompact(_0x5c4cc2))['join']('\x0a');if(_0x48af26['dPMTH'](_0x48af26['AcQwb'],typeof _0x22a7f3)&&_0x48af26[_0x4fee2f(0xe4)](null,_0x22a7f3)){if(_0x48af26[_0x4fee2f(0xda)](hasDeepNesting,_0x22a7f3,0x0)){const _0x1b2cac=_0x22a7f3;if(!Object[_0x4fee2f(0xe2)](_0x1b2cac)[_0x4fee2f(0xe8)](_0x5c70fe=>'object'!=typeof _0x5c70fe||null===_0x5c70fe))return JSON[_0x4fee2f(0xdd)](_0x22a7f3);}return convertObjectToCompact(_0x22a7f3);}return _0x48af26['qHInI'](formatValue,_0x22a7f3);}function convertObjectToCompact(_0x2af2cf){const _0x304011=_0x27c0,_0x288bc0={'qqAMd':function(_0x52247b,_0x51f546,_0x329835,_0x52ae17){return _0x52247b(_0x51f546,_0x329835,_0x52ae17);}},_0x4f00d6=_0x288bc0['qqAMd'](flattenObject,_0x2af2cf,'',0x0)[_0x304011(0xe1)](([_0x21dc36,_0x30aff5])=>'object'==typeof _0x30aff5&&null!==_0x30aff5?_0x21dc36+':\x20'+JSON['stringify'](_0x30aff5):_0x21dc36+':\x20'+formatValue(_0x30aff5));return _0x4f00d6[_0x304011(0xef)]<=0x4&&_0x4f00d6['every'](_0x56e603=>_0x56e603['length']<0x3c)?_0x4f00d6[_0x304011(0xd3)](',\x20'):_0x4f00d6[_0x304011(0xd3)]('\x0a');}function _0x1220(){const _0x3930a9=['string','boolean','3105772RwhgrL','push','text','cleaned','xNPYj','biyvh','join','uQOma','1phRpYu','6585YXozIn','8gLYiHU','pBtMh','dtnkC','Llrpi','bWXXz','52716004QpPDgU','stringify','tnaxW','replace','null','map','values','PrqPF','WwnUy','5262nYiLBW','5254329DdUVLO','1944100SjOFUq','some','isArray','5757948OenPnt','trim','4hzDgRs','object','markdownCleaned','length','number','startsWith','endsWith','count','cydlo','opzeX','6469225YVAYIh'];_0x1220=function(){return _0x3930a9;};return _0x1220();}function replaceJsonBlocks(_0x49ccb7){const _0x557801=_0x27c0,_0x17e2b9={'biyvh':function(_0x37a915,_0x105edd){return _0x37a915(_0x105edd);},'tnaxW':function(_0x310b19,_0x5136fd){return _0x310b19===_0x5136fd;},'xNPYj':function(_0x3041f7,_0x3ffb00){return _0x3041f7!==_0x3ffb00;}};let _0x5c814e=0x0,_0x11bd3d=_0x49ccb7;return _0x11bd3d=_0x11bd3d['replace'](FENCED_JSON_BLOCK,(_0x57cc05,_0x1cf95b)=>{const _0x1c9c15=_0x27c0,_0xf7a453=_0x1cf95b[_0x1c9c15(0xeb)]();if(!_0x17e2b9['biyvh'](isJsonLike,_0xf7a453))return _0x57cc05;const _0x4b7230=tryParseJson(_0xf7a453);return _0x17e2b9[_0x1c9c15(0xde)](null,_0x4b7230)?_0x57cc05:(_0x5c814e++,_0x17e2b9[_0x1c9c15(0xd2)](convertJsonToCompact,_0x4b7230));}),_0x11bd3d=_0x11bd3d[_0x557801(0xdf)](INLINE_JSON_OBJECT,(_0x1b4ce8,_0x3615a6)=>{const _0x46414a=_0x557801,_0x1f9305=tryParseJson(_0x3615a6);return _0x17e2b9['tnaxW'](null,_0x1f9305)||_0x46414a(0xed)!=typeof _0x1f9305||Array[_0x46414a(0xe9)](_0x1f9305)?_0x1b4ce8:(_0x5c814e++,_0x17e2b9['biyvh'](convertJsonToCompact,_0x1f9305));}),_0x11bd3d=_0x11bd3d['replace'](INLINE_JSON_ARRAY,(_0x284bde,_0x450a02)=>{const _0x13029c=_0x557801,_0x390ed2=tryParseJson(_0x450a02);return _0x17e2b9[_0x13029c(0xd1)](null,_0x390ed2)&&Array[_0x13029c(0xe9)](_0x390ed2)?0x0===_0x390ed2[_0x13029c(0xef)]?_0x284bde:(_0x5c814e++,convertJsonToCompact(_0x390ed2)):_0x284bde;}),{'text':_0x11bd3d,'count':_0x5c814e};}function cleanMarkdown(_0x2f9467){const _0x1ece7e=_0x27c0,_0x39be9b={'pBtMh':function(_0x242b7d,_0x5a0b55){return _0x242b7d!==_0x5a0b55;}};let _0xfbc05e=!0x1,_0x5e16b5=_0x2f9467;const _0x4a3f3d=_0x5e16b5;_0x5e16b5=_0x5e16b5['replace'](CONSECUTIVE_BLANK_LINES,'\x0a\x0a'),_0x39be9b[_0x1ece7e(0xd8)](_0x5e16b5,_0x4a3f3d)&&(_0xfbc05e=!0x0);const _0x56ed3a=_0x5e16b5;_0x5e16b5=_0x5e16b5[_0x1ece7e(0xdf)](EXCESSIVE_LIST_INDENT,'\x20\x20$2\x20'),_0x39be9b['pBtMh'](_0x5e16b5,_0x56ed3a)&&(_0xfbc05e=!0x0);const _0x3998bc=_0x5e16b5;_0x5e16b5=_0x5e16b5['replace'](BOLD_WHOLE_PARAGRAPH,'$1'),_0x5e16b5!==_0x3998bc&&(_0xfbc05e=!0x0);const _0x5bf31e=_0x5e16b5;return _0x5e16b5=_0x5e16b5['replace'](ITALIC_WHOLE_PARAGRAPH,'$1'),_0x39be9b['pBtMh'](_0x5e16b5,_0x5bf31e)&&(_0xfbc05e=!0x0),{'text':_0x5e16b5,'cleaned':_0xfbc05e};}export function optimizeFormat(_0x448660){const _0x2f7685=_0x27c0,_0x307147={'VEomG':function(_0x3890a6,_0xe58e15){return _0x3890a6(_0xe58e15);}};if(!_0x448660||!_0x448660['trim']())return{'text':'','jsonBlocksConverted':0x0,'markdownCleaned':!0x1};const _0x1cf76f=_0x307147['VEomG'](replaceJsonBlocks,_0x448660),_0x4b2b34=_0x307147['VEomG'](cleanMarkdown,_0x1cf76f[_0x2f7685(0xcf)]);return{'text':_0x4b2b34[_0x2f7685(0xcf)],'jsonBlocksConverted':_0x1cf76f[_0x2f7685(0xc7)],'markdownCleaned':_0x4b2b34[_0x2f7685(0xd0)]};}function _0x27c0(_0x344b6c,_0x189b68){_0x344b6c=_0x344b6c-0xc4;const _0x122004=_0x1220();let _0x27c046=_0x122004[_0x344b6c];return _0x27c046;}export const formatOptimizerStrategy={'name':'format-optimizer','compress'(_0x26fa9a,_0x49ba36={}){const _0x24f177=_0x27c0,_0x2fa0fa={'cydlo':function(_0x572675,_0x4280c0){return _0x572675(_0x4280c0);},'uQOma':function(_0x5b02a0,_0x20f844){return _0x5b02a0/_0x20f844;}};if(!_0x26fa9a||!_0x26fa9a['trim']())return{'text':'','originalTokens':0x0,'compressedTokens':0x0,'compressionRatio':0x1,'strategyUsed':'format-optimizer','qualityEstimate':0x1,'wasTruncated':!0x1};const _0x212d36=_0x2fa0fa[_0x24f177(0xc8)](countTokens,_0x26fa9a),_0x3fc8fd=optimizeFormat(_0x26fa9a),_0x26dc29=_0x2fa0fa[_0x24f177(0xc8)](countTokens,_0x3fc8fd[_0x24f177(0xcf)]),_0x240eab=_0x26dc29>0x0?_0x2fa0fa[_0x24f177(0xd4)](_0x212d36,_0x26dc29):0x1,_0x15bf63=_0x3fc8fd['jsonBlocksConverted']>0x0?0.92:_0x3fc8fd[_0x24f177(0xee)]?0.98:0x1;return{'text':_0x3fc8fd['text'],'originalTokens':_0x212d36,'compressedTokens':_0x26dc29,'compressionRatio':_0x240eab,'strategyUsed':'format-optimizer','qualityEstimate':_0x15bf63,'wasTruncated':!0x1};}};
1
+ const _0x10acd5=_0x48bb;(function(_0x32d5b4,_0x51fa68){const _0x8cf7fd=_0x48bb,_0x35b8fc=_0x32d5b4();while(!![]){try{const _0x58e9ad=parseInt(_0x8cf7fd(0x1f9))/0x1*(-parseInt(_0x8cf7fd(0x204))/0x2)+parseInt(_0x8cf7fd(0x1f2))/0x3+-parseInt(_0x8cf7fd(0x1ea))/0x4+-parseInt(_0x8cf7fd(0x1e1))/0x5*(-parseInt(_0x8cf7fd(0x201))/0x6)+parseInt(_0x8cf7fd(0x1fe))/0x7*(-parseInt(_0x8cf7fd(0x1ec))/0x8)+-parseInt(_0x8cf7fd(0x1f8))/0x9*(parseInt(_0x8cf7fd(0x1f7))/0xa)+parseInt(_0x8cf7fd(0x1ee))/0xb;if(_0x58e9ad===_0x51fa68)break;else _0x35b8fc['push'](_0x35b8fc['shift']());}catch(_0x11ea22){_0x35b8fc['push'](_0x35b8fc['shift']());}}}(_0x4f25,0xbbc50));function _0x4f25(){const _0x5a39e9=['startsWith','map','object','1013160svgTdv','format-optimizer','cBYjr','string','jsonBlocksConverted','1517330MoMIzg','63icGvyi','8SyZrFd','split','join','every','pMOSK','2605379BhBEyj','nWYbi','SQCEU','144viPeVI','fiSFY','count','228428LtQpDX','length','KoesB','ZvhjJ','1|2|5|0|4|3','bKyDU','null','ycJHj','push','QtJVj','stringify','some','isArray','qIfio','replace','307120mKMIaS','markdownCleaned','JDixq','values','SWtFw','parse','Uzybp','\x20\x20$2\x20','SOVtJ','792272JDRBtP','NVonx','24jdvzRf','text','24724810rabwuP'];_0x4f25=function(){return _0x5a39e9;};return _0x4f25();}import{countTokens}from'../../shared/token-counter.js';const FENCED_JSON_BLOCK=/```(?:json)?\s*\n([\s\S]*?)```/g,INLINE_JSON_OBJECT=/(?<![\\:a-zA-Z0-9_"'])(\{[\s\S]*?\})(?![\\a-zA-Z0-9_])/g,INLINE_JSON_ARRAY=/(?<![\\:a-zA-Z0-9_"'])(\[[\s\S]*?\])(?![\\a-zA-Z0-9_])/g,CONSECUTIVE_BLANK_LINES=/\n{3,}/g,EXCESSIVE_LIST_INDENT=/^( {4,}|\t{2,})([*\-+]|\d+\.)\s/gm,BOLD_WHOLE_PARAGRAPH=/^\*\*([^*\n]+)\*\*$/gm,ITALIC_WHOLE_PARAGRAPH=/^_([^_\n]+)_$/gm,MAX_FLATTEN_DEPTH=0x2;function isJsonLike(_0xbd5ade){const _0x48c015=_0x48bb,_0x3b68cf=_0xbd5ade['trim']();return _0x3b68cf[_0x48c015(0x1ef)]('{')&&_0x3b68cf['endsWith']('}')||_0x3b68cf['startsWith']('[')&&_0x3b68cf['endsWith'](']');}function tryParseJson(_0x329b74){const _0x51672d=_0x48bb;try{return JSON[_0x51672d(0x1e6)](_0x329b74);}catch{return null;}}function flattenObject(_0x3548b5,_0x35fb84,_0x416752){const _0x44ac31=_0x48bb,_0x4521b3={'PzQGy':function(_0xba2dd0,_0x5df925){return _0xba2dd0!==_0x5df925;},'QtJVj':function(_0x2f2614,_0x27aa47){return _0x2f2614==_0x27aa47;},'MEBmA':_0x44ac31(0x1f1)},_0x428aa8=[];for(const [_0x54b3bd,_0x12223b]of Object['entries'](_0x3548b5)){const _0x4d1566=_0x35fb84?_0x35fb84+'.'+_0x54b3bd:_0x54b3bd;_0x4521b3['PzQGy'](null,_0x12223b)&&_0x4521b3[_0x44ac31(0x1db)](_0x4521b3['MEBmA'],typeof _0x12223b)&&!Array[_0x44ac31(0x1de)](_0x12223b)&&_0x416752<0x2?_0x428aa8['push'](...flattenObject(_0x12223b,_0x4d1566,_0x416752+0x1)):_0x428aa8[_0x44ac31(0x1da)]([_0x4d1566,_0x12223b]);}return _0x428aa8;}function formatValue(_0x16fdd2){const _0x24f6b1=_0x48bb,_0xfb00de={'KoesB':_0x24f6b1(0x208),'gAWCr':function(_0x2df880,_0xe6e4f7){return _0x2df880==_0xe6e4f7;},'Uzybp':'number','THYkp':'boolean','BoeLc':_0x24f6b1(0x1d8),'tVBYQ':function(_0x3502d3,_0x5e154f){return _0x3502d3===_0x5e154f;},'pMOSK':_0x24f6b1(0x1f5)},_0x571b05=_0xfb00de[_0x24f6b1(0x206)][_0x24f6b1(0x1fa)]('|');let _0x113e53=0x0;while(!![]){switch(_0x571b05[_0x113e53++]){case'0':if(_0xfb00de['gAWCr'](_0xfb00de[_0x24f6b1(0x1e7)],typeof _0x16fdd2)||_0xfb00de['THYkp']==typeof _0x16fdd2)return String(_0x16fdd2);continue;case'1':if(null===_0x16fdd2)return _0xfb00de['BoeLc'];continue;case'2':if(_0xfb00de['tVBYQ'](void 0x0,_0x16fdd2))return'undefined';continue;case'3':return JSON['stringify'](_0x16fdd2);case'4':if(Array['isArray'](_0x16fdd2))return'['+_0x16fdd2[_0x24f6b1(0x1f0)](_0x2868c9=>'object'==typeof _0x2868c9&&null!==_0x2868c9?JSON[_0x24f6b1(0x1dc)](_0x2868c9):formatValue(_0x2868c9))[_0x24f6b1(0x1fb)](',\x20')+']';continue;case'5':if(_0xfb00de['gAWCr'](_0xfb00de[_0x24f6b1(0x1fd)],typeof _0x16fdd2))return _0x16fdd2;continue;}break;}}function hasDeepNesting(_0x2c9d97,_0x599fb4){const _0x300da0=_0x48bb,_0x5b5675={'SOVtJ':function(_0x10920a,_0x16a72e){return _0x10920a>_0x16a72e;},'ycJHj':'object'};return _0x5b5675[_0x300da0(0x1e9)](_0x599fb4,0x2)||_0x5b5675[_0x300da0(0x1d9)]==typeof _0x2c9d97&&null!==_0x2c9d97&&(Array[_0x300da0(0x1de)](_0x2c9d97)?_0x2c9d97[_0x300da0(0x1dd)](_0x533cab=>hasDeepNesting(_0x533cab,_0x599fb4+0x1)):Object[_0x300da0(0x1e4)](_0x2c9d97)[_0x300da0(0x1dd)](_0x3bc9a7=>hasDeepNesting(_0x3bc9a7,_0x599fb4+0x1)));}function convertJsonToCompact(_0x4665a1){const _0x32a1b5=_0x48bb,_0x3619d7={'qIfio':function(_0x1c3ce2,_0x2ef2da,_0x487fc7){return _0x1c3ce2(_0x2ef2da,_0x487fc7);},'SQCEU':function(_0x3fcc85,_0x51ec3e){return _0x3fcc85(_0x51ec3e);},'zwBfK':function(_0x38c841,_0x5a1ca4){return _0x38c841(_0x5a1ca4);}};if(Array[_0x32a1b5(0x1de)](_0x4665a1))return _0x4665a1[_0x32a1b5(0x1fc)](_0x41806a=>_0x32a1b5(0x1f1)!=typeof _0x41806a||null===_0x41806a)?'['+_0x4665a1[_0x32a1b5(0x1f0)](formatValue)['join'](',\x20')+']':_0x3619d7[_0x32a1b5(0x1df)](hasDeepNesting,_0x4665a1,0x0)?JSON[_0x32a1b5(0x1dc)](_0x4665a1):_0x4665a1['map'](_0x1c8815=>'object'!=typeof _0x1c8815||null===_0x1c8815||Array['isArray'](_0x1c8815)?formatValue(_0x1c8815):convertObjectToCompact(_0x1c8815))[_0x32a1b5(0x1fb)]('\x0a');if(_0x32a1b5(0x1f1)==typeof _0x4665a1&&null!==_0x4665a1){if(_0x3619d7[_0x32a1b5(0x1df)](hasDeepNesting,_0x4665a1,0x0)){const _0x4f192a=_0x4665a1;if(!Object[_0x32a1b5(0x1e4)](_0x4f192a)[_0x32a1b5(0x1dd)](_0x49a9f4=>'object'!=typeof _0x49a9f4||null===_0x49a9f4))return JSON['stringify'](_0x4665a1);}return _0x3619d7[_0x32a1b5(0x200)](convertObjectToCompact,_0x4665a1);}return _0x3619d7['zwBfK'](formatValue,_0x4665a1);}function convertObjectToCompact(_0xaaef09){const _0x3f863c=_0x48bb,_0x4b3321={'bKyDU':function(_0xd3e93,_0x158bce,_0x2f390a,_0x474337){return _0xd3e93(_0x158bce,_0x2f390a,_0x474337);},'SWtFw':function(_0x54a363,_0x36ff85){return _0x54a363<=_0x36ff85;}},_0x4d5ccd=_0x4b3321[_0x3f863c(0x209)](flattenObject,_0xaaef09,'',0x0)['map'](([_0x5b3683,_0x412e26])=>'object'==typeof _0x412e26&&null!==_0x412e26?_0x5b3683+':\x20'+JSON[_0x3f863c(0x1dc)](_0x412e26):_0x5b3683+':\x20'+formatValue(_0x412e26));return _0x4b3321[_0x3f863c(0x1e5)](_0x4d5ccd[_0x3f863c(0x205)],0x4)&&_0x4d5ccd[_0x3f863c(0x1fc)](_0x15306b=>_0x15306b[_0x3f863c(0x205)]<0x3c)?_0x4d5ccd['join'](',\x20'):_0x4d5ccd[_0x3f863c(0x1fb)]('\x0a');}function replaceJsonBlocks(_0x5cfc59){const _0x47aec8=_0x48bb,_0x2f1436={'cBYjr':function(_0x1baa66,_0x4fbd74){return _0x1baa66(_0x4fbd74);},'LUhIq':function(_0x3e9349,_0x5c711c){return _0x3e9349===_0x5c711c;},'wDWhZ':function(_0xd4e50e,_0x53cc6f){return _0xd4e50e(_0x53cc6f);},'RRkIQ':function(_0x245321,_0x47a6ce){return _0x245321(_0x47a6ce);}};let _0x191ac0=0x0,_0x3cc497=_0x5cfc59;return _0x3cc497=_0x3cc497['replace'](FENCED_JSON_BLOCK,(_0xa0e972,_0x479877)=>{const _0xfa1693=_0x479877['trim']();if(!_0x2f1436['cBYjr'](isJsonLike,_0xfa1693))return _0xa0e972;const _0x1d5934=tryParseJson(_0xfa1693);return null===_0x1d5934?_0xa0e972:(_0x191ac0++,convertJsonToCompact(_0x1d5934));}),_0x3cc497=_0x3cc497['replace'](INLINE_JSON_OBJECT,(_0x5c318a,_0x3e284f)=>{const _0x6ed197=_0x48bb,_0x56e74d=_0x2f1436[_0x6ed197(0x1f4)](tryParseJson,_0x3e284f);return _0x2f1436['LUhIq'](null,_0x56e74d)||_0x6ed197(0x1f1)!=typeof _0x56e74d||Array[_0x6ed197(0x1de)](_0x56e74d)?_0x5c318a:(_0x191ac0++,_0x2f1436['wDWhZ'](convertJsonToCompact,_0x56e74d));}),_0x3cc497=_0x3cc497[_0x47aec8(0x1e0)](INLINE_JSON_ARRAY,(_0x1d28d8,_0x179c7e)=>{const _0x208a9c=_0x47aec8,_0x586394=tryParseJson(_0x179c7e);return null!==_0x586394&&Array[_0x208a9c(0x1de)](_0x586394)?_0x2f1436['LUhIq'](0x0,_0x586394['length'])?_0x1d28d8:(_0x191ac0++,_0x2f1436['RRkIQ'](convertJsonToCompact,_0x586394)):_0x1d28d8;}),{'text':_0x3cc497,'count':_0x191ac0};}function cleanMarkdown(_0x13b3eb){const _0x8959d5=_0x48bb,_0x3d2928={'NVonx':_0x8959d5(0x1e8)};let _0x59c78d=!0x1,_0x43016e=_0x13b3eb;const _0x151f42=_0x43016e;_0x43016e=_0x43016e[_0x8959d5(0x1e0)](CONSECUTIVE_BLANK_LINES,'\x0a\x0a'),_0x43016e!==_0x151f42&&(_0x59c78d=!0x0);const _0x284ee3=_0x43016e;_0x43016e=_0x43016e['replace'](EXCESSIVE_LIST_INDENT,_0x3d2928[_0x8959d5(0x1eb)]),_0x43016e!==_0x284ee3&&(_0x59c78d=!0x0);const _0x1849b=_0x43016e;_0x43016e=_0x43016e['replace'](BOLD_WHOLE_PARAGRAPH,'$1'),_0x43016e!==_0x1849b&&(_0x59c78d=!0x0);const _0x4e0540=_0x43016e;return _0x43016e=_0x43016e['replace'](ITALIC_WHOLE_PARAGRAPH,'$1'),_0x43016e!==_0x4e0540&&(_0x59c78d=!0x0),{'text':_0x43016e,'cleaned':_0x59c78d};}export function optimizeFormat(_0x4223f6){const _0x393ce0=_0x48bb,_0x29ac70={'ZvhjJ':function(_0x30710c,_0x22091d){return _0x30710c(_0x22091d);}};if(!_0x4223f6||!_0x4223f6['trim']())return{'text':'','jsonBlocksConverted':0x0,'markdownCleaned':!0x1};const _0x327686=_0x29ac70[_0x393ce0(0x207)](replaceJsonBlocks,_0x4223f6),_0x41a17e=cleanMarkdown(_0x327686[_0x393ce0(0x1ed)]);return{'text':_0x41a17e[_0x393ce0(0x1ed)],'jsonBlocksConverted':_0x327686[_0x393ce0(0x203)],'markdownCleaned':_0x41a17e['cleaned']};}function _0x48bb(_0x1a7a53,_0x67e083){_0x1a7a53=_0x1a7a53-0x1d8;const _0x4f2508=_0x4f25();let _0x48bbd8=_0x4f2508[_0x1a7a53];return _0x48bbd8;}export const formatOptimizerStrategy={'name':_0x10acd5(0x1f3),'compress'(_0x4986e6,_0x205d05={}){const _0x1dcba7=_0x10acd5,_0x12b655={'nWYbi':_0x1dcba7(0x1f3),'JDixq':function(_0x2115c4,_0x597d39){return _0x2115c4(_0x597d39);},'UWvXy':function(_0x20faec,_0x431b0a){return _0x20faec(_0x431b0a);},'fiSFY':function(_0x344494,_0x186303){return _0x344494>_0x186303;}};if(!_0x4986e6||!_0x4986e6['trim']())return{'text':'','originalTokens':0x0,'compressedTokens':0x0,'compressionRatio':0x1,'strategyUsed':_0x12b655['nWYbi'],'qualityEstimate':0x1,'wasTruncated':!0x1};const _0x216bbc=countTokens(_0x4986e6),_0x350aaa=_0x12b655[_0x1dcba7(0x1e3)](optimizeFormat,_0x4986e6),_0x49c67f=_0x12b655['UWvXy'](countTokens,_0x350aaa['text']),_0x1fcfdc=_0x12b655['fiSFY'](_0x49c67f,0x0)?_0x216bbc/_0x49c67f:0x1,_0x29a6c8=_0x12b655[_0x1dcba7(0x202)](_0x350aaa[_0x1dcba7(0x1f6)],0x0)?0.92:_0x350aaa[_0x1dcba7(0x1e2)]?0.98:0x1;return{'text':_0x350aaa[_0x1dcba7(0x1ed)],'originalTokens':_0x216bbc,'compressedTokens':_0x49c67f,'compressionRatio':_0x1fcfdc,'strategyUsed':_0x12b655[_0x1dcba7(0x1ff)],'qualityEstimate':_0x29a6c8,'wasTruncated':!0x1};}};
package/dist/plugin/compression/index.js CHANGED
@@ -1 +1 @@
1
- (function(_0x5054f9,_0x156d11){var _0x3ef9ab=_0x2304,_0x4c1e56=_0x5054f9();while(!![]){try{var _0x292a90=-parseInt(_0x3ef9ab(0x142))/0x1+-parseInt(_0x3ef9ab(0x13e))/0x2*(-parseInt(_0x3ef9ab(0x144))/0x3)+parseInt(_0x3ef9ab(0x13f))/0x4+-parseInt(_0x3ef9ab(0x13c))/0x5*(parseInt(_0x3ef9ab(0x145))/0x6)+parseInt(_0x3ef9ab(0x143))/0x7*(-parseInt(_0x3ef9ab(0x13d))/0x8)+-parseInt(_0x3ef9ab(0x140))/0x9*(-parseInt(_0x3ef9ab(0x13a))/0xa)+-parseInt(_0x3ef9ab(0x141))/0xb*(-parseInt(_0x3ef9ab(0x13b))/0xc);if(_0x292a90===_0x156d11)break;else _0x4c1e56['push'](_0x4c1e56['shift']());}catch(_0x347fe5){_0x4c1e56['push'](_0x4c1e56['shift']());}}}(_0x265e,0x85ec1));function _0x2304(_0x1186be,_0x1ac35e){_0x1186be=_0x1186be-0x13a;var _0x265e9b=_0x265e();var _0x2304b2=_0x265e9b[_0x1186be];return _0x2304b2;}export{splitSentences,extractContentWords,STOPWORDS,isStopword}from'./text-utils.js';function _0x265e(){var _0x37689d=['6638004ubCLPB','1089145XorrXJ','6634480vGYQhI','239156IFHNQp','2927936LzglvC','72JwAfrk','11TOIMav','128897DssSLy','7neXZtb','9CZMrFR','6rnkOsf','100870KEDMnS'];_0x265e=function(){return _0x37689d;};return _0x265e();}export{compressRuleBased,ruleBasedStrategy}from'./rule-based.js';export{compressStructural,structuralStrategy}from'./structural.js';export{compressTokenBudget,tokenBudgetStrategy}from'./token-budget.js';export{compressChunked,chunkedStrategy}from'./chunked.js';export{estimateQuality,quickQualityScore}from'./quality.js';export{CompressionPipeline,createDefaultPipeline}from'./pipeline.js';export{anonymizeText,deanonymizeText,anonymizeStrategy}from'./anonymize.js';export{deduplicateText,dedupStrategy}from'./dedup.js';export{CompressionCache,createCachedPipeline}from'./cache.js';export{optimizeFormat,formatOptimizerStrategy}from'./format-optimizer.js';
1
+ (function(_0x526e68,_0x50f8a6){var _0x2e5edc=_0x338e,_0xdbf2ac=_0x526e68();while(!![]){try{var _0x2e7d62=parseInt(_0x2e5edc(0x1b4))/0x1*(-parseInt(_0x2e5edc(0x1af))/0x2)+parseInt(_0x2e5edc(0x1b5))/0x3*(parseInt(_0x2e5edc(0x1b7))/0x4)+parseInt(_0x2e5edc(0x1ae))/0x5*(parseInt(_0x2e5edc(0x1b0))/0x6)+-parseInt(_0x2e5edc(0x1ad))/0x7+-parseInt(_0x2e5edc(0x1b2))/0x8*(parseInt(_0x2e5edc(0x1ac))/0x9)+-parseInt(_0x2e5edc(0x1b1))/0xa*(parseInt(_0x2e5edc(0x1b3))/0xb)+parseInt(_0x2e5edc(0x1b6))/0xc;if(_0x2e7d62===_0x50f8a6)break;else _0xdbf2ac['push'](_0xdbf2ac['shift']());}catch(_0x180ee8){_0xdbf2ac['push'](_0xdbf2ac['shift']());}}}(_0x348b,0xa68f3));export{splitSentences,extractContentWords,STOPWORDS,isStopword}from'./text-utils.js';function _0x338e(_0x20ca28,_0x447e24){_0x20ca28=_0x20ca28-0x1ac;var _0x348bf3=_0x348b();var _0x338e48=_0x348bf3[_0x20ca28];return _0x338e48;}export{compressRuleBased,ruleBasedStrategy}from'./rule-based.js';export{compressStructural,structuralStrategy}from'./structural.js';export{compressTokenBudget,tokenBudgetStrategy}from'./token-budget.js';export{compressChunked,chunkedStrategy}from'./chunked.js';export{estimateQuality,quickQualityScore}from'./quality.js';export{CompressionPipeline,createDefaultPipeline}from'./pipeline.js';export{anonymizeText,deanonymizeText,anonymizeStrategy}from'./anonymize.js';export{deduplicateText,dedupStrategy}from'./dedup.js';export{CompressionCache,createCachedPipeline}from'./cache.js';function _0x348b(){var _0x887f3=['640088okudND','584837McvqOa','13WpZHKu','9HwKkml','35728356dCHpkf','496556Yjghsa','9GGolPp','4387775GpqEAK','1215oJhojc','164534EhdpiC','5562OuopND','210RnvphC'];_0x348b=function(){return _0x887f3;};return _0x348b();}export{optimizeFormat,formatOptimizerStrategy}from'./format-optimizer.js';
package/dist/plugin/compression/pipeline.js CHANGED
@@ -1 +1 @@
1
- const _0x15a0aa=_0x3b61;(function(_0x170cbf,_0x2323a3){const _0xfcd950=_0x3b61,_0x279ed0=_0x170cbf();while(!![]){try{const _0x182d80=-parseInt(_0xfcd950(0x1ba))/0x1*(parseInt(_0xfcd950(0x1b3))/0x2)+parseInt(_0xfcd950(0x1b8))/0x3+parseInt(_0xfcd950(0x1a3))/0x4+parseInt(_0xfcd950(0x1ac))/0x5*(parseInt(_0xfcd950(0x1a7))/0x6)+parseInt(_0xfcd950(0x1a2))/0x7+parseInt(_0xfcd950(0x1b7))/0x8+-parseInt(_0xfcd950(0x1b5))/0x9*(parseInt(_0xfcd950(0x1aa))/0xa);if(_0x182d80===_0x2323a3)break;else _0x279ed0['push'](_0x279ed0['shift']());}catch(_0x582341){_0x279ed0['push'](_0x279ed0['shift']());}}}(_0x1e17,0x5e37f));import{countTokens}from'../../shared/token-counter.js';import{ruleBasedStrategy}from'./rule-based.js';function _0x1e17(){const _0x43dc1d=['13086yLyZPR','text','891wyVkVJ','executeStage','3880520nidnGX','1675173rvCxFT','now','116GJreLz','VaPrz','Uszsw','5017656XiAlfL','1220996TnmNoP','stopOnTarget','join','wasTruncated','6gGehdt','compress','jbJFG','122990XlwdKh','trim','1484975dsiErd','minQuality','runStages','buildFinalResult','map','targetTokens','strategies'];_0x1e17=function(){return _0x43dc1d;};return _0x1e17();}function _0x3b61(_0xe7b4d3,_0x94dee2){_0xe7b4d3=_0xe7b4d3-0x1a1;const _0x1e1754=_0x1e17();let _0x3b61cd=_0x1e1754[_0xe7b4d3];return _0x3b61cd;}import{structuralStrategy}from'./structural.js';import{tokenBudgetStrategy}from'./token-budget.js';import{quickQualityScore}from'./quality.js';const DEFAULT_TARGET_TOKENS=0x1f4,DEFAULT_MIN_QUALITY=0.8;function buildEmptyPipelineResult(){return{'text':'','originalTokens':0x0,'compressedTokens':0x0,'compressionRatio':0x1,'strategyUsed':'pipeline','qualityEstimate':0x1,'wasTruncated':!0x1,'stages':[],'qualityReport':{'overallScore':0x1,'informationRetention':0x1,'structuralIntegrity':0x1,'keyTermPreservation':0x1},'targetMet':!0x0};}function buildPassthroughResult(_0x199233,_0xa5c579){return{'text':_0x199233,'originalTokens':_0xa5c579,'compressedTokens':_0xa5c579,'compressionRatio':0x1,'strategyUsed':'pipeline','qualityEstimate':0x1,'wasTruncated':!0x1,'stages':[],'qualityReport':{'overallScore':0x1,'informationRetention':0x1,'structuralIntegrity':0x1,'keyTermPreservation':0x1},'targetMet':!0x0};}export class CompressionPipeline{[_0x15a0aa(0x1b2)];[_0x15a0aa(0x1b1)];['minQuality'];['stopOnTarget'];constructor(_0x4245f8){const _0x196eea=_0x15a0aa;this['strategies']=_0x4245f8['strategies'],this['targetTokens']=_0x4245f8['targetTokens']??0x1f4,this[_0x196eea(0x1ad)]=_0x4245f8[_0x196eea(0x1ad)]??0.8,this[_0x196eea(0x1a4)]=_0x4245f8['stopOnTarget']??!0x0;}[_0x15a0aa(0x1a8)](_0x59dd11,_0x582d61){const _0xb2d3e0=_0x15a0aa,_0x41460a={'xKAiI':function(_0x557804){return _0x557804();},'jbJFG':function(_0x5e701a,_0x1e083e){return _0x5e701a(_0x1e083e);},'Uszsw':function(_0x3d38ad,_0x2efd21,_0x821ca1){return _0x3d38ad(_0x2efd21,_0x821ca1);}},_0x53bf2c=_0x582d61??this[_0xb2d3e0(0x1b1)];if(!_0x59dd11||!_0x59dd11[_0xb2d3e0(0x1ab)]())return _0x41460a['xKAiI'](buildEmptyPipelineResult);const _0x5833ff=_0x59dd11,_0x31f15b=_0x41460a[_0xb2d3e0(0x1a9)](countTokens,_0x59dd11);if(_0x31f15b<=_0x53bf2c)return _0x41460a[_0xb2d3e0(0x1a1)](buildPassthroughResult,_0x59dd11,_0x31f15b);const {compressedText:_0x166d68,stages:_0x5ef6db,wasTruncated:_0x24e9d1}=this[_0xb2d3e0(0x1ae)](_0x5833ff,_0x31f15b,_0x53bf2c);return this[_0xb2d3e0(0x1af)](_0x5833ff,_0x31f15b,_0x166d68,_0x5ef6db,_0x24e9d1,_0x53bf2c);}[_0x15a0aa(0x1ae)](_0x2f0876,_0x592e1e,_0x291065){const _0x331b30=_0x15a0aa,_0x35b896={'BeOqW':function(_0x2aa57f,_0xe342c6){return _0x2aa57f<=_0xe342c6;},'ZhxJa':function(_0x4a367d,_0x1706e0){return _0x4a367d>_0x1706e0;}};let _0x5cd851=_0x2f0876;const _0x277ae3=[];let _0x3944ed=!0x1;for(const _0x31c5f2 of this[_0x331b30(0x1b2)]){const _0x4de77f=countTokens(_0x5cd851);if(this[_0x331b30(0x1a4)]&&_0x35b896['BeOqW'](_0x4de77f,_0x291065))break;const _0x441767=this['executeStage'](_0x31c5f2,_0x5cd851,_0x291065,_0x592e1e);if(quickQualityScore(_0x2f0876,_0x441767['text'])<this[_0x331b30(0x1ad)]&&_0x35b896['ZhxJa'](_0x277ae3['length'],0x0))break;_0x277ae3['push'](_0x441767['stageMetrics']),_0x5cd851=_0x441767[_0x331b30(0x1b4)],_0x441767[_0x331b30(0x1a6)]&&(_0x3944ed=!0x0);}return{'compressedText':_0x5cd851,'stages':_0x277ae3,'wasTruncated':_0x3944ed};}[_0x15a0aa(0x1b6)](_0x41068f,_0x1463ce,_0x3c5b68,_0x26905f){const _0x46f687=_0x15a0aa,_0x122e7d={'mXFrQ':function(_0x39616c,_0x13edc3){return _0x39616c(_0x13edc3);},'rAXsv':function(_0x155cf0,_0x3874fa){return _0x155cf0-_0x3874fa;},'bsEqv':function(_0x213b4e,_0x49ebdb){return _0x213b4e-_0x49ebdb;}},_0x55b5ca=_0x122e7d['mXFrQ'](countTokens,_0x1463ce),_0x53430a=performance['now'](),_0x3e54b6={'maxTokens':_0x3c5b68,'targetRatio':_0x3c5b68/_0x26905f},_0x444fa2=_0x41068f[_0x46f687(0x1a8)](_0x1463ce,_0x3e54b6),_0x574fbb=_0x122e7d['rAXsv'](performance[_0x46f687(0x1b9)](),_0x53430a);return{'text':_0x444fa2[_0x46f687(0x1b4)],'wasTruncated':_0x444fa2[_0x46f687(0x1a6)],'stageMetrics':{'strategyName':_0x41068f['name'],'inputTokens':_0x55b5ca,'outputTokens':_0x444fa2['compressedTokens'],'savingsRatio':_0x55b5ca>0x0?_0x122e7d['bsEqv'](0x1,_0x444fa2['compressedTokens']/_0x55b5ca):0x0,'durationMs':_0x574fbb}};}['buildFinalResult'](_0x3b37be,_0x51ae62,_0x5835b0,_0x22b72e,_0x4f066a,_0x5bd806){const _0x53a602=_0x15a0aa,_0xa582f5={'lHtXm':function(_0x19dadb,_0x4f9db0){return _0x19dadb>_0x4f9db0;},'VaPrz':function(_0x4fe5d0,_0x3cb655){return _0x4fe5d0/_0x3cb655;}},_0x214b85=countTokens(_0x5835b0),_0xd6cf14=quickQualityScore(_0x3b37be,_0x5835b0);return{'text':_0x5835b0,'originalTokens':_0x51ae62,'compressedTokens':_0x214b85,'compressionRatio':_0xa582f5['lHtXm'](_0x214b85,0x0)?_0xa582f5[_0x53a602(0x1bb)](_0x51ae62,_0x214b85):0x1,'strategyUsed':_0x22b72e[_0x53a602(0x1b0)](_0x6ef779=>_0x6ef779['strategyName'])[_0x53a602(0x1a5)]('+'),'qualityEstimate':_0xd6cf14,'wasTruncated':_0x4f066a,'stages':_0x22b72e,'qualityReport':{'overallScore':_0xd6cf14,'informationRetention':_0xd6cf14,'structuralIntegrity':_0xd6cf14,'keyTermPreservation':_0xd6cf14},'targetMet':_0x214b85<=_0x5bd806};}}export function createDefaultPipeline(_0x596a99){return new CompressionPipeline({'strategies':[ruleBasedStrategy,structuralStrategy,tokenBudgetStrategy],'targetTokens':_0x596a99,'minQuality':0.8,'stopOnTarget':!0x0});}
1
+ function _0x31db(_0x2dcf43,_0xa2f0c6){_0x2dcf43=_0x2dcf43-0x157;const _0x113c05=_0x113c();let _0x31db70=_0x113c05[_0x2dcf43];return _0x31db70;}const _0x5386a7=_0x31db;(function(_0x2d93b7,_0xe1be7f){const _0x1d3062=_0x31db,_0x525634=_0x2d93b7();while(!![]){try{const _0x4d23d1=-parseInt(_0x1d3062(0x166))/0x1+parseInt(_0x1d3062(0x16f))/0x2+-parseInt(_0x1d3062(0x16a))/0x3*(-parseInt(_0x1d3062(0x177))/0x4)+parseInt(_0x1d3062(0x160))/0x5*(-parseInt(_0x1d3062(0x161))/0x6)+-parseInt(_0x1d3062(0x15a))/0x7+parseInt(_0x1d3062(0x159))/0x8*(parseInt(_0x1d3062(0x15d))/0x9)+parseInt(_0x1d3062(0x174))/0xa;if(_0x4d23d1===_0xe1be7f)break;else _0x525634['push'](_0x525634['shift']());}catch(_0x5b3f5c){_0x525634['push'](_0x525634['shift']());}}}(_0x113c,0x69f39));import{countTokens}from'../../shared/token-counter.js';import{ruleBasedStrategy}from'./rule-based.js';import{structuralStrategy}from'./structural.js';import{tokenBudgetStrategy}from'./token-budget.js';function _0x113c(){const _0x41e93e=['jWyuf','pipeline','wasTruncated','637605TBIhot','targetTokens','text','YcrlX','213YIAPgQ','name','minQuality','stopOnTarget','push','433530vzvSeE','strategyName','now','zNZSb','map','1335780DPOleA','RKdrS','strategies','10668zQFJsl','compressedTokens','length','stageMetrics','38896fAvjLg','751303gQteik','WBbBu','runStages','1395IlYRcX','LuDWD','ygOKZ','135mutiys','25422BXmioZ','trim'];_0x113c=function(){return _0x41e93e;};return _0x113c();}import{quickQualityScore}from'./quality.js';const DEFAULT_TARGET_TOKENS=0x1f4,DEFAULT_MIN_QUALITY=0.8;function buildEmptyPipelineResult(){const _0x35c668=_0x31db,_0x156940={'RKdrS':_0x35c668(0x164)};return{'text':'','originalTokens':0x0,'compressedTokens':0x0,'compressionRatio':0x1,'strategyUsed':_0x156940[_0x35c668(0x175)],'qualityEstimate':0x1,'wasTruncated':!0x1,'stages':[],'qualityReport':{'overallScore':0x1,'informationRetention':0x1,'structuralIntegrity':0x1,'keyTermPreservation':0x1},'targetMet':!0x0};}function buildPassthroughResult(_0x4c5d1f,_0x2f0eff){return{'text':_0x4c5d1f,'originalTokens':_0x2f0eff,'compressedTokens':_0x2f0eff,'compressionRatio':0x1,'strategyUsed':'pipeline','qualityEstimate':0x1,'wasTruncated':!0x1,'stages':[],'qualityReport':{'overallScore':0x1,'informationRetention':0x1,'structuralIntegrity':0x1,'keyTermPreservation':0x1},'targetMet':!0x0};}export class CompressionPipeline{[_0x5386a7(0x176)];[_0x5386a7(0x167)];['minQuality'];['stopOnTarget'];constructor(_0x59b9dd){const _0x83e9ef=_0x5386a7;this[_0x83e9ef(0x176)]=_0x59b9dd[_0x83e9ef(0x176)],this['targetTokens']=_0x59b9dd['targetTokens']??0x1f4,this['minQuality']=_0x59b9dd[_0x83e9ef(0x16c)]??0.8,this[_0x83e9ef(0x16d)]=_0x59b9dd[_0x83e9ef(0x16d)]??!0x0;}['compress'](_0x4a36b6,_0x47d66a){const _0x368b97=_0x5386a7,_0x4713b2={'WBbBu':function(_0x42f60b){return _0x42f60b();}},_0x5dae95=_0x47d66a??this['targetTokens'];if(!_0x4a36b6||!_0x4a36b6[_0x368b97(0x162)]())return _0x4713b2[_0x368b97(0x15b)](buildEmptyPipelineResult);const _0x108c2e=_0x4a36b6,_0x544297=countTokens(_0x4a36b6);if(_0x544297<=_0x5dae95)return buildPassthroughResult(_0x4a36b6,_0x544297);const {compressedText:_0x2979a7,stages:_0x3eb7b8,wasTruncated:_0x477893}=this[_0x368b97(0x15c)](_0x108c2e,_0x544297,_0x5dae95);return this['buildFinalResult'](_0x108c2e,_0x544297,_0x2979a7,_0x3eb7b8,_0x477893,_0x5dae95);}[_0x5386a7(0x15c)](_0x137b70,_0x50e4b5,_0x4462b4){const _0x48ea91=_0x5386a7,_0x44eca8={'lRYyc':function(_0x38585a,_0x2fd702){return _0x38585a<_0x2fd702;},'YcrlX':function(_0x189ed0,_0x366af6,_0x1133a4){return _0x189ed0(_0x366af6,_0x1133a4);},'ygOKZ':function(_0x120414,_0x26d851){return _0x120414>_0x26d851;}};let _0xd742ef=_0x137b70;const _0x150a23=[];let _0x36774c=!0x1;for(const _0x186ae9 of this[_0x48ea91(0x176)]){const _0x1cf7d1=countTokens(_0xd742ef);if(this['stopOnTarget']&&_0x1cf7d1<=_0x4462b4)break;const _0x299138=this['executeStage'](_0x186ae9,_0xd742ef,_0x4462b4,_0x50e4b5);if(_0x44eca8['lRYyc'](_0x44eca8[_0x48ea91(0x169)](quickQualityScore,_0x137b70,_0x299138['text']),this['minQuality'])&&_0x44eca8[_0x48ea91(0x15f)](_0x150a23[_0x48ea91(0x157)],0x0))break;_0x150a23[_0x48ea91(0x16e)](_0x299138[_0x48ea91(0x158)]),_0xd742ef=_0x299138[_0x48ea91(0x168)],_0x299138[_0x48ea91(0x165)]&&(_0x36774c=!0x0);}return{'compressedText':_0xd742ef,'stages':_0x150a23,'wasTruncated':_0x36774c};}['executeStage'](_0x5e3d58,_0x18a49d,_0x19620a,_0x42e2db){const _0x1b8474=_0x5386a7,_0x1346c4={'LuDWD':function(_0x319456,_0x4faf71){return _0x319456/_0x4faf71;}},_0x45e356=countTokens(_0x18a49d),_0x265944=performance[_0x1b8474(0x171)](),_0x8e0fd4={'maxTokens':_0x19620a,'targetRatio':_0x19620a/_0x42e2db},_0x24bc9f=_0x5e3d58['compress'](_0x18a49d,_0x8e0fd4),_0x2c94f9=performance[_0x1b8474(0x171)]()-_0x265944;return{'text':_0x24bc9f[_0x1b8474(0x168)],'wasTruncated':_0x24bc9f[_0x1b8474(0x165)],'stageMetrics':{'strategyName':_0x5e3d58[_0x1b8474(0x16b)],'inputTokens':_0x45e356,'outputTokens':_0x24bc9f[_0x1b8474(0x178)],'savingsRatio':_0x45e356>0x0?0x1-_0x1346c4[_0x1b8474(0x15e)](_0x24bc9f['compressedTokens'],_0x45e356):0x0,'durationMs':_0x2c94f9}};}['buildFinalResult'](_0x4e28a4,_0x435b47,_0x3f79fa,_0x550eb6,_0x36d0c8,_0x772399){const _0x3cf6d0=_0x5386a7,_0x641b53={'jWyuf':function(_0x3ae86a,_0x5c0ba8,_0x598456){return _0x3ae86a(_0x5c0ba8,_0x598456);},'LtTQB':function(_0x3c0fd2,_0xdcb27){return _0x3c0fd2/_0xdcb27;},'zNZSb':function(_0xa058c2,_0x43bb42){return _0xa058c2<=_0x43bb42;}},_0x443fdb=countTokens(_0x3f79fa),_0x263479=_0x641b53[_0x3cf6d0(0x163)](quickQualityScore,_0x4e28a4,_0x3f79fa);return{'text':_0x3f79fa,'originalTokens':_0x435b47,'compressedTokens':_0x443fdb,'compressionRatio':_0x443fdb>0x0?_0x641b53['LtTQB'](_0x435b47,_0x443fdb):0x1,'strategyUsed':_0x550eb6[_0x3cf6d0(0x173)](_0x1f0328=>_0x1f0328[_0x3cf6d0(0x170)])['join']('+'),'qualityEstimate':_0x263479,'wasTruncated':_0x36d0c8,'stages':_0x550eb6,'qualityReport':{'overallScore':_0x263479,'informationRetention':_0x263479,'structuralIntegrity':_0x263479,'keyTermPreservation':_0x263479},'targetMet':_0x641b53[_0x3cf6d0(0x172)](_0x443fdb,_0x772399)};}}export function createDefaultPipeline(_0x1a85f2){return new CompressionPipeline({'strategies':[ruleBasedStrategy,structuralStrategy,tokenBudgetStrategy],'targetTokens':_0x1a85f2,'minQuality':0.8,'stopOnTarget':!0x0});}
package/dist/plugin/compression/quality.js CHANGED
@@ -1 +1 @@
1
- function _0x1dde(){const _0x21a2e3=['165887kDghJE','has','6318MGSyOw','8UVgLWy','54OxMWdf','2WGcuGr','FOijN','TjOok','8937szrTQV','match','LXaII','36145SUNLOw','slice','4751772HOrkvr','size','filter','ymOaX','QTaNl','map','length','overallScore','QkRWb','eDcGs','95084IrAMxf','425005PfLGAO','970qmjnwj','qjZYh','sort','12yUtcoX'];_0x1dde=function(){return _0x21a2e3;};return _0x1dde();}(function(_0x336a68,_0x4ef4f6){const _0x5325da=_0x59b8,_0x2148a5=_0x336a68();while(!![]){try{const _0x2d5a28=parseInt(_0x5325da(0x1f6))/0x1*(-parseInt(_0x5325da(0x1fb))/0x2)+parseInt(_0x5325da(0x1f8))/0x3*(-parseInt(_0x5325da(0x1f5))/0x4)+parseInt(_0x5325da(0x201))/0x5*(-parseInt(_0x5325da(0x1fa))/0x6)+-parseInt(_0x5325da(0x1f1))/0x7*(-parseInt(_0x5325da(0x1f9))/0x8)+parseInt(_0x5325da(0x1fe))/0x9*(-parseInt(_0x5325da(0x1f2))/0xa)+-parseInt(_0x5325da(0x1f0))/0xb+parseInt(_0x5325da(0x203))/0xc;if(_0x2d5a28===_0x4ef4f6)break;else _0x2148a5['push'](_0x2148a5['shift']());}catch(_0x188363){_0x2148a5['push'](_0x2148a5['shift']());}}}(_0x1dde,0x1bf21));function _0x59b8(_0x647bf0,_0x595272){_0x647bf0=_0x647bf0-0x1ea;const _0x1dde2a=_0x1dde();let _0x59b817=_0x1dde2a[_0x647bf0];return _0x59b817;}import{extractContentWords,STOPWORDS}from'./text-utils.js';const INFORMATION_RETENTION_WEIGHT=0.4,STRUCTURAL_INTEGRITY_WEIGHT=0.3,KEY_TERM_PRESERVATION_WEIGHT=0.3,MAX_KEY_TERMS=0xa,STRUCTURAL_MARKERS=[{'pattern':/\n\n/g,'weight':0x1},{'pattern':/^[-*]\s/gm,'weight':0x1},{'pattern':/^#{1,6}\s/gm,'weight':0x1},{'pattern':/```/g,'weight':0x1},{'pattern':/\d{4}[-/.]\d{1,2}[-/.]\d{1,2}/g,'weight':0x2},{'pattern':/\d+[.,]?\d*\s*[€$%]/g,'weight':0x2},{'pattern':/\b\d{3,}\b/g,'weight':1.5}];function getContentTerms(_0x917eaf){const _0x48e8e4=_0x59b8;return new Set(extractContentWords(_0x917eaf)[_0x48e8e4(0x205)](_0x899d6c=>!STOPWORDS['has'](_0x899d6c)));}function measureInformationRetention(_0x11cf71,_0x4bbcf3){const _0xe902f3=_0x59b8,_0x3f3ab6={'TjOok':function(_0x3346ad,_0x54aa8b){return _0x3346ad===_0x54aa8b;},'ymOaX':function(_0x177c3b,_0xb6cc2d){return _0x177c3b(_0xb6cc2d);},'QkRWb':function(_0x239810,_0x1639c6){return _0x239810/_0x1639c6;}},_0x31aec5=getContentTerms(_0x11cf71);if(_0x3f3ab6[_0xe902f3(0x1fd)](0x0,_0x31aec5[_0xe902f3(0x204)]))return 0x1;const _0x407dba=_0x3f3ab6[_0xe902f3(0x206)](getContentTerms,_0x4bbcf3);let _0xfceb7d=0x0;for(const _0x459446 of _0x31aec5)_0x407dba[_0xe902f3(0x1f7)](_0x459446)&&_0xfceb7d++;return _0x3f3ab6[_0xe902f3(0x1ee)](_0xfceb7d,_0x31aec5[_0xe902f3(0x204)]);}function measureStructuralIntegrity(_0x2c760b,_0x538d7e){const _0x364fc9=_0x59b8,_0x29690c={'eDcGs':function(_0x27fd56,_0x1770db){return _0x27fd56>_0x1770db;},'hqAxC':function(_0x164308,_0x30b148){return _0x164308*_0x30b148;},'VMIRA':function(_0x470f5b,_0x3866e8){return _0x470f5b/_0x3866e8;},'LXaII':function(_0x37af19,_0x4622ef){return _0x37af19/_0x4622ef;}};let _0x394a60=0x0,_0x358f3e=0x0;for(const {pattern:_0x5a9804,weight:_0x24a12d}of STRUCTURAL_MARKERS){const _0x550c58=_0x2c760b[_0x364fc9(0x1ff)](_0x5a9804),_0xc72bf6=_0x538d7e[_0x364fc9(0x1ff)](_0x5a9804);if(_0x550c58&&_0x29690c[_0x364fc9(0x1ef)](_0x550c58[_0x364fc9(0x1ec)],0x0)){const _0x41d6a8=_0x550c58[_0x364fc9(0x1ec)],_0x4a4636=_0xc72bf6?.['length']??0x0;_0x394a60+=_0x24a12d,_0x358f3e+=_0x29690c['hqAxC'](_0x24a12d,Math['min'](0x1,_0x29690c['VMIRA'](_0x4a4636,_0x41d6a8)));}}return _0x29690c['eDcGs'](_0x394a60,0x0)?_0x29690c[_0x364fc9(0x200)](_0x358f3e,_0x394a60):0x1;}function measureKeyTermPreservation(_0x1c0141,_0x458250){const _0x120398=_0x59b8,_0x4e7b88={'mIlSa':function(_0x2a89a7,_0x2db350){return _0x2a89a7===_0x2db350;}},_0x5cd7c9=extractContentWords(_0x1c0141)['filter'](_0x576804=>!STOPWORDS[_0x120398(0x1f7)](_0x576804));if(0x0===_0x5cd7c9['length'])return 0x1;const _0x3f9a5a={};for(const _0x77da49 of _0x5cd7c9)_0x3f9a5a[_0x77da49]=(_0x3f9a5a[_0x77da49]||0x0)+0x1;const _0x2b4a52=Object['entries'](_0x3f9a5a)[_0x120398(0x1f4)]((_0x463c52,_0x5cfd37)=>_0x5cfd37[0x1]-_0x463c52[0x1])[_0x120398(0x202)](0x0,0xa)[_0x120398(0x1eb)](([_0x1e07d9])=>_0x1e07d9);if(_0x4e7b88['mIlSa'](0x0,_0x2b4a52['length']))return 0x1;const _0x1f036a=_0x458250['toLowerCase']();let _0x539728=0x0;for(const _0x36495f of _0x2b4a52)_0x1f036a['includes'](_0x36495f)&&_0x539728++;return _0x539728/_0x2b4a52['length'];}export function estimateQuality(_0x7f8db4,_0x3199d2){const _0x38f008=_0x59b8,_0x471cb5={'FOijN':function(_0x7bb73a,_0x4f0a6e,_0x345134){return _0x7bb73a(_0x4f0a6e,_0x345134);},'qjZYh':function(_0x627bd7,_0x20b97d,_0xc387b7){return _0x627bd7(_0x20b97d,_0xc387b7);},'VFiGd':function(_0x5b9920,_0x522552){return _0x5b9920+_0x522552;},'QTaNl':function(_0x1594f8,_0x2a47c5){return _0x1594f8*_0x2a47c5;}},_0x27b2dd=measureInformationRetention(_0x7f8db4,_0x3199d2),_0x1b0b09=_0x471cb5[_0x38f008(0x1fc)](measureStructuralIntegrity,_0x7f8db4,_0x3199d2),_0x3fb61c=_0x471cb5[_0x38f008(0x1f3)](measureKeyTermPreservation,_0x7f8db4,_0x3199d2);return{'overallScore':_0x471cb5['VFiGd'](0.4*_0x27b2dd+0.3*_0x1b0b09,_0x471cb5[_0x38f008(0x1ea)](0.3,_0x3fb61c)),'informationRetention':_0x27b2dd,'structuralIntegrity':_0x1b0b09,'keyTermPreservation':_0x3fb61c};}export function quickQualityScore(_0x44e6af,_0x4dcad7){const _0x5f2422=_0x59b8,_0x1670ef={'EzBAJ':function(_0x3544a8,_0x2a2f30,_0x3e9827){return _0x3544a8(_0x2a2f30,_0x3e9827);}};return _0x1670ef['EzBAJ'](estimateQuality,_0x44e6af,_0x4dcad7)[_0x5f2422(0x1ed)];}
1
+ (function(_0x1528bb,_0xa46dc4){const _0x40eac9=_0x100c,_0x49a2ba=_0x1528bb();while(!![]){try{const _0x404aab=-parseInt(_0x40eac9(0x132))/0x1+parseInt(_0x40eac9(0x128))/0x2+-parseInt(_0x40eac9(0x11e))/0x3+parseInt(_0x40eac9(0x131))/0x4+-parseInt(_0x40eac9(0x11c))/0x5+-parseInt(_0x40eac9(0x122))/0x6+-parseInt(_0x40eac9(0x11a))/0x7*(-parseInt(_0x40eac9(0x12b))/0x8);if(_0x404aab===_0xa46dc4)break;else _0x49a2ba['push'](_0x49a2ba['shift']());}catch(_0x5ec1f2){_0x49a2ba['push'](_0x49a2ba['shift']());}}}(_0x468a,0xed43d));import{extractContentWords,STOPWORDS}from'./text-utils.js';const INFORMATION_RETENTION_WEIGHT=0.4,STRUCTURAL_INTEGRITY_WEIGHT=0.3,KEY_TERM_PRESERVATION_WEIGHT=0.3,MAX_KEY_TERMS=0xa,STRUCTURAL_MARKERS=[{'pattern':/\n\n/g,'weight':0x1},{'pattern':/^[-*]\s/gm,'weight':0x1},{'pattern':/^#{1,6}\s/gm,'weight':0x1},{'pattern':/```/g,'weight':0x1},{'pattern':/\d{4}[-/.]\d{1,2}[-/.]\d{1,2}/g,'weight':0x2},{'pattern':/\d+[.,]?\d*\s*[€$%]/g,'weight':0x2},{'pattern':/\b\d{3,}\b/g,'weight':1.5}];function getContentTerms(_0x1fc745){const _0xa97091=_0x100c,_0x18f51a={'mYFQo':function(_0x4ecff7,_0x5efc3e){return _0x4ecff7(_0x5efc3e);}};return new Set(_0x18f51a[_0xa97091(0x130)](extractContentWords,_0x1fc745)['filter'](_0x3235b9=>!STOPWORDS[_0xa97091(0x11f)](_0x3235b9)));}function measureInformationRetention(_0x34128c,_0x1d1ea5){const _0xcb1c47=_0x100c,_0x3fee2a={'HTxst':function(_0x53f8f5,_0xade641){return _0x53f8f5(_0xade641);},'hzhkH':function(_0x55b480,_0x3c4f4d){return _0x55b480/_0x3c4f4d;}},_0x5496bb=_0x3fee2a[_0xcb1c47(0x12d)](getContentTerms,_0x34128c);if(0x0===_0x5496bb['size'])return 0x1;const _0x13a089=_0x3fee2a['HTxst'](getContentTerms,_0x1d1ea5);let _0x153243=0x0;for(const _0x502a48 of _0x5496bb)_0x13a089['has'](_0x502a48)&&_0x153243++;return _0x3fee2a['hzhkH'](_0x153243,_0x5496bb[_0xcb1c47(0x127)]);}function measureStructuralIntegrity(_0x15cd69,_0x119bac){const _0xc3a81a=_0x100c,_0x433b39={'BJsue':function(_0x324703,_0x15b4bf){return _0x324703>_0x15b4bf;},'dCRiP':function(_0x4415f0,_0x15497c){return _0x4415f0*_0x15497c;}};let _0x25b463=0x0,_0x192f48=0x0;for(const {pattern:_0x6832f2,weight:_0x20ec38}of STRUCTURAL_MARKERS){const _0x29b3a7=_0x15cd69['match'](_0x6832f2),_0xe172e1=_0x119bac[_0xc3a81a(0x12e)](_0x6832f2);if(_0x29b3a7&&_0x433b39[_0xc3a81a(0x124)](_0x29b3a7['length'],0x0)){const _0x104562=_0x29b3a7['length'],_0x3f77de=_0xe172e1?.['length']??0x0;_0x25b463+=_0x20ec38,_0x192f48+=_0x433b39['dCRiP'](_0x20ec38,Math['min'](0x1,_0x3f77de/_0x104562));}}return _0x25b463>0x0?_0x192f48/_0x25b463:0x1;}function _0x468a(){const _0x553a62=['HTxst','match','entries','mYFQo','1397860OEwExE','1484710zQlmJF','182rJrmHQ','rFepx','3593310ylfGcU','filter','5695740oBrckQ','has','length','RlmNa','6111204sTunSt','NFgtn','BJsue','includes','slice','size','3167120UDPhsC','map','MHHEs','1279784lTsYkJ','overallScore'];_0x468a=function(){return _0x553a62;};return _0x468a();}function _0x100c(_0x4dde2f,_0x14280b){_0x4dde2f=_0x4dde2f-0x11a;const _0x468ada=_0x468a();let _0x100c83=_0x468ada[_0x4dde2f];return _0x100c83;}function measureKeyTermPreservation(_0x2516e3,_0x447094){const _0x63f194=_0x100c,_0x1a7c4c={'MHHEs':function(_0x2e9ffe,_0x1f0985){return _0x2e9ffe(_0x1f0985);},'RlmNa':function(_0x40265e,_0x4cf444){return _0x40265e===_0x4cf444;}},_0x34865b=_0x1a7c4c[_0x63f194(0x12a)](extractContentWords,_0x2516e3)[_0x63f194(0x11d)](_0x5d05f1=>!STOPWORDS['has'](_0x5d05f1));if(_0x1a7c4c[_0x63f194(0x121)](0x0,_0x34865b[_0x63f194(0x120)]))return 0x1;const _0x1191ec={};for(const _0x370a32 of _0x34865b)_0x1191ec[_0x370a32]=(_0x1191ec[_0x370a32]||0x0)+0x1;const _0x490a82=Object[_0x63f194(0x12f)](_0x1191ec)['sort']((_0x4f4fed,_0x2002f8)=>_0x2002f8[0x1]-_0x4f4fed[0x1])[_0x63f194(0x126)](0x0,0xa)[_0x63f194(0x129)](([_0x1555b5])=>_0x1555b5);if(_0x1a7c4c[_0x63f194(0x121)](0x0,_0x490a82[_0x63f194(0x120)]))return 0x1;const _0x3bf632=_0x447094['toLowerCase']();let _0x519fb9=0x0;for(const _0x12b5d3 of _0x490a82)_0x3bf632[_0x63f194(0x125)](_0x12b5d3)&&_0x519fb9++;return _0x519fb9/_0x490a82['length'];}export function estimateQuality(_0x4b24d6,_0x6a0f7f){const _0x29201e=_0x100c,_0x159884={'rFepx':function(_0x23820a,_0x5d963d){return _0x23820a+_0x5d963d;},'NFgtn':function(_0xccb274,_0x2aedd1){return _0xccb274*_0x2aedd1;}},_0xf3b1a5=measureInformationRetention(_0x4b24d6,_0x6a0f7f),_0x4db17f=measureStructuralIntegrity(_0x4b24d6,_0x6a0f7f),_0x3e03c2=measureKeyTermPreservation(_0x4b24d6,_0x6a0f7f);return{'overallScore':_0x159884[_0x29201e(0x11b)](_0x159884[_0x29201e(0x123)](0.4,_0xf3b1a5)+0.3*_0x4db17f,_0x159884[_0x29201e(0x123)](0.3,_0x3e03c2)),'informationRetention':_0xf3b1a5,'structuralIntegrity':_0x4db17f,'keyTermPreservation':_0x3e03c2};}export function quickQualityScore(_0x3e4150,_0x4e8121){const _0x51e6eb=_0x100c;return estimateQuality(_0x3e4150,_0x4e8121)[_0x51e6eb(0x12c)];}
package/dist/plugin/compression/rule-based.js CHANGED
@@ -1 +1 @@
1
- (function(_0x5b5af9,_0x53e6ba){const _0x20c8ea=_0x1297,_0x5205dd=_0x5b5af9();while(!![]){try{const _0x2e6c9f=-parseInt(_0x20c8ea(0x1aa))/0x1*(parseInt(_0x20c8ea(0x19d))/0x2)+parseInt(_0x20c8ea(0x1a2))/0x3*(-parseInt(_0x20c8ea(0x1a9))/0x4)+parseInt(_0x20c8ea(0x199))/0x5*(parseInt(_0x20c8ea(0x1a6))/0x6)+-parseInt(_0x20c8ea(0x1a1))/0x7+-parseInt(_0x20c8ea(0x1a7))/0x8*(-parseInt(_0x20c8ea(0x1a3))/0x9)+parseInt(_0x20c8ea(0x19a))/0xa+parseInt(_0x20c8ea(0x1a4))/0xb;if(_0x2e6c9f===_0x53e6ba)break;else _0x5205dd['push'](_0x5205dd['shift']());}catch(_0x107e35){_0x5205dd['push'](_0x5205dd['shift']());}}}(_0x439a,0x71439));import{countTokens}from'../../shared/token-counter.js';function _0x439a(){const _0x36a771=['EDzNz','4182696IAIeeg','7929BbciKP','603bwqaeX','15901963rlSQDy','split','6828QpHiGI','68776CuoXpo','min','1176ozAMyK','353dWeiWM','cdvOA','trim','845wvZIEZ','128450lhgMJo','rule-based','replace','2200EMtKTZ','length','removeStopwords'];_0x439a=function(){return _0x36a771;};return _0x439a();}import{isStopword}from'./text-utils.js';const MIN_WORDS_FOR_STOPWORD_REMOVAL=0x1e,BASE_QUALITY_ESTIMATE=0.95,QUALITY_VARIANCE_WEIGHT=0.05,GREETING_START=/^(Sehr\s+geehrte\s+(?:Damen\s+und\s+Herren|Frau|Herr)[^,]*,?\s*|Dear\s+(?:Sir|Madam|Mr|Mrs|Ms)[^,]*,?\s*|Hi\s*,?\s*|Hallo\s*,?\s*|Guten\s+Tag\s*,?\s*|Hello\s*,?\s*)/i,GREETING_END=/\s*(Mit\s+freundlichen\s+Grüßen|Freundliche\s+Grüße|Viele\s+Grüße|Best\s+regards|Kind\s+regards|Sincerely|Regards|Yours\s+(?:truly|faithfully))\s*[.,\s]*$/i,CONTACT_BLOCK=/\n\s*(?:Telefon|Tel\.?|Phone|Fax|Email|E-Mail|Kundennummer|Kd\.?-?Nr\.?):\s*[^\n]+/gi;function stripBoilerplate(_0x29a043){const _0x2d676e=_0x1297;let _0x3ec314=_0x29a043;return _0x3ec314=_0x3ec314['replace'](GREETING_START,''),_0x3ec314=_0x3ec314['replace'](GREETING_END,''),_0x3ec314=_0x3ec314[_0x2d676e(0x19c)](CONTACT_BLOCK,''),_0x3ec314;}function normalizeWhitespace(_0x2e7d78){const _0x7e2f9b=_0x1297;let _0x528781=_0x2e7d78;return _0x528781=_0x528781['replace'](/ {2,}/g,'\x20'),_0x528781=_0x528781[_0x7e2f9b(0x19c)](/\n{3,}/g,'\x0a\x0a'),_0x528781=_0x528781[_0x7e2f9b(0x19c)](/[ \t]+$/gm,''),_0x528781[_0x7e2f9b(0x198)]();}function removeStopwordsFromText(_0x115a14){const _0x52d142=_0x1297,_0x161ccf={'cdvOA':function(_0x169573,_0x3171f2){return _0x169573<=_0x3171f2;}},_0x2273f0=_0x115a14[_0x52d142(0x1a5)](/\s+/);if(_0x161ccf[_0x52d142(0x1ab)](_0x2273f0[_0x52d142(0x19e)],0x1e))return _0x115a14;return _0x2273f0['filter'](_0x4d8002=>!isStopword(_0x4d8002))['join']('\x20');}export function compressRuleBased(_0xae019b,_0x1bae4c={}){const _0x34b7bf=_0x1297,_0xbf2de={'EDzNz':_0x34b7bf(0x19b),'krjhJ':function(_0x2b4154,_0x46b308){return _0x2b4154(_0x46b308);},'dsehq':function(_0x1130b9,_0x35c55d){return _0x1130b9(_0x35c55d);},'hVBGB':function(_0x559aa0,_0x3738a4){return _0x559aa0/_0x3738a4;}};if(!_0xae019b||!_0xae019b[_0x34b7bf(0x198)]())return{'text':'','originalTokens':0x0,'compressedTokens':0x0,'compressionRatio':0x1,'strategyUsed':_0xbf2de[_0x34b7bf(0x1a0)],'qualityEstimate':0x1,'wasTruncated':!0x1};const _0x24ebca=_0xbf2de['krjhJ'](countTokens,_0xae019b);let _0x2c88ec=_0xbf2de['dsehq'](stripBoilerplate,_0xae019b['trim']());_0x2c88ec=normalizeWhitespace(_0x2c88ec),(_0x1bae4c[_0x34b7bf(0x19f)]??!0x0)&&(_0x2c88ec=removeStopwordsFromText(_0x2c88ec));const _0x5481b4=countTokens(_0x2c88ec);return{'text':_0x2c88ec,'originalTokens':_0x24ebca,'compressedTokens':_0x5481b4,'compressionRatio':_0x5481b4>0x0?_0xbf2de['hVBGB'](_0x24ebca,_0x5481b4):0x1,'strategyUsed':_0xbf2de['EDzNz'],'qualityEstimate':Math[_0x34b7bf(0x1a8)](0x1,0.95+_0x5481b4/_0x24ebca*0.05),'wasTruncated':!0x1};}function _0x1297(_0x298d4c,_0x1a8836){_0x298d4c=_0x298d4c-0x198;const _0x439a82=_0x439a();let _0x1297a8=_0x439a82[_0x298d4c];return _0x1297a8;}export const ruleBasedStrategy={'name':'rule-based','compress':compressRuleBased};
1
+ (function(_0x4645aa,_0x43b2cb){const _0x46bf4c=_0x4d0f,_0x2a57b4=_0x4645aa();while(!![]){try{const _0x44af2f=-parseInt(_0x46bf4c(0xa3))/0x1+-parseInt(_0x46bf4c(0xab))/0x2*(parseInt(_0x46bf4c(0x9c))/0x3)+parseInt(_0x46bf4c(0x9e))/0x4+parseInt(_0x46bf4c(0xac))/0x5+-parseInt(_0x46bf4c(0x9f))/0x6+-parseInt(_0x46bf4c(0xa7))/0x7+parseInt(_0x46bf4c(0xa1))/0x8*(parseInt(_0x46bf4c(0x99))/0x9);if(_0x44af2f===_0x43b2cb)break;else _0x2a57b4['push'](_0x2a57b4['shift']());}catch(_0x44b170){_0x2a57b4['push'](_0x2a57b4['shift']());}}}(_0x4281,0xbbd86));import{countTokens}from'../../shared/token-counter.js';import{isStopword}from'./text-utils.js';const MIN_WORDS_FOR_STOPWORD_REMOVAL=0x1e,BASE_QUALITY_ESTIMATE=0.95,QUALITY_VARIANCE_WEIGHT=0.05,GREETING_START=/^(Sehr\s+geehrte\s+(?:Damen\s+und\s+Herren|Frau|Herr)[^,]*,?\s*|Dear\s+(?:Sir|Madam|Mr|Mrs|Ms)[^,]*,?\s*|Hi\s*,?\s*|Hallo\s*,?\s*|Guten\s+Tag\s*,?\s*|Hello\s*,?\s*)/i,GREETING_END=/\s*(Mit\s+freundlichen\s+Grüßen|Freundliche\s+Grüße|Viele\s+Grüße|Best\s+regards|Kind\s+regards|Sincerely|Regards|Yours\s+(?:truly|faithfully))\s*[.,\s]*$/i,CONTACT_BLOCK=/\n\s*(?:Telefon|Tel\.?|Phone|Fax|Email|E-Mail|Kundennummer|Kd\.?-?Nr\.?):\s*[^\n]+/gi;function stripBoilerplate(_0x1a677d){const _0x28d452=_0x4d0f;let _0x33c35b=_0x1a677d;return _0x33c35b=_0x33c35b[_0x28d452(0xa6)](GREETING_START,''),_0x33c35b=_0x33c35b[_0x28d452(0xa6)](GREETING_END,''),_0x33c35b=_0x33c35b[_0x28d452(0xa6)](CONTACT_BLOCK,''),_0x33c35b;}function _0x4d0f(_0x9b1f6,_0x1626b2){_0x9b1f6=_0x9b1f6-0x99;const _0x428151=_0x4281();let _0x4d0f9a=_0x428151[_0x9b1f6];return _0x4d0f9a;}function normalizeWhitespace(_0x3c2f3a){const _0x2c5573=_0x4d0f;let _0x49b4d8=_0x3c2f3a;return _0x49b4d8=_0x49b4d8[_0x2c5573(0xa6)](/ {2,}/g,'\x20'),_0x49b4d8=_0x49b4d8[_0x2c5573(0xa6)](/\n{3,}/g,'\x0a\x0a'),_0x49b4d8=_0x49b4d8['replace'](/[ \t]+$/gm,''),_0x49b4d8['trim']();}function removeStopwordsFromText(_0x5edb6a){const _0xf5f59e=_0x4d0f,_0xfaa07d={'Wyjwg':function(_0xee6fe4,_0x49eeb1){return _0xee6fe4<=_0x49eeb1;}},_0x262326=_0x5edb6a[_0xf5f59e(0xa2)](/\s+/);if(_0xfaa07d[_0xf5f59e(0xaa)](_0x262326[_0xf5f59e(0x9a)],0x1e))return _0x5edb6a;return _0x262326['filter'](_0x44d6de=>!isStopword(_0x44d6de))[_0xf5f59e(0x9d)]('\x20');}function _0x4281(){const _0x37e9d6=['min','Wyjwg','17392kUtBbl','1677150BIVmwp','3708891nIcNdm','length','rule-based','69JPeKZs','join','5514232fOvqGd','8668608QjiglP','NmbXf','40BZITbx','split','745812wgZJYY','cQyeu','trim','replace','4301367HojvRl','hGDcK'];_0x4281=function(){return _0x37e9d6;};return _0x4281();}export function compressRuleBased(_0x5b804f,_0x23f214={}){const _0x4d3b73=_0x4d0f,_0xd75256={'fNAhE':_0x4d3b73(0x9b),'NmbXf':function(_0x100219,_0x3c6344){return _0x100219(_0x3c6344);},'GFFbB':function(_0x570d11,_0x4d6f64){return _0x570d11(_0x4d6f64);},'HBldF':function(_0x440032,_0xf9fcfb){return _0x440032>_0xf9fcfb;},'hGDcK':function(_0x179bdd,_0x514d7a){return _0x179bdd+_0x514d7a;},'cQyeu':function(_0x5f5324,_0x3cd243){return _0x5f5324/_0x3cd243;}};if(!_0x5b804f||!_0x5b804f[_0x4d3b73(0xa5)]())return{'text':'','originalTokens':0x0,'compressedTokens':0x0,'compressionRatio':0x1,'strategyUsed':_0xd75256['fNAhE'],'qualityEstimate':0x1,'wasTruncated':!0x1};const _0x3547ce=countTokens(_0x5b804f);let _0x9f76b0=_0xd75256[_0x4d3b73(0xa0)](stripBoilerplate,_0x5b804f[_0x4d3b73(0xa5)]());_0x9f76b0=normalizeWhitespace(_0x9f76b0),(_0x23f214['removeStopwords']??!0x0)&&(_0x9f76b0=removeStopwordsFromText(_0x9f76b0));const _0x34067a=_0xd75256['GFFbB'](countTokens,_0x9f76b0);return{'text':_0x9f76b0,'originalTokens':_0x3547ce,'compressedTokens':_0x34067a,'compressionRatio':_0xd75256['HBldF'](_0x34067a,0x0)?_0x3547ce/_0x34067a:0x1,'strategyUsed':'rule-based','qualityEstimate':Math[_0x4d3b73(0xa9)](0x1,_0xd75256[_0x4d3b73(0xa8)](0.95,_0xd75256[_0x4d3b73(0xa4)](_0x34067a,_0x3547ce)*0.05)),'wasTruncated':!0x1};}export const ruleBasedStrategy={'name':'rule-based','compress':compressRuleBased};
package/dist/plugin/compression/structural.js CHANGED
@@ -1 +1 @@
1
- (function(_0x812c16,_0x2920e3){const _0x29deb5=_0x5b4c,_0x2b0957=_0x812c16();while(!![]){try{const _0x1d879a=-parseInt(_0x29deb5(0x70))/0x1+-parseInt(_0x29deb5(0x6d))/0x2+-parseInt(_0x29deb5(0x65))/0x3*(parseInt(_0x29deb5(0x83))/0x4)+-parseInt(_0x29deb5(0x72))/0x5*(-parseInt(_0x29deb5(0x75))/0x6)+-parseInt(_0x29deb5(0x79))/0x7*(parseInt(_0x29deb5(0x7d))/0x8)+parseInt(_0x29deb5(0x73))/0x9+parseInt(_0x29deb5(0x6a))/0xa;if(_0x1d879a===_0x2920e3)break;else _0x2b0957['push'](_0x2b0957['shift']());}catch(_0x4bf599){_0x2b0957['push'](_0x2b0957['shift']());}}}(_0x1f00,0xcb5f3));import{countTokens}from'../../shared/token-counter.js';function _0x1f00(){const _0x345bb1=['floor','724ccIXiq','HRVFE','22482VVwyBc','has','zdzRt','entries','score','42349780kAnLGL','sqrt','log','1920122OJxBtI','maxTokens','map','1250969rELpBa','kKrHr','20NnxcoA','7113519IEJzUQ','index','761328czRScS','GhftF','length','sentence','14oNMEax','jGYEQ','IROfU','sort','4529864GPuIlf','structural','min','SCyOf','push'];_0x1f00=function(){return _0x345bb1;};return _0x1f00();}import{splitSentences,extractContentWords,STOPWORDS}from'./text-utils.js';const DEFAULT_TARGET_RATIO=0.45,MIN_EXTRACTIVE_SENTENCES=0x2,MIN_TARGET_TOKENS=0x32,BASE_QUALITY_ESTIMATE=0.85,QUALITY_VARIANCE_WEIGHT=0.15;function computeDocumentFrequency(_0x2bf3c8){const _0x1722ca=_0x5b4c,_0xf19b63={'iYBFb':function(_0x5e6db0,_0x598b71){return _0x5e6db0+_0x598b71;}},_0xf43a72={};for(const _0x5f06ec of _0x2bf3c8){const _0x4082ec=new Set(_0x5f06ec);for(const _0x2eb8f9 of _0x4082ec)STOPWORDS[_0x1722ca(0x66)](_0x2eb8f9)||(_0xf43a72[_0x2eb8f9]=_0xf19b63['iYBFb'](_0xf43a72[_0x2eb8f9]||0x0,0x1));}return _0xf43a72;}function computeTermFrequency(_0x423ded){const _0x384174=_0x5b4c,_0x49f550={};for(const _0x68934e of _0x423ded)STOPWORDS[_0x384174(0x66)](_0x68934e)||(_0x49f550[_0x68934e]=(_0x49f550[_0x68934e]||0x0)+0x1);return _0x49f550;}function buildTfIdfVector(_0xecbba2,_0x74091f,_0x18a411){const _0x2794a6=_0x5b4c,_0x472138={'GAaQP':function(_0x446b44,_0x2afdf1){return _0x446b44+_0x2afdf1;}},_0x13fdd8={};for(const [_0x2fb49f,_0x3dd5dc]of Object['entries'](_0xecbba2)){const _0x543594=Math[_0x2794a6(0x6c)](_0x472138['GAaQP'](_0x18a411,0x1)/((_0x74091f[_0x2fb49f]||0x0)+0x1));_0x13fdd8[_0x2fb49f]=_0x3dd5dc*_0x543594;}return _0x13fdd8;}function _0x5b4c(_0x1c1ad3,_0xfbaf51){_0x1c1ad3=_0x1c1ad3-0x65;const _0x1f000e=_0x1f00();let _0x5b4cc0=_0x1f000e[_0x1c1ad3];return _0x5b4cc0;}function buildTfIdf(_0x106531){const _0x47d323=_0x5b4c,_0x49c6bb={'HRVFE':function(_0x2dd62e,_0x1d6159){return _0x2dd62e(_0x1d6159);}},_0x32599c=_0x106531['length'],_0x162ea7=_0x106531['map'](extractContentWords),_0x45ee35=_0x49c6bb[_0x47d323(0x84)](computeDocumentFrequency,_0x162ea7);return _0x162ea7['map'](_0x29a82a=>buildTfIdfVector(computeTermFrequency(_0x29a82a),_0x45ee35,_0x32599c));}function computeCentroid(_0x2ece8a){const _0xdaf74f=_0x5b4c,_0x414e01={'sdCzZ':function(_0x1d6373,_0x277417){return _0x1d6373+_0x277417;},'Iuvqj':function(_0x3c0022,_0x53310d){return _0x3c0022/_0x53310d;}},_0x17a2df={},_0x237610=_0x2ece8a['length'];for(const _0x5c1e3b of _0x2ece8a)for(const [_0xb62ec5,_0x4b3647]of Object[_0xdaf74f(0x68)](_0x5c1e3b))_0x17a2df[_0xb62ec5]=_0x414e01['sdCzZ'](_0x17a2df[_0xb62ec5]||0x0,_0x414e01['Iuvqj'](_0x4b3647,_0x237610));return _0x17a2df;}function cosineSimilarity(_0x5b4d6f,_0x371d9d){const _0x186be6=_0x5b4c,_0x3fb5ee={'UcUgM':function(_0x2add8c,_0x46be18){return _0x2add8c*_0x46be18;},'kKrHr':function(_0x53ea39,_0x9692a5){return _0x53ea39*_0x9692a5;},'zdzRt':function(_0x4062c5,_0x5f46ce){return _0x4062c5/_0x5f46ce;}},_0x5d5d45=new Set([...Object['keys'](_0x5b4d6f),...Object['keys'](_0x371d9d)]);let _0x4295ba=0x0,_0x3f5014=0x0,_0x43371f=0x0;for(const _0x2d0e6f of _0x5d5d45){const _0x2e6a43=_0x5b4d6f[_0x2d0e6f]||0x0,_0x72bd9f=_0x371d9d[_0x2d0e6f]||0x0;_0x4295ba+=_0x3fb5ee['UcUgM'](_0x2e6a43,_0x72bd9f),_0x3f5014+=_0x2e6a43*_0x2e6a43,_0x43371f+=_0x3fb5ee[_0x186be6(0x71)](_0x72bd9f,_0x72bd9f);}const _0x583de8=_0x3fb5ee['kKrHr'](Math[_0x186be6(0x6b)](_0x3f5014),Math['sqrt'](_0x43371f));return _0x583de8>0x0?_0x3fb5ee[_0x186be6(0x67)](_0x4295ba,_0x583de8):0x0;}function selectTopSentences(_0x588af1,_0x1892a4){const _0x432b54=_0x5b4c,_0x13b528={'jGYEQ':function(_0x34cca4,_0x4dc1a8){return _0x34cca4(_0x4dc1a8);},'GhftF':function(_0x585cc8,_0x1f1870){return _0x585cc8+_0x1f1870;},'VejlL':function(_0xf3d401,_0x3b4a3e){return _0xf3d401<_0x3b4a3e;}},_0x416f0b=[];let _0x354b58=0x0;for(const _0x1fbdd6 of _0x588af1){const _0xfd301b=_0x13b528[_0x432b54(0x7a)](countTokens,_0x1fbdd6['sentence']);if(!(_0x13b528[_0x432b54(0x76)](_0x354b58,_0xfd301b)<=_0x1892a4||_0x13b528['VejlL'](_0x416f0b[_0x432b54(0x77)],0x2)))break;_0x416f0b[_0x432b54(0x81)]({'index':_0x1fbdd6['index'],'sentence':_0x1fbdd6[_0x432b54(0x78)]}),_0x354b58+=_0x13b528[_0x432b54(0x76)](_0xfd301b,0x1);}return _0x416f0b;}function buildEmptyResult(_0x388ba7){return{'text':'','originalTokens':0x0,'compressedTokens':0x0,'compressionRatio':0x1,'strategyUsed':_0x388ba7,'qualityEstimate':0x1,'wasTruncated':!0x1};}export function compressStructural(_0x536ffd,_0x28754b={}){const _0x2e9b8f=_0x5b4c,_0x5682c8={'jzjgr':function(_0x2195c9,_0xe44154){return _0x2195c9(_0xe44154);},'poMEV':function(_0x2a4371,_0x10f008){return _0x2a4371(_0x10f008);},'IROfU':'structural','hvTHv':function(_0x5bbc78,_0x26febe){return _0x5bbc78*_0x26febe;},'xtnZC':function(_0x10648c,_0x3b40cc){return _0x10648c>_0x3b40cc;},'oRdtV':function(_0x48aed5,_0x204826){return _0x48aed5/_0x204826;},'ONlQb':function(_0x86931a,_0x54ee48){return _0x86931a+_0x54ee48;},'SCyOf':function(_0x1275e5,_0x358af8){return _0x1275e5/_0x358af8;}};if(!_0x536ffd||!_0x536ffd['trim']())return _0x5682c8['jzjgr'](buildEmptyResult,'structural');const _0x540ec3=_0x5682c8['poMEV'](countTokens,_0x536ffd),_0x2ab140=_0x28754b['targetRatio']??0.45,_0x4e87f9=splitSentences(_0x536ffd);if(_0x4e87f9['length']<=0x2)return{'text':_0x536ffd,'originalTokens':_0x540ec3,'compressedTokens':_0x540ec3,'compressionRatio':0x1,'strategyUsed':_0x5682c8[_0x2e9b8f(0x7b)],'qualityEstimate':0x1,'wasTruncated':!0x1};const _0x213619=buildTfIdf(_0x4e87f9),_0x5ee87b=computeCentroid(_0x213619),_0x23fa77=_0x4e87f9[_0x2e9b8f(0x6f)]((_0x1f6c1d,_0x307db5)=>({'index':_0x307db5,'sentence':_0x1f6c1d,'score':cosineSimilarity(_0x213619[_0x307db5],_0x5ee87b)}));_0x23fa77[_0x2e9b8f(0x7c)]((_0x432d72,_0x139874)=>_0x139874[_0x2e9b8f(0x69)]-_0x432d72[_0x2e9b8f(0x69)]);const _0x4445f6=selectTopSentences(_0x23fa77,_0x28754b[_0x2e9b8f(0x6e)]?Math['min'](_0x28754b[_0x2e9b8f(0x6e)],Math['floor'](_0x5682c8['hvTHv'](_0x540ec3,_0x2ab140))):Math['max'](0x32,Math[_0x2e9b8f(0x82)](_0x5682c8['hvTHv'](_0x540ec3,_0x2ab140))));_0x4445f6[_0x2e9b8f(0x7c)]((_0x5ee4fa,_0x56c678)=>_0x5ee4fa['index']-_0x56c678[_0x2e9b8f(0x74)]);const _0x23a9d6=_0x4445f6[_0x2e9b8f(0x6f)](_0x14626e=>_0x14626e['sentence'])['join']('.\x20'),_0x5679a1=countTokens(_0x23a9d6);return{'text':_0x23a9d6,'originalTokens':_0x540ec3,'compressedTokens':_0x5679a1,'compressionRatio':_0x5682c8['xtnZC'](_0x5679a1,0x0)?_0x5682c8['oRdtV'](_0x540ec3,_0x5679a1):0x1,'strategyUsed':_0x2e9b8f(0x7e),'qualityEstimate':Math[_0x2e9b8f(0x7f)](0x1,_0x5682c8['ONlQb'](0.85,_0x5682c8[_0x2e9b8f(0x80)](_0x5679a1,_0x540ec3)*0.15)),'wasTruncated':!0x0};}export const structuralStrategy={'name':'structural','compress':compressStructural};
1
+ const _0x25bd93=_0x38a8;(function(_0x27d60a,_0x41c1cb){const _0x1edb68=_0x38a8,_0x4bd41d=_0x27d60a();while(!![]){try{const _0x33638f=parseInt(_0x1edb68(0x1a3))/0x1+-parseInt(_0x1edb68(0x19f))/0x2+-parseInt(_0x1edb68(0x197))/0x3*(parseInt(_0x1edb68(0x1a6))/0x4)+-parseInt(_0x1edb68(0x199))/0x5+parseInt(_0x1edb68(0x1ad))/0x6+parseInt(_0x1edb68(0x194))/0x7+parseInt(_0x1edb68(0x1a8))/0x8;if(_0x33638f===_0x41c1cb)break;else _0x4bd41d['push'](_0x4bd41d['shift']());}catch(_0x141d71){_0x4bd41d['push'](_0x4bd41d['shift']());}}}(_0x4fc3,0x674f6));import{countTokens}from'../../shared/token-counter.js';import{splitSentences,extractContentWords,STOPWORDS}from'./text-utils.js';const DEFAULT_TARGET_RATIO=0.45,MIN_EXTRACTIVE_SENTENCES=0x2,MIN_TARGET_TOKENS=0x32,BASE_QUALITY_ESTIMATE=0.85,QUALITY_VARIANCE_WEIGHT=0.15;function computeDocumentFrequency(_0x35d12b){const _0x260a19={'YSMKi':function(_0x1926cc,_0x1db19b){return _0x1926cc+_0x1db19b;}},_0x2ddd24={};for(const _0x1e94df of _0x35d12b){const _0x5cf33e=new Set(_0x1e94df);for(const _0x563faf of _0x5cf33e)STOPWORDS['has'](_0x563faf)||(_0x2ddd24[_0x563faf]=_0x260a19['YSMKi'](_0x2ddd24[_0x563faf]||0x0,0x1));}return _0x2ddd24;}function computeTermFrequency(_0x27b3bf){const _0x39e615={};for(const _0x166175 of _0x27b3bf)STOPWORDS['has'](_0x166175)||(_0x39e615[_0x166175]=(_0x39e615[_0x166175]||0x0)+0x1);return _0x39e615;}function buildTfIdfVector(_0x361406,_0x45cad2,_0x81b5e7){const _0x292207=_0x38a8,_0x2acd9a={'biSuN':function(_0x5506e4,_0x14269c){return _0x5506e4+_0x14269c;}},_0x517fdf={};for(const [_0x4644e4,_0x4a436b]of Object[_0x292207(0x1a5)](_0x361406)){const _0x76a49c=Math[_0x292207(0x19c)](_0x2acd9a[_0x292207(0x198)](_0x81b5e7,0x1)/((_0x45cad2[_0x4644e4]||0x0)+0x1));_0x517fdf[_0x4644e4]=_0x4a436b*_0x76a49c;}return _0x517fdf;}function _0x38a8(_0x220e89,_0x52c70e){_0x220e89=_0x220e89-0x18c;const _0x4fc375=_0x4fc3();let _0x38a849=_0x4fc375[_0x220e89];return _0x38a849;}function buildTfIdf(_0x3c7a1e){const _0x18205f=_0x38a8,_0x3f8537={'RKYrD':function(_0x269709,_0xdc72c6){return _0x269709(_0xdc72c6);}},_0x21bcba=_0x3c7a1e['length'],_0x1ea6b5=_0x3c7a1e['map'](extractContentWords),_0x331321=_0x3f8537[_0x18205f(0x193)](computeDocumentFrequency,_0x1ea6b5);return _0x1ea6b5[_0x18205f(0x1a2)](_0x49326c=>buildTfIdfVector(computeTermFrequency(_0x49326c),_0x331321,_0x21bcba));}function computeCentroid(_0x36edda){const _0x29f1ec=_0x38a8,_0x49d4ad={'wHshW':function(_0x381cf9,_0x376760){return _0x381cf9/_0x376760;}},_0x1d1def={},_0x653a02=_0x36edda[_0x29f1ec(0x1ac)];for(const _0x79cdee of _0x36edda)for(const [_0x96805b,_0x57df8a]of Object['entries'](_0x79cdee))_0x1d1def[_0x96805b]=(_0x1d1def[_0x96805b]||0x0)+_0x49d4ad['wHshW'](_0x57df8a,_0x653a02);return _0x1d1def;}function cosineSimilarity(_0xfbe740,_0x55afa9){const _0x50e522=_0x38a8,_0x38a199={'mqCon':function(_0x327ab0,_0x29a6ac){return _0x327ab0*_0x29a6ac;},'LPNCD':function(_0x8dab1f,_0x25ebcb){return _0x8dab1f*_0x25ebcb;}},_0x24713a=new Set([...Object['keys'](_0xfbe740),...Object['keys'](_0x55afa9)]);let _0x4015e0=0x0,_0x1e3361=0x0,_0x298444=0x0;for(const _0x12ad00 of _0x24713a){const _0x4e68e1=_0xfbe740[_0x12ad00]||0x0,_0xe6d218=_0x55afa9[_0x12ad00]||0x0;_0x4015e0+=_0x38a199[_0x50e522(0x18e)](_0x4e68e1,_0xe6d218),_0x1e3361+=_0x4e68e1*_0x4e68e1,_0x298444+=_0x38a199[_0x50e522(0x190)](_0xe6d218,_0xe6d218);}const _0x52e5d9=Math[_0x50e522(0x19e)](_0x1e3361)*Math[_0x50e522(0x19e)](_0x298444);return _0x52e5d9>0x0?_0x4015e0/_0x52e5d9:0x0;}function selectTopSentences(_0xa47751,_0x3990b9){const _0x243bfb=_0x38a8,_0x281344={'Zpgqv':function(_0x594dc7,_0x282707){return _0x594dc7(_0x282707);},'YCgRL':function(_0x600ea8,_0x471219){return _0x600ea8<=_0x471219;},'uRxCa':function(_0x4ae2ea,_0x35a540){return _0x4ae2ea+_0x35a540;}},_0x57ca69=[];let _0x7b47b7=0x0;for(const _0x3a4911 of _0xa47751){const _0x32d41c=_0x281344['Zpgqv'](countTokens,_0x3a4911[_0x243bfb(0x18d)]);if(!(_0x281344['YCgRL'](_0x281344[_0x243bfb(0x1a0)](_0x7b47b7,_0x32d41c),_0x3990b9)||_0x57ca69['length']<0x2))break;_0x57ca69[_0x243bfb(0x18f)]({'index':_0x3a4911['index'],'sentence':_0x3a4911[_0x243bfb(0x18d)]}),_0x7b47b7+=_0x281344[_0x243bfb(0x1a0)](_0x32d41c,0x1);}return _0x57ca69;}function _0x4fc3(){const _0x5c0a9f=['index','map','8643RFkaPD','zpAit','entries','4684NCkmgu','score','11359008SnODFF','join','maxTokens','twcYu','length','1043322pYbdkX','sort','sentence','mqCon','push','LPNCD','targetRatio','max','RKYrD','3610341HeQuIE','structural','floor','1959ERuMNF','biSuN','1809840tyViSq','trim','xKyNH','log','min','sqrt','1136760lcsXGL','uRxCa'];_0x4fc3=function(){return _0x5c0a9f;};return _0x4fc3();}function buildEmptyResult(_0x54d795){return{'text':'','originalTokens':0x0,'compressedTokens':0x0,'compressionRatio':0x1,'strategyUsed':_0x54d795,'qualityEstimate':0x1,'wasTruncated':!0x1};}export function compressStructural(_0x35c354,_0x1aa03a={}){const _0x10c100=_0x38a8,_0x1beba7={'vEhwc':function(_0x3d6629,_0xe15543){return _0x3d6629<=_0xe15543;},'zpAit':'structural','xKyNH':function(_0x1d727f,_0x23be1c){return _0x1d727f(_0x23be1c);},'twcYu':function(_0x403d7d,_0x43fe42,_0x5f204f){return _0x403d7d(_0x43fe42,_0x5f204f);},'tpMIf':function(_0x26983e,_0x40779d){return _0x26983e(_0x40779d);},'qcBxx':function(_0x3fe971,_0xb1f691){return _0x3fe971+_0xb1f691;}};if(!_0x35c354||!_0x35c354[_0x10c100(0x19a)]())return buildEmptyResult(_0x10c100(0x195));const _0x30f811=countTokens(_0x35c354),_0x3f13a7=_0x1aa03a[_0x10c100(0x191)]??0.45,_0x20fa04=splitSentences(_0x35c354);if(_0x1beba7['vEhwc'](_0x20fa04['length'],0x2))return{'text':_0x35c354,'originalTokens':_0x30f811,'compressedTokens':_0x30f811,'compressionRatio':0x1,'strategyUsed':_0x1beba7[_0x10c100(0x1a4)],'qualityEstimate':0x1,'wasTruncated':!0x1};const _0x104a5d=_0x1beba7[_0x10c100(0x19b)](buildTfIdf,_0x20fa04),_0x110aa7=_0x1beba7[_0x10c100(0x19b)](computeCentroid,_0x104a5d),_0x312f58=_0x20fa04['map']((_0xb3a559,_0x3bee6f)=>({'index':_0x3bee6f,'sentence':_0xb3a559,'score':cosineSimilarity(_0x104a5d[_0x3bee6f],_0x110aa7)}));_0x312f58[_0x10c100(0x18c)]((_0x3f53fd,_0x4ebe0a)=>_0x4ebe0a['score']-_0x3f53fd[_0x10c100(0x1a7)]);const _0x42faa7=_0x1beba7[_0x10c100(0x1ab)](selectTopSentences,_0x312f58,_0x1aa03a[_0x10c100(0x1aa)]?Math[_0x10c100(0x19d)](_0x1aa03a[_0x10c100(0x1aa)],Math[_0x10c100(0x196)](_0x30f811*_0x3f13a7)):Math[_0x10c100(0x192)](0x32,Math[_0x10c100(0x196)](_0x30f811*_0x3f13a7)));_0x42faa7['sort']((_0x27c5e7,_0x186ccd)=>_0x27c5e7['index']-_0x186ccd[_0x10c100(0x1a1)]);const _0x1ebe2c=_0x42faa7[_0x10c100(0x1a2)](_0x290940=>_0x290940[_0x10c100(0x18d)])[_0x10c100(0x1a9)]('.\x20'),_0x39b803=_0x1beba7['tpMIf'](countTokens,_0x1ebe2c);return{'text':_0x1ebe2c,'originalTokens':_0x30f811,'compressedTokens':_0x39b803,'compressionRatio':_0x39b803>0x0?_0x30f811/_0x39b803:0x1,'strategyUsed':_0x1beba7[_0x10c100(0x1a4)],'qualityEstimate':Math['min'](0x1,_0x1beba7['qcBxx'](0.85,_0x39b803/_0x30f811*0.15)),'wasTruncated':!0x0};}export const structuralStrategy={'name':_0x25bd93(0x195),'compress':compressStructural};
package/dist/plugin/compression/text-utils.js CHANGED
@@ -1 +1 @@
1
- const _0x51adf3=_0x310c;(function(_0x5b2979,_0x1f0b1f){const _0x28d6bd=_0x310c,_0x1ad8ad=_0x5b2979();while(!![]){try{const _0x3e497e=parseInt(_0x28d6bd(0x18c))/0x1*(-parseInt(_0x28d6bd(0x1a7))/0x2)+parseInt(_0x28d6bd(0x182))/0x3+parseInt(_0x28d6bd(0x180))/0x4*(-parseInt(_0x28d6bd(0x17f))/0x5)+-parseInt(_0x28d6bd(0x189))/0x6*(parseInt(_0x28d6bd(0x171))/0x7)+-parseInt(_0x28d6bd(0x193))/0x8+-parseInt(_0x28d6bd(0x1a2))/0x9+parseInt(_0x28d6bd(0x165))/0xa;if(_0x3e497e===_0x1f0b1f)break;else _0x1ad8ad['push'](_0x1ad8ad['shift']());}catch(_0x1be6f2){_0x1ad8ad['push'](_0x1ad8ad['shift']());}}}(_0x3f8b,0xa17e8));function _0x3f8b(){const _0x19cb83=['should','von','length','ca.','will','7LOphhF','muss','vs.','müssen','hat','über','trim','des','oder','ggf.','ein','v.a.','e.g.','i.e.','642830Fzbqwk','24mqDUsK','d.h.','1990014LGNHwK','das','does','haben','bei','inkl.','there','5282364cgwqVh','may','filter','158889YYiOCs','Dr.','could','Ms.','they','replace','wollen','975944DsgSrO','were','nur','u.U.','z.B.','bzw.','evtl.','from','had','auch','toLowerCase','can','split','sind','und','8147844WxAKwr','ich','dass','how','with','2hwEPMb','i.d.R.','noch','about','into','wird','etc.','here','\x00PUNKT\x00','28361300TqStgR','those','Nr.','eine','replaceAll','waren','map'];_0x3f8b=function(){return _0x19cb83;};return _0x3f8b();}const ABBREVIATIONS=[_0x51adf3(0x167),_0x51adf3(0x197),_0x51adf3(0x198),'u.a.',_0x51adf3(0x162),_0x51adf3(0x199),_0x51adf3(0x18d),'Prof.',_0x51adf3(0x16f),_0x51adf3(0x17a),_0x51adf3(0x187),_0x51adf3(0x1a8),'o.ä.',_0x51adf3(0x196),_0x51adf3(0x17c),_0x51adf3(0x181),_0x51adf3(0x17d),_0x51adf3(0x17e),_0x51adf3(0x173),'Mr.','Mrs.',_0x51adf3(0x18f),'Jr.','Sr.'],PLACEHOLDER='\x00PUNKT\x00';export function splitSentences(_0x32c872){const _0x20dbfa=_0x51adf3,_0x11c4b4={'oNknH':_0x20dbfa(0x164)};let _0x542fe7=_0x32c872[_0x20dbfa(0x191)](/\s+/g,'\x20')['trim']();for(const _0x1eb56f of ABBREVIATIONS)_0x542fe7=_0x542fe7['replaceAll'](_0x1eb56f,_0x1eb56f[_0x20dbfa(0x169)]('.',_0x11c4b4['oNknH']));return _0x542fe7[_0x20dbfa(0x19f)](/[.!?]\s+|\n+/)[_0x20dbfa(0x16b)](_0x310770=>_0x310770[_0x20dbfa(0x169)](_0x20dbfa(0x164),'.')[_0x20dbfa(0x177)]())[_0x20dbfa(0x18b)](_0x292b8b=>_0x292b8b[_0x20dbfa(0x16e)]>0x0);}function _0x310c(_0x185480,_0x2fd0f0){_0x185480=_0x185480-0x15e;const _0x3f8b7f=_0x3f8b();let _0x310ca3=_0x3f8b7f[_0x185480];return _0x310ca3;}export function extractContentWords(_0x3b486a){const _0x1dc126=_0x51adf3;return _0x3b486a[_0x1dc126(0x19d)]()[_0x1dc126(0x191)](/[^a-zäöüß\s]/g,'')[_0x1dc126(0x19f)](/\s+/)['filter'](_0x567fd8=>_0x567fd8['length']>=0x3);}export const STOPWORDS=new Set(['der','die',_0x51adf3(0x183),'den','dem',_0x51adf3(0x178),_0x51adf3(0x17b),_0x51adf3(0x168),'einer','eines',_0x51adf3(0x1a1),_0x51adf3(0x179),'aber',_0x51adf3(0x1a4),'ist',_0x51adf3(0x1a0),'war',_0x51adf3(0x16a),_0x51adf3(0x161),'werden','für',_0x51adf3(0x16d),'mit',_0x51adf3(0x186),'nach','aus','auf',_0x51adf3(0x176),'unter',_0x51adf3(0x1a3),'wir','sie','man',_0x51adf3(0x185),_0x51adf3(0x175),'hatte','kann',_0x51adf3(0x172),'können',_0x51adf3(0x174),'sollen',_0x51adf3(0x192),'nicht',_0x51adf3(0x19c),_0x51adf3(0x195),_0x51adf3(0x15e),'the','and','for','are','was',_0x51adf3(0x194),'been','being','have','has',_0x51adf3(0x19b),_0x51adf3(0x184),'did',_0x51adf3(0x170),'would',_0x51adf3(0x18e),_0x51adf3(0x16c),_0x51adf3(0x18a),'might','shall',_0x51adf3(0x19e),'this','that','these',_0x51adf3(0x166),_0x51adf3(0x1a6),_0x51adf3(0x19a),_0x51adf3(0x160),_0x51adf3(0x15f),'than','then','them',_0x51adf3(0x190),'their',_0x51adf3(0x188),_0x51adf3(0x163),'what','which','when','where','who','whom',_0x51adf3(0x1a5),'not','but','its']);export function isStopword(_0x45eab8){return STOPWORDS['has'](_0x45eab8['toLowerCase']()['replace'](/[.,;:!?()'"'-]/g,''));}
1
+ const _0x28d4bc=_0x490a;(function(_0x4af767,_0xdac122){const _0x48d41c=_0x490a,_0x2e24a9=_0x4af767();while(!![]){try{const _0x169f0c=parseInt(_0x48d41c(0x112))/0x1*(-parseInt(_0x48d41c(0x118))/0x2)+parseInt(_0x48d41c(0xf0))/0x3*(-parseInt(_0x48d41c(0xef))/0x4)+-parseInt(_0x48d41c(0xcf))/0x5+-parseInt(_0x48d41c(0xe5))/0x6*(-parseInt(_0x48d41c(0xe0))/0x7)+parseInt(_0x48d41c(0x10f))/0x8*(parseInt(_0x48d41c(0xfe))/0x9)+-parseInt(_0x48d41c(0xd5))/0xa*(-parseInt(_0x48d41c(0x102))/0xb)+-parseInt(_0x48d41c(0xcd))/0xc*(-parseInt(_0x48d41c(0xd3))/0xd);if(_0x169f0c===_0xdac122)break;else _0x2e24a9['push'](_0x2e24a9['shift']());}catch(_0x531bfc){_0x2e24a9['push'](_0x2e24a9['shift']());}}}(_0x5bcf,0x817a1));const ABBREVIATIONS=['Nr.','z.B.',_0x28d4bc(0x11d),'u.a.','etc.',_0x28d4bc(0x116),_0x28d4bc(0xe7),_0x28d4bc(0x10e),_0x28d4bc(0xe2),'ggf.',_0x28d4bc(0xea),_0x28d4bc(0xf2),'o.ä.',_0x28d4bc(0xe4),'v.a.',_0x28d4bc(0xd1),_0x28d4bc(0x111),'i.e.','vs.','Mr.',_0x28d4bc(0x119),'Ms.',_0x28d4bc(0xce),_0x28d4bc(0xfa)],PLACEHOLDER='\x00PUNKT\x00';function _0x490a(_0xc6d5e6,_0x3d701c){_0xc6d5e6=_0xc6d5e6-0xca;const _0x5bcf57=_0x5bcf();let _0x490aa2=_0x5bcf57[_0xc6d5e6];return _0x490aa2;}export function splitSentences(_0x3d8c33){const _0x2cfa0b=_0x28d4bc,_0x48a6f9={'mBOYw':'\x00PUNKT\x00'};let _0x5bd45c=_0x3d8c33['replace'](/\s+/g,'\x20')['trim']();for(const _0x123286 of ABBREVIATIONS)_0x5bd45c=_0x5bd45c[_0x2cfa0b(0xe1)](_0x123286,_0x123286['replaceAll']('.',_0x48a6f9['mBOYw']));return _0x5bd45c['split'](/[.!?]\s+|\n+/)['map'](_0x383b52=>_0x383b52['replaceAll'](_0x2cfa0b(0xcb),'.')['trim']())[_0x2cfa0b(0xe6)](_0x39d61c=>_0x39d61c['length']>0x0);}export function extractContentWords(_0x3fc83b){return _0x3fc83b['toLowerCase']()['replace'](/[^a-zäöüß\s]/g,'')['split'](/\s+/)['filter'](_0x17069c=>_0x17069c['length']>=0x3);}export const STOPWORDS=new Set([_0x28d4bc(0xfc),_0x28d4bc(0xcc),_0x28d4bc(0x108),_0x28d4bc(0x110),_0x28d4bc(0xed),_0x28d4bc(0xe3),_0x28d4bc(0xe9),_0x28d4bc(0x11c),_0x28d4bc(0xf5),'eines',_0x28d4bc(0xd8),'oder',_0x28d4bc(0xd0),_0x28d4bc(0x107),'ist',_0x28d4bc(0x10c),'war','waren','wird','werden',_0x28d4bc(0xe8),_0x28d4bc(0xdc),_0x28d4bc(0x114),_0x28d4bc(0xf4),_0x28d4bc(0x104),'aus','auf',_0x28d4bc(0x100),_0x28d4bc(0xf6),'ich',_0x28d4bc(0x106),'sie','man','haben','hat','hatte','kann',_0x28d4bc(0x10b),_0x28d4bc(0x105),_0x28d4bc(0x109),'sollen','wollen',_0x28d4bc(0xff),_0x28d4bc(0x11b),'nur',_0x28d4bc(0xd4),'the',_0x28d4bc(0xeb),'for',_0x28d4bc(0x113),'was','were',_0x28d4bc(0xda),'being',_0x28d4bc(0xd7),_0x28d4bc(0x10a),_0x28d4bc(0xf7),'does',_0x28d4bc(0xf3),'will',_0x28d4bc(0xee),_0x28d4bc(0xdf),_0x28d4bc(0x101),_0x28d4bc(0x103),_0x28d4bc(0xdd),_0x28d4bc(0xf1),'can',_0x28d4bc(0x117),'that',_0x28d4bc(0xd2),'those','with',_0x28d4bc(0xfb),_0x28d4bc(0xec),_0x28d4bc(0xdb),_0x28d4bc(0x115),'then','them',_0x28d4bc(0x10d),_0x28d4bc(0x11a),'there',_0x28d4bc(0xca),_0x28d4bc(0xd9),'which',_0x28d4bc(0xfd),_0x28d4bc(0xf9),'who','whom','how',_0x28d4bc(0xde),'but',_0x28d4bc(0xf8)]);function _0x5bcf(){const _0x5ca5cc=['have','und','what','been','about','von','might','not','could','14wpPZqJ','replaceAll','ca.','des','u.U.','643686JGvHQw','filter','Dr.','für','ein','inkl.','and','into','dem','would','4BlehJN','32793VODuFw','shall','i.d.R.','did','bei','einer','unter','had','its','where','Sr.','from','der','when','9XbAJso','nicht','über','should','708158ywsRMe','may','nach','können','wir','dass','das','müssen','has','muss','sind','they','Prof.','1176184FhuaAN','den','e.g.','1RGEHWJ','are','mit','than','evtl.','this','410932iJvYfd','Mrs.','their','auch','eine','bzw.','here','\x00PUNKT\x00','die','875820yjTDFj','Jr.','3589420VMpwwA','aber','d.h.','these','13uSXVCA','noch','160iqZAiF','replace'];_0x5bcf=function(){return _0x5ca5cc;};return _0x5bcf();}export function isStopword(_0x17ff7b){const _0x265058=_0x28d4bc;return STOPWORDS[_0x265058(0x10a)](_0x17ff7b['toLowerCase']()[_0x265058(0xd6)](/[.,;:!?()'"'-]/g,''));}
package/dist/plugin/compression/token-budget.js CHANGED
@@ -1 +1 @@
1
- (function(_0x2cba1a,_0x27e730){const _0x3225fe=_0x16b2,_0xcadbea=_0x2cba1a();while(!![]){try{const _0x5d2e5f=parseInt(_0x3225fe(0xfa))/0x1*(parseInt(_0x3225fe(0x105))/0x2)+parseInt(_0x3225fe(0xf4))/0x3+-parseInt(_0x3225fe(0xfb))/0x4*(parseInt(_0x3225fe(0x103))/0x5)+parseInt(_0x3225fe(0xef))/0x6*(-parseInt(_0x3225fe(0x100))/0x7)+parseInt(_0x3225fe(0xee))/0x8+-parseInt(_0x3225fe(0x104))/0x9*(-parseInt(_0x3225fe(0xff))/0xa)+-parseInt(_0x3225fe(0x101))/0xb*(parseInt(_0x3225fe(0xf6))/0xc);if(_0x5d2e5f===_0x27e730)break;else _0xcadbea['push'](_0xcadbea['shift']());}catch(_0x9c38a5){_0xcadbea['push'](_0xcadbea['shift']());}}}(_0x1780,0x3f044));import{countTokens}from'../../shared/token-counter.js';import{splitSentences}from'./text-utils.js';const DEFAULT_TARGET_TOKENS=0x64,MIN_QUALITY_FLOOR=0.5,QUALITY_POSITION_BONUS=0.3;function computeUCurveScore(_0x31086c,_0x81d5c9){const _0x5d2681={'bciQU':function(_0x3d3e6d,_0xd219e9){return _0x3d3e6d-_0xd219e9;}};if(_0x81d5c9<=0x1)return 0x1;const _0x2e8e20=(_0x31086c+0x1)/_0x81d5c9;return 0x2*Math['max'](_0x2e8e20,_0x5d2681['bciQU'](0x1,_0x2e8e20));}function _0x1780(){const _0x577c83=['sentence','40samcBZ','2056663EIMjYC','22JemlNl','sort','10BWlTwZ','1044891Lrtjsy','4euhVfD','tokens','push','346216niwHUe','6FWJYMb','token-budget','tFmsr','BIFMs','score','340278kROEQo','elQZE','1735044bpXPIC','min','RJUBP','maxTokens','127714RtcAZl','70856QCAJnP','index','length'];_0x1780=function(){return _0x577c83;};return _0x1780();}function truncateSingleSentence(_0x4e750b,_0x5666a0,_0x2e9779){const _0x57df24=_0x16b2,_0x50fadc={'tFmsr':function(_0xae3990,_0x47e24c){return _0xae3990+_0x47e24c;},'cFstt':function(_0x4d9825,_0x5b8698){return _0x4d9825>_0x5b8698;}},_0x128116=_0x4e750b['split'](/\s+/);let _0x27f6ea='';for(const _0x5b41f7 of _0x128116){const _0x4e85ca=_0x27f6ea?_0x50fadc[_0x57df24(0xf1)](_0x27f6ea,'\x20')+_0x5b41f7:_0x5b41f7;if(countTokens(_0x4e85ca)>_0x2e9779)break;_0x27f6ea=_0x4e85ca;}const _0x4dc4d1=countTokens(_0x27f6ea);return{'text':_0x27f6ea,'originalTokens':_0x5666a0,'compressedTokens':_0x4dc4d1,'compressionRatio':_0x50fadc['cFstt'](_0x4dc4d1,0x0)?_0x5666a0/_0x4dc4d1:0x1,'strategyUsed':_0x57df24(0xf0),'qualityEstimate':_0x4dc4d1/_0x5666a0,'wasTruncated':!0x0};}function selectSentencesByBudget(_0x3fc62a,_0x17ec2f){const _0x193623=_0x16b2,_0x5d0320={'BgKea':function(_0x140980,_0x28cb24){return _0x140980+_0x28cb24;}},_0x16f8cf=[];let _0x29db46=0x0;for(const _0x388bcd of _0x3fc62a)(_0x5d0320['BgKea'](_0x29db46,_0x388bcd['tokens'])<=_0x17ec2f||0x0===_0x16f8cf[_0x193623(0xfd)])&&(_0x16f8cf[_0x193623(0x107)]({'index':_0x388bcd['index'],'sentence':_0x388bcd[_0x193623(0xfe)]}),_0x29db46+=_0x388bcd[_0x193623(0x106)]+0x1);return _0x16f8cf;}function _0x16b2(_0x556daf,_0x1e890a){_0x556daf=_0x556daf-0xee;const _0x178092=_0x1780();let _0x16b2f5=_0x178092[_0x556daf];return _0x16b2f5;}export function compressTokenBudget(_0x370913,_0x1abc1c={}){const _0x1ad24f=_0x16b2,_0x534fb0={'BIFMs':function(_0x5e7b80,_0x3813df){return _0x5e7b80<=_0x3813df;},'aCOZi':function(_0x10fcc7,_0x3ab7fa){return _0x10fcc7<=_0x3ab7fa;},'elQZE':function(_0x3e655e,_0x1efebf,_0x130581,_0x846968){return _0x3e655e(_0x1efebf,_0x130581,_0x846968);},'ywRao':_0x1ad24f(0xf0),'RJUBP':function(_0x5f07ba,_0x48abaa){return _0x5f07ba/_0x48abaa;}};if(!_0x370913||!_0x370913['trim']())return{'text':'','originalTokens':0x0,'compressedTokens':0x0,'compressionRatio':0x1,'strategyUsed':'token-budget','qualityEstimate':0x1,'wasTruncated':!0x1};const _0x2ca614=countTokens(_0x370913),_0x59d422=_0x1abc1c[_0x1ad24f(0xf9)]??0x64;if(_0x534fb0[_0x1ad24f(0xf2)](_0x2ca614,_0x59d422))return{'text':_0x370913,'originalTokens':_0x2ca614,'compressedTokens':_0x2ca614,'compressionRatio':0x1,'strategyUsed':'token-budget','qualityEstimate':0x1,'wasTruncated':!0x1};const _0x3d950e=splitSentences(_0x370913);if(_0x534fb0['aCOZi'](_0x3d950e['length'],0x1))return _0x534fb0[_0x1ad24f(0xf5)](truncateSingleSentence,_0x370913,_0x2ca614,_0x59d422);const _0x24491d=!0x1!==_0x1abc1c['preserveStructure'],_0x5d6cff=_0x3d950e[_0x1ad24f(0xfd)],_0xb89744=_0x3d950e['map']((_0x24cd23,_0x482f28)=>({'index':_0x482f28,'sentence':_0x24cd23,'tokens':countTokens(_0x24cd23),'score':_0x24491d?computeUCurveScore(_0x482f28,_0x5d6cff):0x1}));_0xb89744[_0x1ad24f(0x102)]((_0x2c1735,_0x563c84)=>_0x563c84['score']-_0x2c1735[_0x1ad24f(0xf3)]);const _0x2b9dba=selectSentencesByBudget(_0xb89744,_0x59d422);_0x2b9dba['sort']((_0x2ab81f,_0x5306f3)=>_0x2ab81f['index']-_0x5306f3[_0x1ad24f(0xfc)]);const _0xcf5706=_0x2b9dba['map'](_0x592adb=>_0x592adb['sentence'])['join']('.\x20'),_0x1c2ffc=countTokens(_0xcf5706);return{'text':_0xcf5706,'originalTokens':_0x2ca614,'compressedTokens':_0x1c2ffc,'compressionRatio':_0x1c2ffc>0x0?_0x2ca614/_0x1c2ffc:0x1,'strategyUsed':_0x534fb0['ywRao'],'qualityEstimate':Math['max'](0.5,Math[_0x1ad24f(0xf7)](0x1,_0x534fb0[_0x1ad24f(0xf8)](_0x1c2ffc,_0x2ca614)+0.3)),'wasTruncated':!0x0};}export const tokenBudgetStrategy={'name':'token-budget','compress':compressTokenBudget};
1
+ function _0x1d71(){const _0x487984=['niMvS','8yDQrmo','length','2000ZsorGj','1908696VWAkWw','split','min','max','21593493iKRUGP','trim','365745NMBuFW','AuxcQ','tptVq','push','39088KTLiCP','26022sxccuW','GHqvP','token-budget','apPBo','score','hgQRg','Zieqa','map','izpQp','6772400BHAHMj','tokens','474295bODJRx','inHso','vgmKt','index','sort'];_0x1d71=function(){return _0x487984;};return _0x1d71();}(function(_0x27d0e0,_0x81e61a){const _0x2a0616=_0x29bd,_0x144674=_0x27d0e0();while(!![]){try{const _0x3d7aa0=-parseInt(_0x2a0616(0x183))/0x1+-parseInt(_0x2a0616(0x16d))/0x2+-parseInt(_0x2a0616(0x173))/0x3+-parseInt(_0x2a0616(0x181))/0x4+-parseInt(_0x2a0616(0x18b))/0x5*(-parseInt(_0x2a0616(0x178))/0x6)+parseInt(_0x2a0616(0x177))/0x7+parseInt(_0x2a0616(0x189))/0x8*(parseInt(_0x2a0616(0x171))/0x9);if(_0x3d7aa0===_0x81e61a)break;else _0x144674['push'](_0x144674['shift']());}catch(_0x4af798){_0x144674['push'](_0x144674['shift']());}}}(_0x1d71,0xdac03));import{countTokens}from'../../shared/token-counter.js';import{splitSentences}from'./text-utils.js';const DEFAULT_TARGET_TOKENS=0x64,MIN_QUALITY_FLOOR=0.5,QUALITY_POSITION_BONUS=0.3;function computeUCurveScore(_0x8c5bf1,_0x17aaa4){const _0x3dd817=_0x29bd,_0x2fa98b={'hgQRg':function(_0x69ab65,_0x1c9828){return _0x69ab65<=_0x1c9828;},'vgmKt':function(_0x178ffb,_0x2f9ca8){return _0x178ffb-_0x2f9ca8;}};if(_0x2fa98b[_0x3dd817(0x17d)](_0x17aaa4,0x1))return 0x1;const _0x2fbe03=(_0x8c5bf1+0x1)/_0x17aaa4;return 0x2*Math['max'](_0x2fbe03,_0x2fa98b[_0x3dd817(0x185)](0x1,_0x2fbe03));}function truncateSingleSentence(_0x51d919,_0x1b7ff1,_0x3ac1c0){const _0x5acf22=_0x29bd,_0xd78177={'vZDkm':function(_0x2520f8,_0x40d222){return _0x2520f8>_0x40d222;},'apPBo':function(_0x2a3b5e,_0x399d9c){return _0x2a3b5e(_0x399d9c);},'niMvS':function(_0x241704,_0x4c66e3){return _0x241704/_0x4c66e3;},'Xbfut':_0x5acf22(0x17a)},_0x5e06ab=_0x51d919[_0x5acf22(0x16e)](/\s+/);let _0x5ce4ac='';for(const _0x22f324 of _0x5e06ab){const _0x369066=_0x5ce4ac?_0x5ce4ac+'\x20'+_0x22f324:_0x22f324;if(_0xd78177['vZDkm'](_0xd78177[_0x5acf22(0x17b)](countTokens,_0x369066),_0x3ac1c0))break;_0x5ce4ac=_0x369066;}const _0x22f8c8=_0xd78177['apPBo'](countTokens,_0x5ce4ac);return{'text':_0x5ce4ac,'originalTokens':_0x1b7ff1,'compressedTokens':_0x22f8c8,'compressionRatio':_0x22f8c8>0x0?_0xd78177[_0x5acf22(0x188)](_0x1b7ff1,_0x22f8c8):0x1,'strategyUsed':_0xd78177['Xbfut'],'qualityEstimate':_0x22f8c8/_0x1b7ff1,'wasTruncated':!0x0};}function selectSentencesByBudget(_0x4ee43b,_0x44c02d){const _0x590166=_0x29bd,_0x18b70f={'Zieqa':function(_0x3ad4e8,_0x28153f){return _0x3ad4e8+_0x28153f;}},_0x42ac79=[];let _0x25fc81=0x0;for(const _0xa5dddd of _0x4ee43b)(_0x18b70f['Zieqa'](_0x25fc81,_0xa5dddd[_0x590166(0x182)])<=_0x44c02d||0x0===_0x42ac79[_0x590166(0x18a)])&&(_0x42ac79[_0x590166(0x176)]({'index':_0xa5dddd['index'],'sentence':_0xa5dddd['sentence']}),_0x25fc81+=_0x18b70f[_0x590166(0x17e)](_0xa5dddd[_0x590166(0x182)],0x1));return _0x42ac79;}export function compressTokenBudget(_0x54c94c,_0xf19c5f={}){const _0x3b1f18=_0x29bd,_0x213879={'GHqvP':function(_0x2ab167,_0xb5fa82){return _0x2ab167(_0xb5fa82);},'AuxcQ':function(_0x823de8,_0x5bac63){return _0x823de8<=_0x5bac63;},'izpQp':'token-budget','yqxNF':function(_0x43db3a,_0x246d33){return _0x43db3a!==_0x246d33;},'ggstt':function(_0xf0dc73,_0x3a0a7b){return _0xf0dc73(_0x3a0a7b);},'inHso':function(_0x2a63a8,_0x320da2){return _0x2a63a8>_0x320da2;},'tptVq':function(_0x5d01ba,_0x10ae92){return _0x5d01ba/_0x10ae92;}};if(!_0x54c94c||!_0x54c94c[_0x3b1f18(0x172)]())return{'text':'','originalTokens':0x0,'compressedTokens':0x0,'compressionRatio':0x1,'strategyUsed':_0x3b1f18(0x17a),'qualityEstimate':0x1,'wasTruncated':!0x1};const _0x16121c=_0x213879['GHqvP'](countTokens,_0x54c94c),_0x12ece8=_0xf19c5f['maxTokens']??0x64;if(_0x213879[_0x3b1f18(0x174)](_0x16121c,_0x12ece8))return{'text':_0x54c94c,'originalTokens':_0x16121c,'compressedTokens':_0x16121c,'compressionRatio':0x1,'strategyUsed':_0x213879[_0x3b1f18(0x180)],'qualityEstimate':0x1,'wasTruncated':!0x1};const _0x2842e8=_0x213879[_0x3b1f18(0x179)](splitSentences,_0x54c94c);if(_0x2842e8[_0x3b1f18(0x18a)]<=0x1)return truncateSingleSentence(_0x54c94c,_0x16121c,_0x12ece8);const _0x5ece78=_0x213879['yqxNF'](!0x1,_0xf19c5f['preserveStructure']),_0x20e5ed=_0x2842e8['length'],_0x89d649=_0x2842e8[_0x3b1f18(0x17f)]((_0x47a1ec,_0xb10900)=>({'index':_0xb10900,'sentence':_0x47a1ec,'tokens':countTokens(_0x47a1ec),'score':_0x5ece78?computeUCurveScore(_0xb10900,_0x20e5ed):0x1}));_0x89d649[_0x3b1f18(0x187)]((_0x153326,_0x1357c4)=>_0x1357c4[_0x3b1f18(0x17c)]-_0x153326['score']);const _0x35b368=selectSentencesByBudget(_0x89d649,_0x12ece8);_0x35b368['sort']((_0x5c8869,_0x50d09e)=>_0x5c8869[_0x3b1f18(0x186)]-_0x50d09e[_0x3b1f18(0x186)]);const _0x5e3b7d=_0x35b368['map'](_0x11b34a=>_0x11b34a['sentence'])['join']('.\x20'),_0x1a411b=_0x213879['ggstt'](countTokens,_0x5e3b7d);return{'text':_0x5e3b7d,'originalTokens':_0x16121c,'compressedTokens':_0x1a411b,'compressionRatio':_0x213879[_0x3b1f18(0x184)](_0x1a411b,0x0)?_0x16121c/_0x1a411b:0x1,'strategyUsed':_0x3b1f18(0x17a),'qualityEstimate':Math[_0x3b1f18(0x170)](0.5,Math[_0x3b1f18(0x16f)](0x1,_0x213879[_0x3b1f18(0x175)](_0x1a411b,_0x16121c)+0.3)),'wasTruncated':!0x0};}function _0x29bd(_0x460c17,_0x1a7cfd){_0x460c17=_0x460c17-0x16d;const _0x1d7184=_0x1d71();let _0x29bd5e=_0x1d7184[_0x460c17];return _0x29bd5e;}export const tokenBudgetStrategy={'name':'token-budget','compress':compressTokenBudget};