edgeflowjs 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/README.md +200 -66
  2. package/dist/backends/index.d.ts +9 -2
  3. package/dist/backends/index.d.ts.map +1 -1
  4. package/dist/backends/index.js +13 -13
  5. package/dist/backends/index.js.map +1 -1
  6. package/dist/backends/onnx.d.ts +11 -4
  7. package/dist/backends/onnx.d.ts.map +1 -1
  8. package/dist/backends/onnx.js +97 -78
  9. package/dist/backends/onnx.js.map +1 -1
  10. package/dist/backends/transformers-adapter.d.ts +99 -0
  11. package/dist/backends/transformers-adapter.d.ts.map +1 -0
  12. package/dist/backends/transformers-adapter.js +171 -0
  13. package/dist/backends/transformers-adapter.js.map +1 -0
  14. package/dist/backends/webgpu.d.ts +7 -5
  15. package/dist/backends/webgpu.d.ts.map +1 -1
  16. package/dist/backends/webgpu.js +7 -5
  17. package/dist/backends/webgpu.js.map +1 -1
  18. package/dist/backends/webnn.d.ts +6 -5
  19. package/dist/backends/webnn.d.ts.map +1 -1
  20. package/dist/backends/webnn.js +6 -5
  21. package/dist/backends/webnn.js.map +1 -1
  22. package/dist/core/composer.d.ts +118 -0
  23. package/dist/core/composer.d.ts.map +1 -0
  24. package/dist/core/composer.js +163 -0
  25. package/dist/core/composer.js.map +1 -0
  26. package/dist/core/device-profiler.d.ts +75 -0
  27. package/dist/core/device-profiler.d.ts.map +1 -0
  28. package/dist/core/device-profiler.js +131 -0
  29. package/dist/core/device-profiler.js.map +1 -0
  30. package/dist/core/index.d.ts +4 -0
  31. package/dist/core/index.d.ts.map +1 -1
  32. package/dist/core/index.js +8 -0
  33. package/dist/core/index.js.map +1 -1
  34. package/dist/core/memory.d.ts +22 -2
  35. package/dist/core/memory.d.ts.map +1 -1
  36. package/dist/core/memory.js +49 -13
  37. package/dist/core/memory.js.map +1 -1
  38. package/dist/core/plugin.d.ts +100 -0
  39. package/dist/core/plugin.d.ts.map +1 -0
  40. package/dist/core/plugin.js +106 -0
  41. package/dist/core/plugin.js.map +1 -0
  42. package/dist/core/runtime.d.ts +4 -0
  43. package/dist/core/runtime.d.ts.map +1 -1
  44. package/dist/core/runtime.js +18 -0
  45. package/dist/core/runtime.js.map +1 -1
  46. package/dist/core/scheduler.d.ts +17 -0
  47. package/dist/core/scheduler.d.ts.map +1 -1
  48. package/dist/core/scheduler.js +101 -3
  49. package/dist/core/scheduler.js.map +1 -1
  50. package/dist/core/types.d.ts +14 -0
  51. package/dist/core/types.d.ts.map +1 -1
  52. package/dist/core/types.js.map +1 -1
  53. package/dist/core/worker.d.ts +202 -0
  54. package/dist/core/worker.d.ts.map +1 -0
  55. package/dist/core/worker.js +477 -0
  56. package/dist/core/worker.js.map +1 -0
  57. package/dist/edgeflow.browser.js +9770 -4383
  58. package/dist/edgeflow.browser.js.map +4 -4
  59. package/dist/edgeflow.browser.min.js +435 -5
  60. package/dist/edgeflow.browser.min.js.map +4 -4
  61. package/dist/index.d.ts +7 -4
  62. package/dist/index.d.ts.map +1 -1
  63. package/dist/index.js +28 -10
  64. package/dist/index.js.map +1 -1
  65. package/dist/pipelines/automatic-speech-recognition.d.ts +63 -0
  66. package/dist/pipelines/automatic-speech-recognition.d.ts.map +1 -0
  67. package/dist/pipelines/automatic-speech-recognition.js +269 -0
  68. package/dist/pipelines/automatic-speech-recognition.js.map +1 -0
  69. package/dist/pipelines/base.d.ts +6 -1
  70. package/dist/pipelines/base.d.ts.map +1 -1
  71. package/dist/pipelines/base.js +12 -2
  72. package/dist/pipelines/base.js.map +1 -1
  73. package/dist/pipelines/feature-extraction.d.ts +5 -40
  74. package/dist/pipelines/feature-extraction.d.ts.map +1 -1
  75. package/dist/pipelines/feature-extraction.js +44 -63
  76. package/dist/pipelines/feature-extraction.js.map +1 -1
  77. package/dist/pipelines/image-classification.d.ts +4 -36
  78. package/dist/pipelines/image-classification.d.ts.map +1 -1
  79. package/dist/pipelines/image-classification.js +22 -60
  80. package/dist/pipelines/image-classification.js.map +1 -1
  81. package/dist/pipelines/image-segmentation.d.ts +221 -0
  82. package/dist/pipelines/image-segmentation.d.ts.map +1 -0
  83. package/dist/pipelines/image-segmentation.js +535 -0
  84. package/dist/pipelines/image-segmentation.js.map +1 -0
  85. package/dist/pipelines/index.d.ts +18 -0
  86. package/dist/pipelines/index.d.ts.map +1 -1
  87. package/dist/pipelines/index.js +51 -2
  88. package/dist/pipelines/index.js.map +1 -1
  89. package/dist/pipelines/object-detection.d.ts +44 -0
  90. package/dist/pipelines/object-detection.d.ts.map +1 -0
  91. package/dist/pipelines/object-detection.js +218 -0
  92. package/dist/pipelines/object-detection.js.map +1 -0
  93. package/dist/pipelines/question-answering.d.ts +41 -0
  94. package/dist/pipelines/question-answering.d.ts.map +1 -0
  95. package/dist/pipelines/question-answering.js +164 -0
  96. package/dist/pipelines/question-answering.js.map +1 -0
  97. package/dist/pipelines/text-classification.d.ts +3 -39
  98. package/dist/pipelines/text-classification.d.ts.map +1 -1
  99. package/dist/pipelines/text-classification.js +29 -67
  100. package/dist/pipelines/text-classification.js.map +1 -1
  101. package/dist/pipelines/text-generation.d.ts +281 -0
  102. package/dist/pipelines/text-generation.d.ts.map +1 -0
  103. package/dist/pipelines/text-generation.js +766 -0
  104. package/dist/pipelines/text-generation.js.map +1 -0
  105. package/dist/pipelines/zero-shot-classification.d.ts +45 -0
  106. package/dist/pipelines/zero-shot-classification.d.ts.map +1 -0
  107. package/dist/pipelines/zero-shot-classification.js +140 -0
  108. package/dist/pipelines/zero-shot-classification.js.map +1 -0
  109. package/dist/tools/benchmark.d.ts +92 -0
  110. package/dist/tools/benchmark.d.ts.map +1 -0
  111. package/dist/tools/benchmark.js +213 -0
  112. package/dist/tools/benchmark.js.map +1 -0
  113. package/dist/tools/debugger.d.ts +258 -0
  114. package/dist/tools/debugger.d.ts.map +1 -0
  115. package/dist/tools/debugger.js +624 -0
  116. package/dist/tools/debugger.js.map +1 -0
  117. package/dist/tools/index.d.ts +8 -0
  118. package/dist/tools/index.d.ts.map +1 -1
  119. package/dist/tools/index.js +16 -0
  120. package/dist/tools/index.js.map +1 -1
  121. package/dist/tools/monitor.d.ts +284 -0
  122. package/dist/tools/monitor.d.ts.map +1 -0
  123. package/dist/tools/monitor.js +921 -0
  124. package/dist/tools/monitor.js.map +1 -0
  125. package/dist/tools/quantization.d.ts +235 -0
  126. package/dist/tools/quantization.d.ts.map +1 -0
  127. package/dist/tools/quantization.js +830 -0
  128. package/dist/tools/quantization.js.map +1 -0
  129. package/dist/utils/hub.d.ts +162 -0
  130. package/dist/utils/hub.d.ts.map +1 -0
  131. package/dist/utils/hub.js +311 -0
  132. package/dist/utils/hub.js.map +1 -0
  133. package/dist/utils/index.d.ts +3 -1
  134. package/dist/utils/index.d.ts.map +1 -1
  135. package/dist/utils/index.js +5 -1
  136. package/dist/utils/index.js.map +1 -1
  137. package/dist/utils/model-loader.d.ts.map +1 -1
  138. package/dist/utils/model-loader.js +106 -30
  139. package/dist/utils/model-loader.js.map +1 -1
  140. package/dist/utils/offline.d.ts +147 -0
  141. package/dist/utils/offline.d.ts.map +1 -0
  142. package/dist/utils/offline.js +405 -0
  143. package/dist/utils/offline.js.map +1 -0
  144. package/dist/utils/preprocessor.d.ts +82 -6
  145. package/dist/utils/preprocessor.d.ts.map +1 -1
  146. package/dist/utils/preprocessor.js +278 -21
  147. package/dist/utils/preprocessor.js.map +1 -1
  148. package/dist/utils/tokenizer.d.ts +197 -72
  149. package/dist/utils/tokenizer.d.ts.map +1 -1
  150. package/dist/utils/tokenizer.js +558 -274
  151. package/dist/utils/tokenizer.js.map +1 -1
  152. package/package.json +26 -11
@@ -1,140 +1,265 @@
1
1
  /**
2
2
  * edgeFlow.js - Tokenizer
3
3
  *
4
- * Lightweight tokenizer implementation for text processing.
5
- * Supports BPE, WordPiece, and basic tokenization.
4
+ * Full-featured tokenizer supporting HuggingFace tokenizer.json format.
5
+ * Supports BPE, WordPiece, and Unigram tokenization.
6
6
  */
7
7
  import { TokenizerConfig, TokenizedOutput } from '../core/types.js';
8
- /**
9
- * Tokenizer model types
10
- */
11
- export type TokenizerModel = 'bpe' | 'wordpiece' | 'unigram' | 'basic';
12
- /**
13
- * Tokenizer options
14
- */
8
+ export type TokenizerModel = 'BPE' | 'WordPiece' | 'Unigram' | 'basic';
15
9
  export interface TokenizerOptions {
16
- /** Tokenizer model type */
17
- model?: TokenizerModel;
18
- /** Vocabulary */
19
- vocab?: Map<string, number> | Record<string, number>;
20
- /** Merges for BPE */
21
- merges?: string[];
22
- /** Add special tokens */
23
10
  addSpecialTokens?: boolean;
24
- /** Maximum length */
25
11
  maxLength?: number;
26
- /** Padding strategy */
27
12
  padding?: 'max_length' | 'longest' | 'do_not_pad';
28
- /** Truncation */
29
13
  truncation?: boolean;
30
- /** Return attention mask */
31
14
  returnAttentionMask?: boolean;
32
- /** Return token type IDs */
33
15
  returnTokenTypeIds?: boolean;
16
+ textPair?: string;
34
17
  }
35
18
  /**
36
- * Tokenizer - Base class for all tokenizers
19
+ * HuggingFace tokenizer.json format
20
+ */
21
+ interface HFTokenizerJSON {
22
+ version?: string;
23
+ truncation?: {
24
+ max_length: number;
25
+ strategy: string;
26
+ };
27
+ padding?: {
28
+ strategy: string;
29
+ pad_id: number;
30
+ pad_token: string;
31
+ };
32
+ added_tokens?: Array<{
33
+ id: number;
34
+ content: string;
35
+ single_word: boolean;
36
+ lstrip: boolean;
37
+ rstrip: boolean;
38
+ normalized: boolean;
39
+ special: boolean;
40
+ }>;
41
+ normalizer?: {
42
+ type: string;
43
+ lowercase?: boolean;
44
+ strip_accents?: boolean;
45
+ [key: string]: unknown;
46
+ };
47
+ pre_tokenizer?: {
48
+ type: string;
49
+ [key: string]: unknown;
50
+ };
51
+ post_processor?: {
52
+ type: string;
53
+ single?: Array<{
54
+ id: string;
55
+ type_id: number;
56
+ } | {
57
+ SpecialToken: {
58
+ id: string;
59
+ type_id: number;
60
+ };
61
+ } | {
62
+ Sequence: {
63
+ id: string;
64
+ type_id: number;
65
+ };
66
+ }>;
67
+ pair?: Array<{
68
+ id: string;
69
+ type_id: number;
70
+ } | {
71
+ SpecialToken: {
72
+ id: string;
73
+ type_id: number;
74
+ };
75
+ } | {
76
+ Sequence: {
77
+ id: string;
78
+ type_id: number;
79
+ };
80
+ }>;
81
+ special_tokens?: Record<string, {
82
+ id: string;
83
+ ids: number[];
84
+ tokens: string[];
85
+ }>;
86
+ [key: string]: unknown;
87
+ };
88
+ decoder?: {
89
+ type: string;
90
+ [key: string]: unknown;
91
+ };
92
+ model: {
93
+ type: string;
94
+ vocab?: Record<string, number>;
95
+ merges?: string[];
96
+ unk_token?: string;
97
+ continuing_subword_prefix?: string;
98
+ end_of_word_suffix?: string;
99
+ fuse_unk?: boolean;
100
+ byte_fallback?: boolean;
101
+ [key: string]: unknown;
102
+ };
103
+ }
104
+ /**
105
+ * Tokenizer - Full-featured tokenizer supporting HuggingFace format
37
106
  */
38
107
  export declare class Tokenizer {
39
- protected vocab: Map<string, number>;
40
- protected reverseVocab: Map<number, string>;
41
- protected config: TokenizerConfig;
42
- protected model: TokenizerModel;
43
- protected merges: Map<string, string>;
44
- constructor(config: Partial<TokenizerConfig>, options?: TokenizerOptions);
108
+ private vocab;
109
+ private reverseVocab;
110
+ private merges;
111
+ private addedTokens;
112
+ private specialTokens;
113
+ private modelType;
114
+ private unkToken;
115
+ private continuingSubwordPrefix;
116
+ private padTokenId;
117
+ private unkTokenId;
118
+ private clsTokenId?;
119
+ private sepTokenId?;
120
+ private maskTokenId?;
121
+ private bosTokenId?;
122
+ private eosTokenId?;
123
+ private maxLength;
124
+ private doLowerCase;
125
+ private stripAccents;
126
+ private postProcessor?;
127
+ private byteEncoder;
128
+ private byteDecoder;
129
+ constructor();
45
130
  /**
46
- * Load vocabulary
131
+ * Initialize byte encoder/decoder for BPE
47
132
  */
48
- protected loadVocab(vocab: Map<string, number> | Record<string, number>): void;
133
+ private initByteEncoder;
49
134
  /**
50
- * Load BPE merges
135
+ * Load from HuggingFace tokenizer.json
51
136
  */
52
- protected loadMerges(merges: string[]): void;
137
+ static fromJSON(json: HFTokenizerJSON | string): Promise<Tokenizer>;
53
138
  /**
54
- * Tokenize text
139
+ * Load from URL (tokenizer.json)
55
140
  */
56
- encode(text: string, options?: {
57
- addSpecialTokens?: boolean;
58
- maxLength?: number;
59
- padding?: 'max_length' | 'longest' | 'do_not_pad';
60
- truncation?: boolean;
61
- returnAttentionMask?: boolean;
62
- returnTokenTypeIds?: boolean;
63
- }): TokenizedOutput;
141
+ static fromUrl(url: string): Promise<Tokenizer>;
64
142
  /**
65
- * Batch encode
143
+ * Load from HuggingFace Hub
66
144
  */
67
- encodeBatch(texts: string[], options?: {
68
- addSpecialTokens?: boolean;
69
- maxLength?: number;
70
- padding?: 'max_length' | 'longest' | 'do_not_pad';
71
- truncation?: boolean;
72
- returnAttentionMask?: boolean;
73
- returnTokenTypeIds?: boolean;
74
- }): TokenizedOutput[];
145
+ static fromHuggingFace(modelId: string, options?: {
146
+ revision?: string;
147
+ }): Promise<Tokenizer>;
75
148
  /**
76
- * Decode token IDs back to text
149
+ * Normalize text
77
150
  */
78
- decode(ids: number[], skipSpecialTokens?: boolean): string;
151
+ private normalize;
79
152
  /**
80
- * Basic tokenization (split by whitespace and punctuation)
153
+ * Pre-tokenize text (split into words)
81
154
  */
82
- protected tokenize(text: string): string[];
155
+ private preTokenize;
83
156
  /**
84
- * Normalize text
157
+ * Encode text to bytes (for BPE)
158
+ */
159
+ private textToBytes;
160
+ /**
161
+ * Decode bytes to text (for BPE)
85
162
  */
86
- protected normalize(text: string): string;
163
+ private bytesToText;
87
164
  /**
88
- * Basic tokenization
165
+ * Get BPE pairs from word
89
166
  */
90
- protected tokenizeBasic(text: string): string[];
167
+ private getPairs;
168
+ /**
169
+ * Apply BPE to a word
170
+ */
171
+ private bpe;
91
172
  /**
92
173
  * WordPiece tokenization
93
174
  */
94
- protected tokenizeWordPiece(text: string): string[];
175
+ private wordPiece;
95
176
  /**
96
- * Tokenize a single word using WordPiece
177
+ * Tokenize a single word
97
178
  */
98
- protected tokenizeWord(word: string): string[];
179
+ private tokenizeWord;
99
180
  /**
100
- * BPE tokenization
181
+ * Greedy longest-match tokenizer for SentencePiece Unigram models.
182
+ * Adds the U+2581 (▁) word-start prefix expected by SPM-based models.
101
183
  */
102
- protected tokenizeBPE(text: string): string[];
184
+ private unigramTokenize;
103
185
  /**
104
- * Add special tokens
186
+ * Main tokenization
105
187
  */
106
- protected addSpecialTokens(tokens: string[]): string[];
188
+ private tokenize;
107
189
  /**
108
190
  * Convert tokens to IDs
109
191
  */
110
- protected convertTokensToIds(tokens: string[]): number[];
192
+ private convertTokensToIds;
111
193
  /**
112
194
  * Convert IDs to tokens
113
195
  */
114
- protected convertIdsToTokens(ids: number[]): string[];
196
+ private convertIdsToTokens;
115
197
  /**
116
- * Check if token is a special token
198
+ * Apply post-processing (add special tokens)
117
199
  */
118
- protected isSpecialToken(token: string): boolean;
200
+ private postProcess;
119
201
  /**
120
- * Detokenize (convert tokens back to text)
202
+ * Encode text
121
203
  */
122
- protected detokenize(tokens: string[]): string;
204
+ encode(text: string, options?: TokenizerOptions): TokenizedOutput;
205
+ /**
206
+ * Batch encode
207
+ */
208
+ encodeBatch(texts: string[], options?: TokenizerOptions): TokenizedOutput[];
209
+ /**
210
+ * Decode IDs to text
211
+ */
212
+ decode(ids: number[], skipSpecialTokens?: boolean): string;
213
+ /**
214
+ * Decode batch
215
+ */
216
+ decodeBatch(batchIds: number[][], skipSpecialTokens?: boolean): string[];
123
217
  /**
124
218
  * Get vocabulary size
125
219
  */
126
220
  get vocabSize(): number;
221
+ /**
222
+ * Get special token IDs
223
+ */
224
+ getSpecialTokenIds(): {
225
+ padTokenId: number;
226
+ unkTokenId: number;
227
+ clsTokenId?: number;
228
+ sepTokenId?: number;
229
+ maskTokenId?: number;
230
+ bosTokenId?: number;
231
+ eosTokenId?: number;
232
+ };
127
233
  /**
128
234
  * Get config
129
235
  */
130
236
  getConfig(): TokenizerConfig;
237
+ /**
238
+ * Check if token is special
239
+ */
240
+ isSpecialToken(token: string): boolean;
241
+ /**
242
+ * Get token ID
243
+ */
244
+ getTokenId(token: string): number | undefined;
245
+ /**
246
+ * Get token from ID
247
+ */
248
+ getToken(id: number): string | undefined;
131
249
  }
132
250
  /**
133
- * Create a basic English tokenizer
251
+ * Create a basic English tokenizer (for testing)
134
252
  */
135
253
  export declare function createBasicTokenizer(): Tokenizer;
136
254
  /**
137
255
  * Load tokenizer from URL
138
256
  */
139
257
  export declare function loadTokenizer(url: string): Promise<Tokenizer>;
258
+ /**
259
+ * Load tokenizer from HuggingFace Hub
260
+ */
261
+ export declare function loadTokenizerFromHub(modelId: string, options?: {
262
+ revision?: string;
263
+ }): Promise<Tokenizer>;
264
+ export {};
140
265
  //# sourceMappingURL=tokenizer.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../../src/utils/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EACL,eAAe,EACf,eAAe,EAGhB,MAAM,kBAAkB,CAAC;AAM1B;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,KAAK,GAAG,WAAW,GAAG,SAAS,GAAG,OAAO,CAAC;AAEvE;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,2BAA2B;IAC3B,KAAK,CAAC,EAAE,cAAc,CAAC;IACvB,iBAAiB;IACjB,KAAK,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrD,qBAAqB;IACrB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,yBAAyB;IACzB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,qBAAqB;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uBAAuB;IACvB,OAAO,CAAC,EAAE,YAAY,GAAG,SAAS,GAAG,YAAY,CAAC;IAClD,iBAAiB;IACjB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,4BAA4B;IAC5B,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,4BAA4B;IAC5B,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAMD;;GAEG;AACH,qBAAa,SAAS;IACpB,SAAS,CAAC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrC,SAAS,CAAC,YAAY,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5C,SAAS,CAAC,MAAM,EAAE,eAAe,CAAC;IAClC,SAAS,CAAC,KAAK,EAAE,cAAc,CAAC;IAChC,SAAS,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAa;gBAEtC,MAAM,EAAE,OAAO,CAAC,eAAe,CAAC,EAAE,OAAO,GAAE,gBAAqB;IA4B5E;;OAEG;IACH,SAAS,CAAC,SAAS,CAAC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI;IAa9E;;OAEG;IACH,SAAS,CAAC,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI;IAS5C;;OAEG;IACH,MAAM,CACJ,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE;QACP,gBAAgB,CAAC,EAAE,OAAO,CAAC;QAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,OAAO,CAAC,EAAE,YAAY,GAAG,SAAS,GAAG,YAAY,CAAC;QAClD,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,mBAAmB,CAAC,EAAE,OAAO,CAAC;QAC9B,kBAAkB,CAAC,EAAE,OAAO,CAAC;KACzB,GACL,eAAe;IAyDlB;;OAEG;IACH,WAAW,CACT,KAAK,EAAE,MAAM,EAAE,EACf,OAAO,GAAE;QACP,gBAAgB,CAAC,EAAE,OAAO,CAAC;QAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,OAAO,CAAC,EAAE,YAAY,GAAG,SAAS,GAAG,YAAY,CAAC;QAClD,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,mBAAmB,CAAC,EAAE,OAAO,CAAC;QAC9B,kBAAkB,CAAC,EAAE,OAAO,CAAC;KACzB,GACL,eAAe,EAAE;IAYpB;;OAEG;IACH,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,iBAAiB,UAAO,GAAG,MAAM;IAWvD;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;IAc1C;;OAEG;IACH,SAAS,CAAC,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAQzC;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;IAI/C;;OAEG;IACH,SAAS,CAAC,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;IAYnD;;OAEG;IACH,SAAS,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;IAmC9C;;OAEG;IACH,SAAS,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;IAyC7C;;OAEG;IACH,SAAS,CAAC,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE;IAkBtD;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE;IAgBxD;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE;IAgBrD;;OAEG;IACH,SAAS,CAAC,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO;IAIhD;;OAEG;IACH,SAAS,CAAC,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM;IAW9C;;OAEG;IACH,IAAI,SAAS,IAAI,MAAM,CAEtB;IAED;;OAEG;IACH,SAAS,IAAI,eAAe;CAG7B;AAMD;;GAEG;AACH,wBAAgB,oBAAoB,IAAI,SAAS,CA0ChD;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAwBnE"}
1
+ {"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../../src/utils/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EACL,eAAe,EACf,eAAe,EAGhB,MAAM,kBAAkB,CAAC;AAM1B,MAAM,MAAM,cAAc,GAAG,KAAK,GAAG,WAAW,GAAG,SAAS,GAAG,OAAO,CAAC;AAEvE,MAAM,WAAW,gBAAgB;IAC/B,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,YAAY,GAAG,SAAS,GAAG,YAAY,CAAC;IAClD,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,UAAU,eAAe;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE;QACX,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,OAAO,CAAC,EAAE;QACR,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,YAAY,CAAC,EAAE,KAAK,CAAC;QACnB,EAAE,EAAE,MAAM,CAAC;QACX,OAAO,EAAE,MAAM,CAAC;QAChB,WAAW,EAAE,OAAO,CAAC;QACrB,MAAM,EAAE,OAAO,CAAC;QAChB,MAAM,EAAE,OAAO,CAAC;QAChB,UAAU,EAAE,OAAO,CAAC;QACpB,OAAO,EAAE,OAAO,CAAC;KAClB,CAAC,CAAC;IACH,UAAU,CAAC,EAAE;QACX,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,aAAa,CAAC,EAAE,OAAO,CAAC;QACxB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;KACxB,CAAC;IACF,aAAa,CAAC,EAAE;QACd,IAAI,EAAE,MAAM,CAAC;QACb,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;KACxB,CAAC;IACF,cAAc,CAAC,EAAE;QACf,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,CAAC,EAAE,KAAK,CAAC;YAAE,EAAE,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,GAAG;YAAE,YAAY,EAAE;gBAAE,EAAE,EAAE,MAAM,CAAC;gBAAC,OAAO,EAAE,MAAM,CAAA;aAAE,CAAA;SAAE,GAAG;YAAE,QAAQ,EAAE;gBAAE,EAAE,EAAE,MAAM,CAAC;gBAAC,OAAO,EAAE,MAAM,CAAA;aAAE,CAAA;SAAE,CAAC,CAAC;QACpJ,IAAI,CAAC,EAAE,KAAK,CAAC;YAAE,EAAE,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,GAAG;YAAE,YAAY,EAAE;gBAAE,EAAE,EAAE,MAAM,CAAC;gBAAC,OAAO,EAAE,MAAM,CAAA;aAAE,CAAA;SAAE,GAAG;YAAE,QAAQ,EAAE;gBAAE,EAAE,EAAE,MAAM,CAAC;gBAAC,OAAO,EAAE,MAAM,CAAA;aAAE,CAAA;SAAE,CAAC,CAAC;QAClJ,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,EAAE,EAAE,MAAM,CAAC;YAAC,GAAG,EAAE,MAAM,EAAE,CAAC;YAAC,MAAM,EAAE,MAAM,EAAE,CAAA;SAAE,CAAC,CAAC;QACjF,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;KACxB,CAAC;IACF,OAAO,CAAC,EAAE;QACR,IAAI,EAAE,MAAM,CAAC;QACb,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;KACxB,CAAC;IACF,KAAK,EAAE;QACL,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC/B,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;QAClB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,yBAAyB,CAAC,EAAE,MAAM,CAAC;QACnC,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,aAAa,CAAC,EAAE,OAAO,CAAC;QACxB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;KACxB,CAAC;CACH;AAOD;;GAEG;AACH,qBAAa,SAAS;IACpB,OAAO,CAAC,KAAK,CAAkC;IAC/C,OAAO,CAAC,YAAY,CAAkC;IACtD,OAAO,CAAC,MAAM,CAAkC;IAChD,OAAO,CAAC,WAAW,CAAkC;IACrD,OAAO,CAAC,aAAa,CAA0B;IAE/C,OAAO,CAAC,SAAS,CAAyB;IAC1C,OAAO,CAAC,QAAQ,CAAmB;IACnC,OAAO,CAAC,uBAAuB,CAAgB;IAG/C,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,UAAU,CAAC,CAAS;IAC5B,OAAO,CAAC,UAAU,CAAC,CAAS;IAC5B,OAAO,CAAC,WAAW,CAAC,CAAS;IAC7B,OAAO,CAAC,UAAU,CAAC,CAAS;IAC5B,OAAO,CAAC,UAAU,CAAC,CAAS;IAG5B,OAAO,CAAC,SAAS,CAAe;IAChC,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,YAAY,CAAkB;IAGtC,OAAO,CAAC,aAAa,CAAC,CAAoC;IAG1D,OAAO,CAAC,WAAW,CAAkC;IACrD,OAAO,CAAC,WAAW,CAAkC;;IAMrD;;OAEG;IACH,OAAO,CAAC,eAAe;IA2BvB;;OAEG;WACU,QAAQ,CAAC,IAAI,EAAE,eAAe,GAAG,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IAkFzE;;OAEG;WACU,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IAYrD;;OAEG;WACU,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,SAAS,CAAC;IAMlG;;OAEG;IACH,OAAO,CAAC,SAAS;IAiBjB;;OAEG;IACH,OAAO,CAAC,WAAW;IAOnB;;OAEG;IACH,OAAO,CAAC,WAAW;IAMnB;;OAEG;IACH,OAAO,CAAC,WAAW;IAQnB;;OAEG;IACH,OAAO,CAAC,QAAQ;IAQhB;;OAEG;IACH,OAAO,CAAC,GAAG;IA+DX;;OAEG;IACH,OAAO,CAAC,SAAS;IAqCjB;;OAEG;IACH,OAAO,CAAC,YAAY;IAqBpB;;;OAGG;IACH,OAAO,CAAC,eAAe;IAiCvB;;OAEG;IACH,OAAO,CAAC,QAAQ;IA6ChB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAe1B;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAI1B;;OAEG;IACH,OAAO,CAAC,WAAW;IA6DnB;;OAEG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,eAAe;IA2ErE;;OAEG;IACH,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,OAAO,GAAE,gBAAqB,GAAG,eAAe,EAAE;IAW/E;;OAEG;IACH,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,iBAAiB,UAAO,GAAG,MAAM;IA6CvD;;OAEG;IACH,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,iBAAiB,UAAO,GAAG,MAAM,EAAE;IAIrE;;OAEG;IACH,IAAI,SAAS,IAAI,MAAM,CAEtB;IAED;;OAEG;IACH,kBAAkB,IAAI;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB;IAYD;;OAEG;IACH,SAAS,IAAI,eAAe;IAc5B;;OAEG;IACH,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO;IAItC;;OAEG;IACH,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAI7C;;OAEG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;CAGzC;AAMD;;GAEG;AACH,wBAAgB,oBAAoB,IAAI,SAAS,CAGhD;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAEnE;AAED;;GAEG;AACH,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,MAAM,EACf,OAAO,CAAC,EAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,GAC9B,OAAO,CAAC,SAAS,CAAC,CAEpB"}