@lokascript/semantic 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/dist/core.d.ts +1246 -0
  2. package/dist/core.js +3073 -0
  3. package/dist/core.js.map +1 -0
  4. package/dist/languages/bn.d.ts +33 -0
  5. package/dist/languages/bn.js +1101 -0
  6. package/dist/languages/bn.js.map +1 -0
  7. package/dist/languages/es-MX.d.ts +23 -0
  8. package/dist/languages/es-MX.js +1676 -0
  9. package/dist/languages/es-MX.js.map +1 -0
  10. package/dist/languages/es.d.ts +3 -42
  11. package/dist/languages/he.d.ts +70 -0
  12. package/dist/languages/he.js +1331 -0
  13. package/dist/languages/he.js.map +1 -0
  14. package/dist/languages/hi.d.ts +36 -0
  15. package/dist/languages/hi.js +1162 -0
  16. package/dist/languages/hi.js.map +1 -0
  17. package/dist/languages/it.d.ts +53 -0
  18. package/dist/languages/it.js +1600 -0
  19. package/dist/languages/it.js.map +1 -0
  20. package/dist/languages/ms.d.ts +32 -0
  21. package/dist/languages/ms.js +1043 -0
  22. package/dist/languages/ms.js.map +1 -0
  23. package/dist/languages/pl.d.ts +37 -0
  24. package/dist/languages/pl.js +1331 -0
  25. package/dist/languages/pl.js.map +1 -0
  26. package/dist/languages/ru.d.ts +37 -0
  27. package/dist/languages/ru.js +1356 -0
  28. package/dist/languages/ru.js.map +1 -0
  29. package/dist/languages/th.d.ts +35 -0
  30. package/dist/languages/th.js +1076 -0
  31. package/dist/languages/th.js.map +1 -0
  32. package/dist/languages/tl.d.ts +32 -0
  33. package/dist/languages/tl.js +1034 -0
  34. package/dist/languages/tl.js.map +1 -0
  35. package/dist/languages/uk.d.ts +37 -0
  36. package/dist/languages/uk.js +1356 -0
  37. package/dist/languages/uk.js.map +1 -0
  38. package/dist/languages/vi.d.ts +59 -0
  39. package/dist/languages/vi.js +1220 -0
  40. package/dist/languages/vi.js.map +1 -0
  41. package/dist/spanish-BedpM-NU.d.ts +43 -0
  42. package/package.json +53 -1
  43. package/src/core.ts +155 -0
@@ -0,0 +1,1600 @@
1
+ // src/registry.ts
2
+ var tokenizers = /* @__PURE__ */ new Map();
3
+ var profiles = /* @__PURE__ */ new Map();
4
+ var patternCache = /* @__PURE__ */ new Map();
5
+ function registerLanguage(code, tokenizer, profile) {
6
+ tokenizers.set(code, tokenizer);
7
+ profiles.set(code, profile);
8
+ patternCache.delete(code);
9
+ }
10
+
11
+ // src/tokenizers/base.ts
12
+ var TokenStreamImpl = class {
13
+ constructor(tokens, language) {
14
+ this.pos = 0;
15
+ this.tokens = tokens;
16
+ this.language = language;
17
+ }
18
+ peek(offset = 0) {
19
+ const index = this.pos + offset;
20
+ if (index < 0 || index >= this.tokens.length) {
21
+ return null;
22
+ }
23
+ return this.tokens[index];
24
+ }
25
+ advance() {
26
+ if (this.isAtEnd()) {
27
+ throw new Error("Unexpected end of token stream");
28
+ }
29
+ return this.tokens[this.pos++];
30
+ }
31
+ isAtEnd() {
32
+ return this.pos >= this.tokens.length;
33
+ }
34
+ mark() {
35
+ return { position: this.pos };
36
+ }
37
+ reset(mark) {
38
+ this.pos = mark.position;
39
+ }
40
+ position() {
41
+ return this.pos;
42
+ }
43
+ /**
44
+ * Get remaining tokens as an array.
45
+ */
46
+ remaining() {
47
+ return this.tokens.slice(this.pos);
48
+ }
49
+ /**
50
+ * Consume tokens while predicate is true.
51
+ */
52
+ takeWhile(predicate) {
53
+ const result = [];
54
+ while (!this.isAtEnd() && predicate(this.peek())) {
55
+ result.push(this.advance());
56
+ }
57
+ return result;
58
+ }
59
+ /**
60
+ * Skip tokens while predicate is true.
61
+ */
62
+ skipWhile(predicate) {
63
+ while (!this.isAtEnd() && predicate(this.peek())) {
64
+ this.advance();
65
+ }
66
+ }
67
+ };
68
+ function createPosition(start, end) {
69
+ return { start, end };
70
+ }
71
+ function createToken(value, kind, position, normalizedOrOptions) {
72
+ if (typeof normalizedOrOptions === "string") {
73
+ return { value, kind, position, normalized: normalizedOrOptions };
74
+ }
75
+ if (normalizedOrOptions) {
76
+ const { normalized: normalized2, stem, stemConfidence } = normalizedOrOptions;
77
+ const token = { value, kind, position };
78
+ if (normalized2 !== void 0) {
79
+ token.normalized = normalized2;
80
+ }
81
+ if (stem !== void 0) {
82
+ token.stem = stem;
83
+ if (stemConfidence !== void 0) {
84
+ token.stemConfidence = stemConfidence;
85
+ }
86
+ }
87
+ return token;
88
+ }
89
+ return { value, kind, position };
90
+ }
91
+ function isWhitespace(char) {
92
+ return /\s/.test(char);
93
+ }
94
+ function isSelectorStart(char) {
95
+ return char === "#" || char === "." || char === "[" || char === "@" || char === "*" || char === "<";
96
+ }
97
+ function isQuote(char) {
98
+ return char === '"' || char === "'" || char === "`" || char === "\u300C" || char === "\u300D";
99
+ }
100
+ function isDigit(char) {
101
+ return /\d/.test(char);
102
+ }
103
+ function isAsciiLetter(char) {
104
+ return /[a-zA-Z]/.test(char);
105
+ }
106
+ function isAsciiIdentifierChar(char) {
107
+ return /[a-zA-Z0-9_-]/.test(char);
108
+ }
109
+ function createLatinCharClassifiers(letterPattern) {
110
+ const isLetter = (char) => letterPattern.test(char);
111
+ const isIdentifierChar = (char) => isLetter(char) || /[0-9_-]/.test(char);
112
+ return { isLetter, isIdentifierChar };
113
+ }
114
+ function extractCssSelector(input, startPos) {
115
+ if (startPos >= input.length) return null;
116
+ const char = input[startPos];
117
+ if (!isSelectorStart(char)) return null;
118
+ let pos = startPos;
119
+ let selector = "";
120
+ if (char === "#" || char === ".") {
121
+ selector += input[pos++];
122
+ while (pos < input.length && isAsciiIdentifierChar(input[pos])) {
123
+ selector += input[pos++];
124
+ }
125
+ if (selector.length <= 1) return null;
126
+ if (pos < input.length && input[pos] === "." && char === "#") {
127
+ const methodStart = pos + 1;
128
+ let methodEnd = methodStart;
129
+ while (methodEnd < input.length && isAsciiIdentifierChar(input[methodEnd])) {
130
+ methodEnd++;
131
+ }
132
+ if (methodEnd < input.length && input[methodEnd] === "(") {
133
+ return selector;
134
+ }
135
+ }
136
+ } else if (char === "[") {
137
+ let depth = 1;
138
+ let inQuote = false;
139
+ let quoteChar = null;
140
+ let escaped = false;
141
+ selector += input[pos++];
142
+ while (pos < input.length && depth > 0) {
143
+ const c = input[pos];
144
+ selector += c;
145
+ if (escaped) {
146
+ escaped = false;
147
+ } else if (c === "\\") {
148
+ escaped = true;
149
+ } else if (inQuote) {
150
+ if (c === quoteChar) {
151
+ inQuote = false;
152
+ quoteChar = null;
153
+ }
154
+ } else {
155
+ if (c === '"' || c === "'" || c === "`") {
156
+ inQuote = true;
157
+ quoteChar = c;
158
+ } else if (c === "[") {
159
+ depth++;
160
+ } else if (c === "]") {
161
+ depth--;
162
+ }
163
+ }
164
+ pos++;
165
+ }
166
+ if (depth !== 0) return null;
167
+ } else if (char === "@") {
168
+ selector += input[pos++];
169
+ while (pos < input.length && isAsciiIdentifierChar(input[pos])) {
170
+ selector += input[pos++];
171
+ }
172
+ if (selector.length <= 1) return null;
173
+ } else if (char === "*") {
174
+ selector += input[pos++];
175
+ while (pos < input.length && isAsciiIdentifierChar(input[pos])) {
176
+ selector += input[pos++];
177
+ }
178
+ if (selector.length <= 1) return null;
179
+ } else if (char === "<") {
180
+ selector += input[pos++];
181
+ if (pos >= input.length || !isAsciiLetter(input[pos])) return null;
182
+ while (pos < input.length && isAsciiIdentifierChar(input[pos])) {
183
+ selector += input[pos++];
184
+ }
185
+ while (pos < input.length) {
186
+ const modChar = input[pos];
187
+ if (modChar === ".") {
188
+ selector += input[pos++];
189
+ if (pos >= input.length || !isAsciiIdentifierChar(input[pos])) {
190
+ return null;
191
+ }
192
+ while (pos < input.length && isAsciiIdentifierChar(input[pos])) {
193
+ selector += input[pos++];
194
+ }
195
+ } else if (modChar === "#") {
196
+ selector += input[pos++];
197
+ if (pos >= input.length || !isAsciiIdentifierChar(input[pos])) {
198
+ return null;
199
+ }
200
+ while (pos < input.length && isAsciiIdentifierChar(input[pos])) {
201
+ selector += input[pos++];
202
+ }
203
+ } else if (modChar === "[") {
204
+ let depth = 1;
205
+ let inQuote = false;
206
+ let quoteChar = null;
207
+ let escaped = false;
208
+ selector += input[pos++];
209
+ while (pos < input.length && depth > 0) {
210
+ const c = input[pos];
211
+ selector += c;
212
+ if (escaped) {
213
+ escaped = false;
214
+ } else if (c === "\\") {
215
+ escaped = true;
216
+ } else if (inQuote) {
217
+ if (c === quoteChar) {
218
+ inQuote = false;
219
+ quoteChar = null;
220
+ }
221
+ } else {
222
+ if (c === '"' || c === "'" || c === "`") {
223
+ inQuote = true;
224
+ quoteChar = c;
225
+ } else if (c === "[") {
226
+ depth++;
227
+ } else if (c === "]") {
228
+ depth--;
229
+ }
230
+ }
231
+ pos++;
232
+ }
233
+ if (depth !== 0) return null;
234
+ } else {
235
+ break;
236
+ }
237
+ }
238
+ while (pos < input.length && isWhitespace(input[pos])) {
239
+ selector += input[pos++];
240
+ }
241
+ if (pos < input.length && input[pos] === "/") {
242
+ selector += input[pos++];
243
+ while (pos < input.length && isWhitespace(input[pos])) {
244
+ selector += input[pos++];
245
+ }
246
+ }
247
+ if (pos >= input.length || input[pos] !== ">") return null;
248
+ selector += input[pos++];
249
+ }
250
+ return selector || null;
251
+ }
252
+ function isPossessiveMarker(input, pos) {
253
+ if (pos >= input.length || input[pos] !== "'") return false;
254
+ if (pos + 1 >= input.length) return false;
255
+ const nextChar = input[pos + 1].toLowerCase();
256
+ if (nextChar !== "s") return false;
257
+ if (pos + 2 >= input.length) return true;
258
+ const afterS = input[pos + 2];
259
+ return isWhitespace(afterS) || afterS === "*" || !isAsciiIdentifierChar(afterS);
260
+ }
261
+ function extractStringLiteral(input, startPos) {
262
+ if (startPos >= input.length) return null;
263
+ const openQuote = input[startPos];
264
+ if (!isQuote(openQuote)) return null;
265
+ if (openQuote === "'" && isPossessiveMarker(input, startPos)) {
266
+ return null;
267
+ }
268
+ const closeQuoteMap = {
269
+ '"': '"',
270
+ "'": "'",
271
+ "`": "`",
272
+ "\u300C": "\u300D"
273
+ };
274
+ const closeQuote = closeQuoteMap[openQuote];
275
+ if (!closeQuote) return null;
276
+ let pos = startPos + 1;
277
+ let literal = openQuote;
278
+ let escaped = false;
279
+ while (pos < input.length) {
280
+ const char = input[pos];
281
+ literal += char;
282
+ if (escaped) {
283
+ escaped = false;
284
+ } else if (char === "\\") {
285
+ escaped = true;
286
+ } else if (char === closeQuote) {
287
+ return literal;
288
+ }
289
+ pos++;
290
+ }
291
+ return literal;
292
+ }
293
+ function isUrlStart(input, pos) {
294
+ if (pos >= input.length) return false;
295
+ const char = input[pos];
296
+ const next = input[pos + 1] || "";
297
+ const third = input[pos + 2] || "";
298
+ if (char === "/" && next !== "/" && /[a-zA-Z0-9._-]/.test(next)) {
299
+ return true;
300
+ }
301
+ if (char === "/" && next === "/" && /[a-zA-Z]/.test(third)) {
302
+ return true;
303
+ }
304
+ if (char === "." && (next === "/" || next === "." && third === "/")) {
305
+ return true;
306
+ }
307
+ const slice = input.slice(pos, pos + 8).toLowerCase();
308
+ if (slice.startsWith("http://") || slice.startsWith("https://")) {
309
+ return true;
310
+ }
311
+ return false;
312
+ }
313
+ function extractUrl(input, startPos) {
314
+ if (!isUrlStart(input, startPos)) return null;
315
+ let pos = startPos;
316
+ let url = "";
317
+ const urlChars = /[a-zA-Z0-9/:._\-?&=%@+~!$'()*,;[\]]/;
318
+ while (pos < input.length) {
319
+ const char = input[pos];
320
+ if (char === "#") {
321
+ if (url.length > 0 && /[a-zA-Z0-9/.]$/.test(url)) {
322
+ url += char;
323
+ pos++;
324
+ while (pos < input.length && /[a-zA-Z0-9_-]/.test(input[pos])) {
325
+ url += input[pos++];
326
+ }
327
+ }
328
+ break;
329
+ }
330
+ if (urlChars.test(char)) {
331
+ url += char;
332
+ pos++;
333
+ } else {
334
+ break;
335
+ }
336
+ }
337
+ if (url.length < 2) return null;
338
+ return url;
339
+ }
340
+ function extractNumber(input, startPos) {
341
+ if (startPos >= input.length) return null;
342
+ const char = input[startPos];
343
+ if (!isDigit(char) && char !== "-" && char !== "+") return null;
344
+ let pos = startPos;
345
+ let number = "";
346
+ if (input[pos] === "-" || input[pos] === "+") {
347
+ number += input[pos++];
348
+ }
349
+ if (pos >= input.length || !isDigit(input[pos])) {
350
+ return null;
351
+ }
352
+ while (pos < input.length && isDigit(input[pos])) {
353
+ number += input[pos++];
354
+ }
355
+ if (pos < input.length && input[pos] === ".") {
356
+ number += input[pos++];
357
+ while (pos < input.length && isDigit(input[pos])) {
358
+ number += input[pos++];
359
+ }
360
+ }
361
+ if (pos < input.length) {
362
+ const suffix = input.slice(pos, pos + 2);
363
+ if (suffix === "ms") {
364
+ number += "ms";
365
+ } else if (input[pos] === "s" || input[pos] === "m" || input[pos] === "h") {
366
+ number += input[pos];
367
+ }
368
+ }
369
+ return number;
370
+ }
371
+ var _BaseTokenizer = class _BaseTokenizer {
372
+ constructor() {
373
+ /** Keywords derived from profile, sorted longest-first for greedy matching */
374
+ this.profileKeywords = [];
375
+ /** Map for O(1) keyword lookups by lowercase native word */
376
+ this.profileKeywordMap = /* @__PURE__ */ new Map();
377
+ }
378
+ /**
379
+ * Initialize keyword mappings from a language profile.
380
+ * Builds a list of native→english mappings from:
381
+ * - profile.keywords (primary + alternatives)
382
+ * - profile.references (me, it, you, etc.)
383
+ * - profile.roleMarkers (into, from, with, etc.)
384
+ *
385
+ * Results are sorted longest-first for greedy matching (important for non-space languages).
386
+ * Extras take precedence over profile entries when there are duplicates.
387
+ *
388
+ * @param profile - Language profile containing keyword translations
389
+ * @param extras - Additional keyword entries to include (literals, positional, events)
390
+ */
391
+ initializeKeywordsFromProfile(profile, extras = []) {
392
+ const keywordMap = /* @__PURE__ */ new Map();
393
+ if (profile.keywords) {
394
+ for (const [normalized2, translation] of Object.entries(profile.keywords)) {
395
+ keywordMap.set(translation.primary, {
396
+ native: translation.primary,
397
+ normalized: translation.normalized || normalized2
398
+ });
399
+ if (translation.alternatives) {
400
+ for (const alt of translation.alternatives) {
401
+ keywordMap.set(alt, {
402
+ native: alt,
403
+ normalized: translation.normalized || normalized2
404
+ });
405
+ }
406
+ }
407
+ }
408
+ }
409
+ if (profile.references) {
410
+ for (const [normalized2, native] of Object.entries(profile.references)) {
411
+ keywordMap.set(native, { native, normalized: normalized2 });
412
+ }
413
+ }
414
+ if (profile.roleMarkers) {
415
+ for (const [role, marker] of Object.entries(profile.roleMarkers)) {
416
+ if (marker.primary) {
417
+ keywordMap.set(marker.primary, { native: marker.primary, normalized: role });
418
+ }
419
+ if (marker.alternatives) {
420
+ for (const alt of marker.alternatives) {
421
+ keywordMap.set(alt, { native: alt, normalized: role });
422
+ }
423
+ }
424
+ }
425
+ }
426
+ for (const extra of extras) {
427
+ keywordMap.set(extra.native, extra);
428
+ }
429
+ this.profileKeywords = Array.from(keywordMap.values()).sort(
430
+ (a, b) => b.native.length - a.native.length
431
+ );
432
+ this.profileKeywordMap = /* @__PURE__ */ new Map();
433
+ for (const keyword of this.profileKeywords) {
434
+ this.profileKeywordMap.set(keyword.native.toLowerCase(), keyword);
435
+ const normalized2 = this.removeDiacritics(keyword.native);
436
+ if (normalized2 !== keyword.native && !this.profileKeywordMap.has(normalized2.toLowerCase())) {
437
+ this.profileKeywordMap.set(normalized2.toLowerCase(), keyword);
438
+ }
439
+ }
440
+ }
441
+ /**
442
+ * Remove diacritical marks from a word for normalization.
443
+ * Primarily for Arabic (shadda, fatha, kasra, damma, sukun, etc.)
444
+ * but could be extended for other languages.
445
+ *
446
+ * @param word - Word to normalize
447
+ * @returns Word without diacritics
448
+ */
449
+ removeDiacritics(word) {
450
+ return word.replace(/[\u064B-\u0652\u0670]/g, "");
451
+ }
452
+ /**
453
+ * Try to match a keyword from profile at the current position.
454
+ * Uses longest-first greedy matching (important for non-space languages).
455
+ *
456
+ * @param input - Input string
457
+ * @param pos - Current position
458
+ * @returns Token if matched, null otherwise
459
+ */
460
+ tryProfileKeyword(input, pos) {
461
+ for (const entry of this.profileKeywords) {
462
+ if (input.slice(pos).startsWith(entry.native)) {
463
+ return createToken(
464
+ entry.native,
465
+ "keyword",
466
+ createPosition(pos, pos + entry.native.length),
467
+ entry.normalized
468
+ );
469
+ }
470
+ }
471
+ return null;
472
+ }
473
+ /**
474
+ * Check if the remaining input starts with any known keyword.
475
+ * Useful for non-space languages to detect word boundaries.
476
+ *
477
+ * @param input - Input string
478
+ * @param pos - Current position
479
+ * @returns true if a keyword starts at this position
480
+ */
481
+ isKeywordStart(input, pos) {
482
+ const remaining = input.slice(pos);
483
+ return this.profileKeywords.some((entry) => remaining.startsWith(entry.native));
484
+ }
485
+ /**
486
+ * Look up a keyword by native word (case-insensitive).
487
+ * O(1) lookup using the keyword map.
488
+ *
489
+ * @param native - Native word to look up
490
+ * @returns KeywordEntry if found, undefined otherwise
491
+ */
492
+ lookupKeyword(native) {
493
+ return this.profileKeywordMap.get(native.toLowerCase());
494
+ }
495
+ /**
496
+ * Check if a word is a known keyword (case-insensitive).
497
+ * O(1) lookup using the keyword map.
498
+ *
499
+ * @param native - Native word to check
500
+ * @returns true if the word is a keyword
501
+ */
502
+ isKeyword(native) {
503
+ return this.profileKeywordMap.has(native.toLowerCase());
504
+ }
505
+ /**
506
+ * Set the morphological normalizer for this tokenizer.
507
+ */
508
+ setNormalizer(normalizer) {
509
+ this.normalizer = normalizer;
510
+ }
511
+ /**
512
+ * Try to normalize a word using the morphological normalizer.
513
+ * Returns null if no normalizer is set or normalization fails.
514
+ *
515
+ * Note: We don't check isNormalizable() here because the individual tokenizers
516
+ * historically called normalize() directly without that check. The normalize()
517
+ * method itself handles returning noChange() for words that can't be normalized.
518
+ */
519
+ tryNormalize(word) {
520
+ if (!this.normalizer) return null;
521
+ const result = this.normalizer.normalize(word);
522
+ if (result.stem !== word && result.confidence >= 0.7) {
523
+ return result;
524
+ }
525
+ return null;
526
+ }
527
+ /**
528
+ * Try morphological normalization and keyword lookup.
529
+ *
530
+ * If the word can be normalized to a stem that matches a known keyword,
531
+ * returns a keyword token with morphological metadata (stem, stemConfidence).
532
+ *
533
+ * This is the common pattern for handling conjugated verbs across languages:
534
+ * 1. Normalize the word (e.g., "toggled" → "toggle")
535
+ * 2. Look up the stem in the keyword map
536
+ * 3. Create a token with both the original form and stem metadata
537
+ *
538
+ * @param word - The word to normalize and look up
539
+ * @param startPos - Start position for the token
540
+ * @param endPos - End position for the token
541
+ * @returns Token if stem matches a keyword, null otherwise
542
+ */
543
+ tryMorphKeywordMatch(word, startPos, endPos) {
544
+ const result = this.tryNormalize(word);
545
+ if (!result) return null;
546
+ const stemEntry = this.lookupKeyword(result.stem);
547
+ if (!stemEntry) return null;
548
+ const tokenOptions = {
549
+ normalized: stemEntry.normalized,
550
+ stem: result.stem,
551
+ stemConfidence: result.confidence
552
+ };
553
+ return createToken(word, "keyword", createPosition(startPos, endPos), tokenOptions);
554
+ }
555
+ /**
556
+ * Try to extract a CSS selector at the current position.
557
+ */
558
+ trySelector(input, pos) {
559
+ const selector = extractCssSelector(input, pos);
560
+ if (selector) {
561
+ return createToken(selector, "selector", createPosition(pos, pos + selector.length));
562
+ }
563
+ return null;
564
+ }
565
+ /**
566
+ * Try to extract an event modifier at the current position.
567
+ * Event modifiers are .once, .debounce(N), .throttle(N), .queue(strategy)
568
+ */
569
+ tryEventModifier(input, pos) {
570
+ if (input[pos] !== ".") {
571
+ return null;
572
+ }
573
+ const match = input.slice(pos).match(/^\.(?:once|debounce|throttle|queue)(?:\(([^)]+)\))?(?:\s|$|\.)/);
574
+ if (!match) {
575
+ return null;
576
+ }
577
+ const fullMatch = match[0].replace(/(\s|\.)$/, "");
578
+ const modifierName = fullMatch.slice(1).split("(")[0];
579
+ const value = match[1];
580
+ const token = createToken(
581
+ fullMatch,
582
+ "event-modifier",
583
+ createPosition(pos, pos + fullMatch.length)
584
+ );
585
+ return {
586
+ ...token,
587
+ metadata: {
588
+ modifierName,
589
+ value: value ? modifierName === "queue" ? value : parseInt(value, 10) : void 0
590
+ }
591
+ };
592
+ }
593
+ /**
594
+ * Try to extract a string literal at the current position.
595
+ */
596
+ tryString(input, pos) {
597
+ const literal = extractStringLiteral(input, pos);
598
+ if (literal) {
599
+ return createToken(literal, "literal", createPosition(pos, pos + literal.length));
600
+ }
601
+ return null;
602
+ }
603
+ /**
604
+ * Try to extract a number at the current position.
605
+ */
606
+ tryNumber(input, pos) {
607
+ const number = extractNumber(input, pos);
608
+ if (number) {
609
+ return createToken(number, "literal", createPosition(pos, pos + number.length));
610
+ }
611
+ return null;
612
+ }
613
+ /**
614
+ * Try to match a time unit from a list of patterns.
615
+ *
616
+ * @param input - Input string
617
+ * @param pos - Position after the number
618
+ * @param timeUnits - Array of time unit mappings (native pattern → standard suffix)
619
+ * @param skipWhitespace - Whether to skip whitespace before time unit (default: false)
620
+ * @returns Object with matched suffix and new position, or null if no match
621
+ */
622
+ tryMatchTimeUnit(input, pos, timeUnits, skipWhitespace = false) {
623
+ let unitPos = pos;
624
+ if (skipWhitespace) {
625
+ while (unitPos < input.length && isWhitespace(input[unitPos])) {
626
+ unitPos++;
627
+ }
628
+ }
629
+ const remaining = input.slice(unitPos);
630
+ for (const unit of timeUnits) {
631
+ const candidate = remaining.slice(0, unit.length);
632
+ const matches = unit.caseInsensitive ? candidate.toLowerCase() === unit.pattern.toLowerCase() : candidate === unit.pattern;
633
+ if (matches) {
634
+ if (unit.notFollowedBy) {
635
+ const nextChar = remaining[unit.length] || "";
636
+ if (nextChar === unit.notFollowedBy) continue;
637
+ }
638
+ if (unit.checkBoundary) {
639
+ const nextChar = remaining[unit.length] || "";
640
+ if (isAsciiIdentifierChar(nextChar)) continue;
641
+ }
642
+ return { suffix: unit.suffix, endPos: unitPos + unit.length };
643
+ }
644
+ }
645
+ return null;
646
+ }
647
+ /**
648
+ * Parse a base number (sign, integer, decimal) without time units.
649
+ * Returns the number string and end position.
650
+ *
651
+ * @param input - Input string
652
+ * @param startPos - Start position
653
+ * @param allowSign - Whether to allow +/- sign (default: true)
654
+ * @returns Object with number string and end position, or null
655
+ */
656
+ parseBaseNumber(input, startPos, allowSign = true) {
657
+ let pos = startPos;
658
+ let number = "";
659
+ if (allowSign && (input[pos] === "-" || input[pos] === "+")) {
660
+ number += input[pos++];
661
+ }
662
+ if (pos >= input.length || !isDigit(input[pos])) {
663
+ return null;
664
+ }
665
+ while (pos < input.length && isDigit(input[pos])) {
666
+ number += input[pos++];
667
+ }
668
+ if (pos < input.length && input[pos] === ".") {
669
+ number += input[pos++];
670
+ while (pos < input.length && isDigit(input[pos])) {
671
+ number += input[pos++];
672
+ }
673
+ }
674
+ if (!number || number === "-" || number === "+") return null;
675
+ return { number, endPos: pos };
676
+ }
677
+ /**
678
+ * Try to extract a number with native language time units.
679
+ *
680
+ * This is a template method that handles the common pattern:
681
+ * 1. Parse the base number (sign, integer, decimal)
682
+ * 2. Try to match native language time units
683
+ * 3. Fall back to standard time units (ms, s, m, h)
684
+ *
685
+ * @param input - Input string
686
+ * @param pos - Start position
687
+ * @param nativeTimeUnits - Language-specific time unit mappings
688
+ * @param options - Configuration options
689
+ * @returns Token if number found, null otherwise
690
+ */
691
+ tryNumberWithTimeUnits(input, pos, nativeTimeUnits, options = {}) {
692
+ const { allowSign = true, skipWhitespace = false } = options;
693
+ const baseResult = this.parseBaseNumber(input, pos, allowSign);
694
+ if (!baseResult) return null;
695
+ let { number, endPos } = baseResult;
696
+ const allUnits = [...nativeTimeUnits, ..._BaseTokenizer.STANDARD_TIME_UNITS];
697
+ const timeMatch = this.tryMatchTimeUnit(input, endPos, allUnits, skipWhitespace);
698
+ if (timeMatch) {
699
+ number += timeMatch.suffix;
700
+ endPos = timeMatch.endPos;
701
+ }
702
+ return createToken(number, "literal", createPosition(pos, endPos));
703
+ }
704
+ /**
705
+ * Try to extract a URL at the current position.
706
+ * Handles /path, ./path, ../path, //domain.com, http://, https://
707
+ */
708
+ tryUrl(input, pos) {
709
+ const url = extractUrl(input, pos);
710
+ if (url) {
711
+ return createToken(url, "url", createPosition(pos, pos + url.length));
712
+ }
713
+ return null;
714
+ }
715
+ /**
716
+ * Try to extract a variable reference (:varname) at the current position.
717
+ * In hyperscript, :x refers to a local variable named x.
718
+ */
719
+ tryVariableRef(input, pos) {
720
+ if (input[pos] !== ":") return null;
721
+ if (pos + 1 >= input.length) return null;
722
+ if (!isAsciiIdentifierChar(input[pos + 1])) return null;
723
+ let endPos = pos + 1;
724
+ while (endPos < input.length && isAsciiIdentifierChar(input[endPos])) {
725
+ endPos++;
726
+ }
727
+ const varRef = input.slice(pos, endPos);
728
+ return createToken(varRef, "identifier", createPosition(pos, endPos));
729
+ }
730
+ /**
731
+ * Try to extract an operator or punctuation token at the current position.
732
+ * Handles two-character operators (==, !=, etc.) and single-character operators.
733
+ */
734
+ tryOperator(input, pos) {
735
+ const twoChar = input.slice(pos, pos + 2);
736
+ if (["==", "!=", "<=", ">=", "&&", "||", "->"].includes(twoChar)) {
737
+ return createToken(twoChar, "operator", createPosition(pos, pos + 2));
738
+ }
739
+ const oneChar = input[pos];
740
+ if (["<", ">", "!", "+", "-", "*", "/", "="].includes(oneChar)) {
741
+ return createToken(oneChar, "operator", createPosition(pos, pos + 1));
742
+ }
743
+ if (["(", ")", "{", "}", ",", ";", ":"].includes(oneChar)) {
744
+ return createToken(oneChar, "punctuation", createPosition(pos, pos + 1));
745
+ }
746
+ return null;
747
+ }
748
+ /**
749
+ * Try to match a multi-character particle from a list.
750
+ *
751
+ * Used by languages like Japanese, Korean, and Chinese that have
752
+ * multi-character particles (e.g., Japanese から, まで, より).
753
+ *
754
+ * @param input - Input string
755
+ * @param pos - Current position
756
+ * @param particles - Array of multi-character particles to match
757
+ * @returns Token if matched, null otherwise
758
+ */
759
+ tryMultiCharParticle(input, pos, particles) {
760
+ for (const particle of particles) {
761
+ if (input.slice(pos, pos + particle.length) === particle) {
762
+ return createToken(particle, "particle", createPosition(pos, pos + particle.length));
763
+ }
764
+ }
765
+ return null;
766
+ }
767
+ };
768
+ /**
769
+ * Configuration for native language time units.
770
+ * Maps patterns to their standard suffix (ms, s, m, h).
771
+ */
772
+ _BaseTokenizer.STANDARD_TIME_UNITS = [
773
+ { pattern: "ms", suffix: "ms", length: 2 },
774
+ { pattern: "s", suffix: "s", length: 1, checkBoundary: true },
775
+ { pattern: "m", suffix: "m", length: 1, checkBoundary: true, notFollowedBy: "s" },
776
+ { pattern: "h", suffix: "h", length: 1, checkBoundary: true }
777
+ ];
778
+ var BaseTokenizer = _BaseTokenizer;
779
+
780
+ // src/tokenizers/morphology/types.ts
781
+ function noChange(word) {
782
+ return { stem: word, confidence: 1 };
783
+ }
784
+ function normalized(stem, confidence, metadata) {
785
+ if (metadata) {
786
+ return { stem, confidence, metadata };
787
+ }
788
+ return { stem, confidence };
789
+ }
790
+
791
+ // src/tokenizers/morphology/italian-normalizer.ts
792
+ function isItalianSpecificLetter(char) {
793
+ return /[àèéìíîòóùúÀÈÉÌÍÎÒÓÙÚ]/.test(char);
794
+ }
795
+ function looksLikeItalianVerb(word) {
796
+ const lower = word.toLowerCase();
797
+ if (lower.endsWith("are") || lower.endsWith("ere") || lower.endsWith("ire")) return true;
798
+ if (lower.endsWith("ando") || lower.endsWith("endo")) return true;
799
+ if (lower.endsWith("ato") || lower.endsWith("uto") || lower.endsWith("ito")) return true;
800
+ if (lower.endsWith("arsi") || lower.endsWith("ersi") || lower.endsWith("irsi")) return true;
801
+ for (const char of word) {
802
+ if (isItalianSpecificLetter(char)) return true;
803
+ }
804
+ return false;
805
+ }
806
+ var REFLEXIVE_SUFFIXES = ["si", "mi", "ti", "ci", "vi"];
807
+ var ARE_ENDINGS = [
808
+ // Gerund (-ando)
809
+ { ending: "ando", stem: "are", confidence: 0.88, type: "gerund" },
810
+ // Past participle (-ato)
811
+ { ending: "ato", stem: "are", confidence: 0.88, type: "participle" },
812
+ { ending: "ata", stem: "are", confidence: 0.88, type: "participle" },
813
+ { ending: "ati", stem: "are", confidence: 0.88, type: "participle" },
814
+ { ending: "ate", stem: "are", confidence: 0.88, type: "participle" },
815
+ // Present indicative
816
+ { ending: "o", stem: "are", confidence: 0.75, type: "present" },
817
+ // io
818
+ { ending: "i", stem: "are", confidence: 0.72, type: "present" },
819
+ // tu
820
+ { ending: "a", stem: "are", confidence: 0.75, type: "present" },
821
+ // lui/lei
822
+ { ending: "iamo", stem: "are", confidence: 0.85, type: "present" },
823
+ // noi
824
+ { ending: "ate", stem: "are", confidence: 0.85, type: "present" },
825
+ // voi
826
+ { ending: "ano", stem: "are", confidence: 0.85, type: "present" },
827
+ // loro
828
+ // Imperfect
829
+ { ending: "avo", stem: "are", confidence: 0.88, type: "past" },
830
+ // io
831
+ { ending: "avi", stem: "are", confidence: 0.88, type: "past" },
832
+ // tu
833
+ { ending: "ava", stem: "are", confidence: 0.88, type: "past" },
834
+ // lui/lei
835
+ { ending: "avamo", stem: "are", confidence: 0.88, type: "past" },
836
+ // noi
837
+ { ending: "avate", stem: "are", confidence: 0.88, type: "past" },
838
+ // voi
839
+ { ending: "avano", stem: "are", confidence: 0.88, type: "past" },
840
+ // loro
841
+ // Preterite (passato remoto)
842
+ { ending: "ai", stem: "are", confidence: 0.85, type: "past" },
843
+ // io
844
+ { ending: "asti", stem: "are", confidence: 0.88, type: "past" },
845
+ // tu
846
+ { ending: "\xF2", stem: "are", confidence: 0.85, type: "past" },
847
+ // lui/lei
848
+ { ending: "ammo", stem: "are", confidence: 0.88, type: "past" },
849
+ // noi
850
+ { ending: "aste", stem: "are", confidence: 0.88, type: "past" },
851
+ // voi
852
+ { ending: "arono", stem: "are", confidence: 0.88, type: "past" },
853
+ // loro
854
+ // Subjunctive present
855
+ { ending: "i", stem: "are", confidence: 0.72, type: "subjunctive" },
856
+ // io/tu/lui (ambiguous)
857
+ { ending: "ino", stem: "are", confidence: 0.82, type: "subjunctive" },
858
+ // loro
859
+ // Imperative
860
+ { ending: "a", stem: "are", confidence: 0.75, type: "imperative" },
861
+ // tu
862
+ // Infinitive
863
+ { ending: "are", stem: "are", confidence: 0.92, type: "dictionary" }
864
+ ];
865
+ var ERE_ENDINGS = [
866
+ // Gerund (-endo)
867
+ { ending: "endo", stem: "ere", confidence: 0.88, type: "gerund" },
868
+ // Past participle (-uto)
869
+ { ending: "uto", stem: "ere", confidence: 0.85, type: "participle" },
870
+ { ending: "uta", stem: "ere", confidence: 0.85, type: "participle" },
871
+ { ending: "uti", stem: "ere", confidence: 0.85, type: "participle" },
872
+ { ending: "ute", stem: "ere", confidence: 0.85, type: "participle" },
873
+ // Present indicative
874
+ { ending: "o", stem: "ere", confidence: 0.72, type: "present" },
875
+ // io
876
+ { ending: "i", stem: "ere", confidence: 0.72, type: "present" },
877
+ // tu
878
+ { ending: "e", stem: "ere", confidence: 0.72, type: "present" },
879
+ // lui/lei
880
+ { ending: "iamo", stem: "ere", confidence: 0.85, type: "present" },
881
+ // noi
882
+ { ending: "ete", stem: "ere", confidence: 0.85, type: "present" },
883
+ // voi
884
+ { ending: "ono", stem: "ere", confidence: 0.82, type: "present" },
885
+ // loro
886
+ // Imperfect
887
+ { ending: "evo", stem: "ere", confidence: 0.88, type: "past" },
888
+ // io
889
+ { ending: "evi", stem: "ere", confidence: 0.88, type: "past" },
890
+ // tu
891
+ { ending: "eva", stem: "ere", confidence: 0.88, type: "past" },
892
+ // lui/lei
893
+ { ending: "evamo", stem: "ere", confidence: 0.88, type: "past" },
894
+ // noi
895
+ { ending: "evate", stem: "ere", confidence: 0.88, type: "past" },
896
+ // voi
897
+ { ending: "evano", stem: "ere", confidence: 0.88, type: "past" },
898
+ // loro
899
+ // Preterite
900
+ { ending: "ei", stem: "ere", confidence: 0.85, type: "past" },
901
+ // io
902
+ { ending: "etti", stem: "ere", confidence: 0.85, type: "past" },
903
+ // io (variant)
904
+ { ending: "esti", stem: "ere", confidence: 0.88, type: "past" },
905
+ // tu
906
+ { ending: "\xE9", stem: "ere", confidence: 0.85, type: "past" },
907
+ // lui/lei
908
+ { ending: "ette", stem: "ere", confidence: 0.85, type: "past" },
909
+ // lui/lei (variant)
910
+ { ending: "emmo", stem: "ere", confidence: 0.88, type: "past" },
911
+ // noi
912
+ { ending: "este", stem: "ere", confidence: 0.88, type: "past" },
913
+ // voi
914
+ { ending: "erono", stem: "ere", confidence: 0.88, type: "past" },
915
+ // loro
916
+ { ending: "ettero", stem: "ere", confidence: 0.88, type: "past" },
917
+ // loro (variant)
918
+ // Infinitive
919
+ { ending: "ere", stem: "ere", confidence: 0.92, type: "dictionary" }
920
+ ];
921
+ var IRE_ENDINGS = [
922
+ // Gerund (-endo)
923
+ { ending: "endo", stem: "ire", confidence: 0.85, type: "gerund" },
924
+ // Past participle (-ito)
925
+ { ending: "ito", stem: "ire", confidence: 0.85, type: "participle" },
926
+ { ending: "ita", stem: "ire", confidence: 0.85, type: "participle" },
927
+ { ending: "iti", stem: "ire", confidence: 0.85, type: "participle" },
928
+ { ending: "ite", stem: "ire", confidence: 0.85, type: "participle" },
929
+ // Present indicative (standard)
930
+ { ending: "o", stem: "ire", confidence: 0.7, type: "present" },
931
+ // io
932
+ { ending: "i", stem: "ire", confidence: 0.7, type: "present" },
933
+ // tu
934
+ { ending: "e", stem: "ire", confidence: 0.7, type: "present" },
935
+ // lui/lei
936
+ { ending: "iamo", stem: "ire", confidence: 0.85, type: "present" },
937
+ // noi
938
+ { ending: "ite", stem: "ire", confidence: 0.85, type: "present" },
939
+ // voi
940
+ { ending: "ono", stem: "ire", confidence: 0.78, type: "present" },
941
+ // loro
942
+ // Present indicative (-isco verbs)
943
+ { ending: "isco", stem: "ire", confidence: 0.85, type: "present" },
944
+ // io
945
+ { ending: "isci", stem: "ire", confidence: 0.85, type: "present" },
946
+ // tu
947
+ { ending: "isce", stem: "ire", confidence: 0.85, type: "present" },
948
+ // lui/lei
949
+ { ending: "iscono", stem: "ire", confidence: 0.88, type: "present" },
950
+ // loro
951
+ // Imperfect
952
+ { ending: "ivo", stem: "ire", confidence: 0.88, type: "past" },
953
+ // io
954
+ { ending: "ivi", stem: "ire", confidence: 0.88, type: "past" },
955
+ // tu
956
+ { ending: "iva", stem: "ire", confidence: 0.88, type: "past" },
957
+ // lui/lei
958
+ { ending: "ivamo", stem: "ire", confidence: 0.88, type: "past" },
959
+ // noi
960
+ { ending: "ivate", stem: "ire", confidence: 0.88, type: "past" },
961
+ // voi
962
+ { ending: "ivano", stem: "ire", confidence: 0.88, type: "past" },
963
+ // loro
964
+ // Preterite
965
+ { ending: "ii", stem: "ire", confidence: 0.85, type: "past" },
966
+ // io
967
+ { ending: "isti", stem: "ire", confidence: 0.88, type: "past" },
968
+ // tu
969
+ { ending: "\xEC", stem: "ire", confidence: 0.85, type: "past" },
970
+ // lui/lei
971
+ { ending: "immo", stem: "ire", confidence: 0.88, type: "past" },
972
+ // noi
973
+ { ending: "iste", stem: "ire", confidence: 0.88, type: "past" },
974
+ // voi
975
+ { ending: "irono", stem: "ire", confidence: 0.88, type: "past" },
976
+ // loro
977
+ // Infinitive
978
+ { ending: "ire", stem: "ire", confidence: 0.92, type: "dictionary" }
979
+ ];
980
+ var ALL_ENDINGS = [...ARE_ENDINGS, ...ERE_ENDINGS, ...IRE_ENDINGS].sort(
981
+ (a, b) => b.ending.length - a.ending.length
982
+ );
983
+ var ItalianMorphologicalNormalizer = class {
984
+ constructor() {
985
+ this.language = "it";
986
+ }
987
+ /**
988
+ * Check if a word might be an Italian verb that can be normalized.
989
+ */
990
+ isNormalizable(word) {
991
+ if (word.length < 3) return false;
992
+ return looksLikeItalianVerb(word);
993
+ }
994
+ /**
995
+ * Normalize an Italian word to its infinitive form.
996
+ */
997
+ normalize(word) {
998
+ const lower = word.toLowerCase();
999
+ if (lower.endsWith("are") || lower.endsWith("ere") || lower.endsWith("ire")) {
1000
+ if (!REFLEXIVE_SUFFIXES.some(
1001
+ (s) => lower.endsWith(s + "are") || lower.endsWith(s + "ere") || lower.endsWith(s + "ire")
1002
+ )) {
1003
+ return noChange(word);
1004
+ }
1005
+ }
1006
+ const reflexiveResult = this.tryReflexiveNormalization(lower);
1007
+ if (reflexiveResult) return reflexiveResult;
1008
+ const conjugationResult = this.tryConjugationNormalization(lower);
1009
+ if (conjugationResult) return conjugationResult;
1010
+ return noChange(word);
1011
+ }
1012
+ /**
1013
+ * Try to normalize a reflexive verb.
1014
+ * Reflexive verbs end with -si, -mi, -ti, -ci, -vi attached to infinitive.
1015
+ *
1016
+ * In Italian, reflexive infinitives drop the final -e before attaching the pronoun:
1017
+ * mostrare + si → mostrarsi (not mostraresi)
1018
+ * nascondere + si → nascondersi
1019
+ *
1020
+ * Examples:
1021
+ * mostrarsi → mostrare
1022
+ * nascondersi → nascondere
1023
+ */
1024
+ tryReflexiveNormalization(word) {
1025
+ for (const suffix of REFLEXIVE_SUFFIXES) {
1026
+ if (word.endsWith(suffix)) {
1027
+ const withoutReflexive = word.slice(0, -suffix.length);
1028
+ if (withoutReflexive.endsWith("ar") || withoutReflexive.endsWith("er") || withoutReflexive.endsWith("ir")) {
1029
+ const infinitive = withoutReflexive + "e";
1030
+ return normalized(infinitive, 0.88, {
1031
+ removedSuffixes: [suffix],
1032
+ conjugationType: "reflexive"
1033
+ });
1034
+ }
1035
+ if (withoutReflexive.endsWith("are") || withoutReflexive.endsWith("ere") || withoutReflexive.endsWith("ire")) {
1036
+ return normalized(withoutReflexive, 0.88, {
1037
+ removedSuffixes: [suffix],
1038
+ conjugationType: "reflexive"
1039
+ });
1040
+ }
1041
+ const innerResult = this.tryConjugationNormalization(withoutReflexive);
1042
+ if (innerResult && innerResult.stem !== withoutReflexive) {
1043
+ return normalized(innerResult.stem, innerResult.confidence * 0.95, {
1044
+ removedSuffixes: [suffix, ...innerResult.metadata?.removedSuffixes || []],
1045
+ conjugationType: "reflexive"
1046
+ });
1047
+ }
1048
+ }
1049
+ }
1050
+ return null;
1051
+ }
1052
+ /**
1053
+ * Try to normalize a conjugated verb to its infinitive.
1054
+ */
1055
+ tryConjugationNormalization(word) {
1056
+ for (const rule of ALL_ENDINGS) {
1057
+ if (word.endsWith(rule.ending)) {
1058
+ const stemBase = word.slice(0, -rule.ending.length);
1059
+ if (stemBase.length < 2) continue;
1060
+ const infinitive = stemBase + rule.stem;
1061
+ return normalized(infinitive, rule.confidence, {
1062
+ removedSuffixes: [rule.ending],
1063
+ conjugationType: rule.type
1064
+ });
1065
+ }
1066
+ }
1067
+ return null;
1068
+ }
1069
+ };
1070
+ var italianMorphologicalNormalizer = new ItalianMorphologicalNormalizer();
1071
+
1072
+ // src/generators/profiles/italian.ts
1073
+ var italianProfile = {
1074
+ code: "it",
1075
+ name: "Italian",
1076
+ nativeName: "Italiano",
1077
+ direction: "ltr",
1078
+ wordOrder: "SVO",
1079
+ markingStrategy: "preposition",
1080
+ usesSpaces: true,
1081
+ // Infinitive is standard for Italian software UI (Salvare, Annullare, Aprire)
1082
+ // This matches macOS, Windows, and web app conventions
1083
+ defaultVerbForm: "infinitive",
1084
+ verb: {
1085
+ position: "start",
1086
+ subjectDrop: true
1087
+ },
1088
+ references: {
1089
+ me: "io",
1090
+ // "I/me"
1091
+ it: "esso",
1092
+ // "it"
1093
+ you: "tu",
1094
+ // "you"
1095
+ result: "risultato",
1096
+ event: "evento",
1097
+ target: "obiettivo",
1098
+ body: "corpo"
1099
+ },
1100
+ possessive: {
1101
+ marker: "di",
1102
+ // Italian uses "di" for general possession
1103
+ markerPosition: "before-property",
1104
+ usePossessiveAdjectives: true,
1105
+ specialForms: {
1106
+ me: "mio",
1107
+ // "my" (possessive adjective)
1108
+ it: "suo",
1109
+ // "its"
1110
+ you: "tuo"
1111
+ // "your"
1112
+ },
1113
+ keywords: {
1114
+ // "my" variants (masculine/feminine singular/plural)
1115
+ mio: "me",
1116
+ mia: "me",
1117
+ miei: "me",
1118
+ mie: "me",
1119
+ // "your" variants
1120
+ tuo: "you",
1121
+ tua: "you",
1122
+ tuoi: "you",
1123
+ tue: "you",
1124
+ // "its/his/her" variants
1125
+ suo: "it",
1126
+ sua: "it",
1127
+ suoi: "it",
1128
+ sue: "it"
1129
+ }
1130
+ },
1131
+ roleMarkers: {
1132
+ destination: { primary: "in", alternatives: ["su", "a"], position: "before" },
1133
+ source: { primary: "da", alternatives: ["di"], position: "before" },
1134
+ patient: { primary: "", position: "before" },
1135
+ style: { primary: "con", position: "before" }
1136
+ },
1137
+ keywords: {
1138
+ // Class/Attribute operations
1139
+ toggle: { primary: "commutare", alternatives: ["alternare", "cambiare"], normalized: "toggle" },
1140
+ add: { primary: "aggiungere", alternatives: ["aggiungi"], normalized: "add" },
1141
+ remove: { primary: "rimuovere", alternatives: ["eliminare", "togliere"], normalized: "remove" },
1142
+ // Content operations
1143
+ put: { primary: "mettere", alternatives: ["inserire"], normalized: "put" },
1144
+ append: { primary: "aggiungere", normalized: "append" },
1145
+ prepend: { primary: "anteporre", normalized: "prepend" },
1146
+ take: { primary: "prendere", normalized: "take" },
1147
+ make: { primary: "fare", alternatives: ["creare"], normalized: "make" },
1148
+ clone: { primary: "clonare", alternatives: ["copiare"], normalized: "clone" },
1149
+ swap: { primary: "scambiare", alternatives: ["cambiare"], normalized: "swap" },
1150
+ morph: { primary: "trasformare", alternatives: ["convertire"], normalized: "morph" },
1151
+ // Variable operations
1152
+ set: { primary: "impostare", alternatives: ["definire"], normalized: "set" },
1153
+ get: { primary: "ottenere", alternatives: ["prendere"], normalized: "get" },
1154
+ increment: { primary: "incrementare", alternatives: ["aumentare"], normalized: "increment" },
1155
+ decrement: { primary: "decrementare", alternatives: ["diminuire"], normalized: "decrement" },
1156
+ log: { primary: "registrare", normalized: "log" },
1157
+ // Visibility
1158
+ show: { primary: "mostrare", alternatives: ["visualizzare"], normalized: "show" },
1159
+ hide: { primary: "nascondere", normalized: "hide" },
1160
+ transition: { primary: "transizione", alternatives: ["animare"], normalized: "transition" },
1161
+ // Events
1162
+ on: { primary: "su", alternatives: ["quando", "al"], normalized: "on" },
1163
+ trigger: { primary: "scatenare", alternatives: ["attivare"], normalized: "trigger" },
1164
+ send: { primary: "inviare", normalized: "send" },
1165
+ // DOM focus
1166
+ focus: { primary: "focalizzare", normalized: "focus" },
1167
+ blur: { primary: "sfuocare", normalized: "blur" },
1168
+ // Navigation
1169
+ go: { primary: "andare", alternatives: ["navigare", "vai"], normalized: "go" },
1170
+ // Async
1171
+ wait: { primary: "aspettare", alternatives: ["attendere"], normalized: "wait" },
1172
+ fetch: { primary: "recuperare", normalized: "fetch" },
1173
+ settle: { primary: "stabilizzare", normalized: "settle" },
1174
+ // Control flow
1175
+ if: { primary: "se", normalized: "if" },
1176
+ when: { primary: "quando", normalized: "when" },
1177
+ where: { primary: "dove", normalized: "where" },
1178
+ else: { primary: "altrimenti", normalized: "else" },
1179
+ repeat: { primary: "ripetere", normalized: "repeat" },
1180
+ for: { primary: "per", normalized: "for" },
1181
+ while: { primary: "mentre", normalized: "while" },
1182
+ continue: { primary: "continuare", normalized: "continue" },
1183
+ halt: { primary: "fermare", normalized: "halt" },
1184
+ throw: { primary: "lanciare", normalized: "throw" },
1185
+ call: { primary: "chiamare", normalized: "call" },
1186
+ return: { primary: "ritornare", normalized: "return" },
1187
+ then: { primary: "allora", alternatives: ["poi", "quindi"], normalized: "then" },
1188
+ and: { primary: "e", alternatives: ["anche"], normalized: "and" },
1189
+ end: { primary: "fine", normalized: "end" },
1190
+ // Advanced
1191
+ js: { primary: "js", normalized: "js" },
1192
+ async: { primary: "asincrono", normalized: "async" },
1193
+ tell: { primary: "dire", normalized: "tell" },
1194
+ default: { primary: "predefinito", normalized: "default" },
1195
+ init: { primary: "inizializzare", alternatives: ["inizia"], normalized: "init" },
1196
+ behavior: { primary: "comportamento", normalized: "behavior" },
1197
+ install: { primary: "installare", normalized: "install" },
1198
+ measure: { primary: "misurare", normalized: "measure" },
1199
+ // Modifiers
1200
+ into: { primary: "in", alternatives: ["dentro"], normalized: "into" },
1201
+ before: { primary: "prima", normalized: "before" },
1202
+ after: { primary: "dopo", normalized: "after" },
1203
+ // Common event names (for event handler patterns)
1204
+ click: { primary: "clic", alternatives: ["clicca"], normalized: "click" },
1205
+ hover: { primary: "passaggio", alternatives: ["sorvolo"], normalized: "hover" },
1206
+ submit: { primary: "invio", alternatives: ["inviare"], normalized: "submit" },
1207
+ input: { primary: "inserimento", alternatives: ["input"], normalized: "input" },
1208
+ change: { primary: "cambio", alternatives: ["cambiamento"], normalized: "change" },
1209
+ // Event modifiers
1210
+ until: { primary: "fino", normalized: "until" },
1211
+ event: { primary: "evento", normalized: "event" },
1212
+ from: { primary: "da", alternatives: ["di"], normalized: "from" }
1213
+ },
1214
+ eventHandler: {
1215
+ keyword: { primary: "su", alternatives: ["al", "quando"], normalized: "on" },
1216
+ sourceMarker: { primary: "da", alternatives: ["di"], position: "before" },
1217
+ conditionalKeyword: { primary: "quando", alternatives: ["se"] },
1218
+ // Event marker: al (at/upon), used in SVO pattern
1219
+ // Pattern: al [event] [verb] [patient] su [destination?]
1220
+ // Example: al clic commutare .active su #button
1221
+ eventMarker: { primary: "al", alternatives: ["allo", "alla"], position: "before" },
1222
+ temporalMarkers: ["quando", "al"]
1223
+ // temporal conjunctions (when)
1224
+ }
1225
+ };
1226
+
1227
+ // src/tokenizers/italian.ts
1228
+ var { isLetter: isItalianLetter, isIdentifierChar: isItalianIdentifierChar } = createLatinCharClassifiers(/[a-zA-ZàèéìíîòóùúÀÈÉÌÍÎÒÓÙÚ]/);
1229
+ var ITALIAN_TIME_UNITS = [
1230
+ { pattern: "millisecondi", suffix: "ms", length: 12, caseInsensitive: true },
1231
+ { pattern: "millisecondo", suffix: "ms", length: 12, caseInsensitive: true },
1232
+ { pattern: "secondi", suffix: "s", length: 7, caseInsensitive: true },
1233
+ { pattern: "secondo", suffix: "s", length: 7, caseInsensitive: true },
1234
+ { pattern: "minuti", suffix: "m", length: 6, caseInsensitive: true },
1235
+ { pattern: "minuto", suffix: "m", length: 6, caseInsensitive: true },
1236
+ { pattern: "ore", suffix: "h", length: 3, caseInsensitive: true },
1237
+ { pattern: "ora", suffix: "h", length: 3, caseInsensitive: true }
1238
+ ];
1239
+ var PREPOSITIONS = /* @__PURE__ */ new Set([
1240
+ "in",
1241
+ // in, into
1242
+ "a",
1243
+ // to, at
1244
+ "di",
1245
+ // of, from
1246
+ "da",
1247
+ // from, by
1248
+ "con",
1249
+ // with
1250
+ "su",
1251
+ // on
1252
+ "per",
1253
+ // for
1254
+ "tra",
1255
+ // between
1256
+ "fra",
1257
+ // between (variant)
1258
+ "dopo",
1259
+ // after
1260
+ "prima",
1261
+ // before
1262
+ "dentro",
1263
+ // inside
1264
+ "fuori",
1265
+ // outside
1266
+ "sopra",
1267
+ // above
1268
+ "sotto",
1269
+ // under
1270
+ // Articulated prepositions
1271
+ "al",
1272
+ // a + il
1273
+ "allo",
1274
+ // a + lo
1275
+ "alla",
1276
+ // a + la
1277
+ "ai",
1278
+ // a + i
1279
+ "agli",
1280
+ // a + gli
1281
+ "alle",
1282
+ // a + le
1283
+ "del",
1284
+ // di + il
1285
+ "dello",
1286
+ // di + lo
1287
+ "della",
1288
+ // di + la
1289
+ "dei",
1290
+ // di + i
1291
+ "degli",
1292
+ // di + gli
1293
+ "delle",
1294
+ // di + le
1295
+ "dal",
1296
+ // da + il
1297
+ "dallo",
1298
+ // da + lo
1299
+ "dalla",
1300
+ // da + la
1301
+ "dai",
1302
+ // da + i
1303
+ "dagli",
1304
+ // da + gli
1305
+ "dalle",
1306
+ // da + le
1307
+ "nel",
1308
+ // in + il
1309
+ "nello",
1310
+ // in + lo
1311
+ "nella",
1312
+ // in + la
1313
+ "nei",
1314
+ // in + i
1315
+ "negli",
1316
+ // in + gli
1317
+ "nelle",
1318
+ // in + le
1319
+ "sul",
1320
+ // su + il
1321
+ "sullo",
1322
+ // su + lo
1323
+ "sulla",
1324
+ // su + la
1325
+ "sui",
1326
+ // su + i
1327
+ "sugli",
1328
+ // su + gli
1329
+ "sulle"
1330
+ // su + le
1331
+ ]);
1332
+ var ITALIAN_EXTRAS = [
1333
+ // Values/Literals
1334
+ { native: "vero", normalized: "true" },
1335
+ { native: "falso", normalized: "false" },
1336
+ { native: "nullo", normalized: "null" },
1337
+ { native: "indefinito", normalized: "undefined" },
1338
+ // Positional
1339
+ { native: "primo", normalized: "first" },
1340
+ { native: "prima", normalized: "first" },
1341
+ { native: "ultimo", normalized: "last" },
1342
+ { native: "ultima", normalized: "last" },
1343
+ { native: "prossimo", normalized: "next" },
1344
+ { native: "successivo", normalized: "next" },
1345
+ { native: "precedente", normalized: "previous" },
1346
+ { native: "vicino", normalized: "closest" },
1347
+ { native: "padre", normalized: "parent" },
1348
+ // Events
1349
+ { native: "clic", normalized: "click" },
1350
+ { native: "click", normalized: "click" },
1351
+ { native: "fare clic", normalized: "click" },
1352
+ { native: "input", normalized: "input" },
1353
+ { native: "cambio", normalized: "change" },
1354
+ { native: "invio", normalized: "submit" },
1355
+ { native: "tasto gi\xF9", normalized: "keydown" },
1356
+ { native: "tasto su", normalized: "keyup" },
1357
+ { native: "mouse sopra", normalized: "mouseover" },
1358
+ { native: "mouse fuori", normalized: "mouseout" },
1359
+ { native: "fuoco", normalized: "focus" },
1360
+ { native: "sfuocatura", normalized: "blur" },
1361
+ { native: "caricamento", normalized: "load" },
1362
+ { native: "scorrimento", normalized: "scroll" },
1363
+ // References
1364
+ { native: "io", normalized: "me" },
1365
+ { native: "me", normalized: "me" },
1366
+ { native: "destinazione", normalized: "target" },
1367
+ // Time units
1368
+ { native: "secondo", normalized: "s" },
1369
+ { native: "secondi", normalized: "s" },
1370
+ { native: "millisecondo", normalized: "ms" },
1371
+ { native: "millisecondi", normalized: "ms" },
1372
+ { native: "minuto", normalized: "m" },
1373
+ { native: "minuti", normalized: "m" },
1374
+ { native: "ora", normalized: "h" },
1375
+ { native: "ore", normalized: "h" },
1376
+ // Multi-word phrases
1377
+ { native: "fino a", normalized: "until" },
1378
+ { native: "prima di", normalized: "before" },
1379
+ { native: "dopo di", normalized: "after" },
1380
+ { native: "dentro di", normalized: "into" },
1381
+ { native: "fuori di", normalized: "out" },
1382
+ // Override profile conflicts (aggiungere is both add and append in profile, prefer add)
1383
+ { native: "aggiungere", normalized: "add" },
1384
+ // Imperative forms (profile has infinitives)
1385
+ { native: "aggiungi", normalized: "add" },
1386
+ { native: "rimuovi", normalized: "remove" },
1387
+ { native: "elimina", normalized: "remove" },
1388
+ { native: "togli", normalized: "remove" },
1389
+ { native: "metti", normalized: "put" },
1390
+ { native: "inserisci", normalized: "put" },
1391
+ { native: "prendi", normalized: "take" },
1392
+ { native: "fai", normalized: "make" },
1393
+ { native: "crea", normalized: "make" },
1394
+ { native: "clona", normalized: "clone" },
1395
+ { native: "copia", normalized: "clone" },
1396
+ { native: "imposta", normalized: "set" },
1397
+ { native: "ottieni", normalized: "get" },
1398
+ { native: "incrementa", normalized: "increment" },
1399
+ { native: "aumenta", normalized: "increment" },
1400
+ { native: "decrementa", normalized: "decrement" },
1401
+ { native: "diminuisci", normalized: "decrement" },
1402
+ { native: "registra", normalized: "log" },
1403
+ { native: "mostra", normalized: "show" },
1404
+ { native: "visualizza", normalized: "show" },
1405
+ { native: "nascondi", normalized: "hide" },
1406
+ { native: "anima", normalized: "transition" },
1407
+ { native: "scatena", normalized: "trigger" },
1408
+ { native: "attiva", normalized: "trigger" },
1409
+ { native: "invia", normalized: "send" },
1410
+ { native: "focalizza", normalized: "focus" },
1411
+ { native: "sfuoca", normalized: "blur" },
1412
+ { native: "vai", normalized: "go" },
1413
+ { native: "naviga", normalized: "go" },
1414
+ { native: "aspetta", normalized: "wait" },
1415
+ { native: "attendi", normalized: "wait" },
1416
+ { native: "recupera", normalized: "fetch" },
1417
+ { native: "stabilizza", normalized: "settle" },
1418
+ { native: "ripeti", normalized: "repeat" },
1419
+ { native: "continua", normalized: "continue" },
1420
+ { native: "ferma", normalized: "halt" },
1421
+ { native: "lancia", normalized: "throw" },
1422
+ { native: "chiama", normalized: "call" },
1423
+ { native: "ritorna", normalized: "return" },
1424
+ { native: "inizializza", normalized: "init" },
1425
+ { native: "installa", normalized: "install" },
1426
+ { native: "misura", normalized: "measure" },
1427
+ // Logical/conditional
1428
+ { native: "e", normalized: "and" },
1429
+ { native: "o", normalized: "or" },
1430
+ { native: "non", normalized: "not" },
1431
+ { native: "\xE8", normalized: "is" },
1432
+ { native: "esiste", normalized: "exists" },
1433
+ { native: "vuoto", normalized: "empty" },
1434
+ // Synonyms not in profile
1435
+ { native: "toggle", normalized: "toggle" },
1436
+ { native: "di", normalized: "tell" }
1437
+ ];
1438
+ var ItalianTokenizer = class extends BaseTokenizer {
1439
+ constructor() {
1440
+ super();
1441
+ this.language = "it";
1442
+ this.direction = "ltr";
1443
+ this.initializeKeywordsFromProfile(italianProfile, ITALIAN_EXTRAS);
1444
+ this.normalizer = new ItalianMorphologicalNormalizer();
1445
+ }
1446
+ tokenize(input) {
1447
+ const tokens = [];
1448
+ let pos = 0;
1449
+ while (pos < input.length) {
1450
+ if (isWhitespace(input[pos])) {
1451
+ pos++;
1452
+ continue;
1453
+ }
1454
+ if (isSelectorStart(input[pos])) {
1455
+ const modifierToken = this.tryEventModifier(input, pos);
1456
+ if (modifierToken) {
1457
+ tokens.push(modifierToken);
1458
+ pos = modifierToken.position.end;
1459
+ continue;
1460
+ }
1461
+ const selectorToken = this.trySelector(input, pos);
1462
+ if (selectorToken) {
1463
+ tokens.push(selectorToken);
1464
+ pos = selectorToken.position.end;
1465
+ continue;
1466
+ }
1467
+ }
1468
+ if (isQuote(input[pos])) {
1469
+ const stringToken = this.tryString(input, pos);
1470
+ if (stringToken) {
1471
+ tokens.push(stringToken);
1472
+ pos = stringToken.position.end;
1473
+ continue;
1474
+ }
1475
+ }
1476
+ if (isUrlStart(input, pos)) {
1477
+ const urlToken = this.tryUrl(input, pos);
1478
+ if (urlToken) {
1479
+ tokens.push(urlToken);
1480
+ pos = urlToken.position.end;
1481
+ continue;
1482
+ }
1483
+ }
1484
+ if (isDigit(input[pos]) || input[pos] === "-" && pos + 1 < input.length && isDigit(input[pos + 1])) {
1485
+ const numberToken = this.extractItalianNumber(input, pos);
1486
+ if (numberToken) {
1487
+ tokens.push(numberToken);
1488
+ pos = numberToken.position.end;
1489
+ continue;
1490
+ }
1491
+ }
1492
+ const varToken = this.tryVariableRef(input, pos);
1493
+ if (varToken) {
1494
+ tokens.push(varToken);
1495
+ pos = varToken.position.end;
1496
+ continue;
1497
+ }
1498
+ const phraseToken = this.tryMultiWordPhrase(input, pos);
1499
+ if (phraseToken) {
1500
+ tokens.push(phraseToken);
1501
+ pos = phraseToken.position.end;
1502
+ continue;
1503
+ }
1504
+ if (isItalianLetter(input[pos])) {
1505
+ const wordToken = this.extractItalianWord(input, pos);
1506
+ if (wordToken) {
1507
+ tokens.push(wordToken);
1508
+ pos = wordToken.position.end;
1509
+ continue;
1510
+ }
1511
+ }
1512
+ const operatorToken = this.tryOperator(input, pos);
1513
+ if (operatorToken) {
1514
+ tokens.push(operatorToken);
1515
+ pos = operatorToken.position.end;
1516
+ continue;
1517
+ }
1518
+ pos++;
1519
+ }
1520
+ return new TokenStreamImpl(tokens, "it");
1521
+ }
1522
+ classifyToken(token) {
1523
+ const lower = token.toLowerCase();
1524
+ if (PREPOSITIONS.has(lower)) return "particle";
1525
+ if (this.isKeyword(lower)) return "keyword";
1526
+ if (token.startsWith("#") || token.startsWith(".") || token.startsWith("[")) return "selector";
1527
+ if (token.startsWith('"') || token.startsWith("'")) return "literal";
1528
+ if (/^\d/.test(token)) return "literal";
1529
+ if (["==", "!=", "<=", ">=", "<", ">", "&&", "||", "!"].includes(token)) return "operator";
1530
+ return "identifier";
1531
+ }
1532
+ /**
1533
+ * Try to match multi-word phrases that function as single units.
1534
+ * Multi-word phrases are included in profileKeywords and sorted longest-first,
1535
+ * so they'll be matched before their constituent words.
1536
+ */
1537
+ tryMultiWordPhrase(input, pos) {
1538
+ for (const entry of this.profileKeywords) {
1539
+ if (!entry.native.includes(" ")) continue;
1540
+ const phrase = entry.native;
1541
+ const candidate = input.slice(pos, pos + phrase.length).toLowerCase();
1542
+ if (candidate === phrase.toLowerCase()) {
1543
+ const nextPos = pos + phrase.length;
1544
+ if (nextPos >= input.length || isWhitespace(input[nextPos]) || !isItalianLetter(input[nextPos])) {
1545
+ return createToken(
1546
+ input.slice(pos, pos + phrase.length),
1547
+ "keyword",
1548
+ createPosition(pos, nextPos),
1549
+ entry.normalized
1550
+ );
1551
+ }
1552
+ }
1553
+ }
1554
+ return null;
1555
+ }
1556
+ /**
1557
+ * Extract an Italian word.
1558
+ *
1559
+ * Uses morphological normalization to handle:
1560
+ * - Reflexive verbs (mostrarsi → mostrare)
1561
+ * - Verb conjugations (alternando → alternare)
1562
+ */
1563
+ extractItalianWord(input, startPos) {
1564
+ let pos = startPos;
1565
+ let word = "";
1566
+ while (pos < input.length && isItalianIdentifierChar(input[pos])) {
1567
+ word += input[pos++];
1568
+ }
1569
+ if (!word) return null;
1570
+ const lower = word.toLowerCase();
1571
+ if (PREPOSITIONS.has(lower)) {
1572
+ return createToken(word, "particle", createPosition(startPos, pos));
1573
+ }
1574
+ const keywordEntry = this.lookupKeyword(lower);
1575
+ if (keywordEntry) {
1576
+ return createToken(word, "keyword", createPosition(startPos, pos), keywordEntry.normalized);
1577
+ }
1578
+ const morphToken = this.tryMorphKeywordMatch(lower, startPos, pos);
1579
+ if (morphToken) return morphToken;
1580
+ return createToken(word, "identifier", createPosition(startPos, pos));
1581
+ }
1582
+ /**
1583
+ * Extract a number, including Italian time unit suffixes.
1584
+ */
1585
+ extractItalianNumber(input, startPos) {
1586
+ return this.tryNumberWithTimeUnits(input, startPos, ITALIAN_TIME_UNITS, {
1587
+ allowSign: true,
1588
+ skipWhitespace: true
1589
+ });
1590
+ }
1591
+ };
1592
+ var italianTokenizer = new ItalianTokenizer();
1593
+
1594
+ // src/languages/it.ts
1595
+ registerLanguage("it", italianTokenizer, italianProfile);
1596
+ export {
1597
+ italianProfile,
1598
+ italianTokenizer
1599
+ };
1600
+ //# sourceMappingURL=it.js.map