@lokascript/semantic 1.3.0 → 2.0.0

package/dist/languages/pl.js

@@ -8,7 +8,7 @@ function registerLanguage(code, tokenizer, profile) {
   patternCache.delete(code);
 }
 
-// src/tokenizers/base.ts
+// src/tokenizers/token-utils.ts
 var TokenStreamImpl = class {
   constructor(tokens, language) {
     this.pos = 0;
@@ -73,18 +73,15 @@ function createToken(value, kind, position, normalizedOrOptions) {
     return { value, kind, position, normalized: normalizedOrOptions };
   }
   if (normalizedOrOptions) {
-    const { normalized, stem, stemConfidence } = normalizedOrOptions;
-    const token = { value, kind, position };
-    if (normalized !== void 0) {
-      token.normalized = normalized;
-    }
-    if (stem !== void 0) {
-      token.stem = stem;
-      if (stemConfidence !== void 0) {
-        token.stemConfidence = stemConfidence;
-      }
-    }
-    return token;
+    const { normalized: normalized2, stem, stemConfidence } = normalizedOrOptions;
+    return {
+      value,
+      kind,
+      position,
+      ...normalized2 !== void 0 && { normalized: normalized2 },
+      ...stem !== void 0 && { stem },
+      ...stemConfidence !== void 0 && { stemConfidence }
+    };
   }
   return { value, kind, position };
 }
@@ -106,11 +103,15 @@ function isAsciiLetter(char) {
 function isAsciiIdentifierChar(char) {
   return /[a-zA-Z0-9_-]/.test(char);
 }
+
+// src/tokenizers/char-classifiers.ts
 function createLatinCharClassifiers(letterPattern) {
   const isLetter = (char) => letterPattern.test(char);
   const isIdentifierChar = (char) => isLetter(char) || /[0-9_-]/.test(char);
   return { isLetter, isIdentifierChar };
 }
+
+// src/tokenizers/extractors.ts
 function extractCssSelector(input, startPos) {
   if (startPos >= input.length) return null;
   const char = input[startPos];
@@ -368,6 +369,8 @@ function extractNumber(input, startPos) {
   }
   return number;
 }
+
+// src/tokenizers/base-tokenizer.ts
 var _BaseTokenizer = class _BaseTokenizer {
   constructor() {
     /** Keywords derived from profile, sorted longest-first for greedy matching */
@@ -375,6 +378,34 @@ var _BaseTokenizer = class _BaseTokenizer {
     /** Map for O(1) keyword lookups by lowercase native word */
     this.profileKeywordMap = /* @__PURE__ */ new Map();
   }
+  /**
+   * Check if current position is a property access (obj.prop) vs CSS selector (.active).
+   * Property access: no whitespace before '.', previous token is identifier/keyword/selector.
+   * Also detects standalone method calls: .identifier( pattern.
+   *
+   * Returns true if '.' was emitted as an operator token and pos should advance by 1.
+   * Returns false if this is a CSS selector and should be handled by trySelector().
+   */
+  tryPropertyAccess(input, pos, tokens) {
+    if (input[pos] !== ".") return false;
+    const lastToken = tokens[tokens.length - 1];
+    const hasWhitespaceBefore = lastToken && lastToken.position.end < pos;
+    const isPropertyAccess = lastToken && !hasWhitespaceBefore && (lastToken.kind === "identifier" || lastToken.kind === "keyword" || lastToken.kind === "selector");
+    if (isPropertyAccess) {
+      tokens.push(createToken(".", "operator", createPosition(pos, pos + 1)));
+      return true;
+    }
+    const methodStart = pos + 1;
+    let methodEnd = methodStart;
+    while (methodEnd < input.length && isAsciiIdentifierChar(input[methodEnd])) {
+      methodEnd++;
+    }
+    if (methodEnd < input.length && input[methodEnd] === "(") {
+      tokens.push(createToken(".", "operator", createPosition(pos, pos + 1)));
+      return true;
+    }
+    return false;
+  }
   /**
    * Initialize keyword mappings from a language profile.
    * Builds a list of native→english mappings from:
@@ -391,24 +422,29 @@ var _BaseTokenizer = class _BaseTokenizer {
   initializeKeywordsFromProfile(profile, extras = []) {
     const keywordMap = /* @__PURE__ */ new Map();
     if (profile.keywords) {
-      for (const [normalized, translation] of Object.entries(profile.keywords)) {
+      for (const [normalized2, translation] of Object.entries(profile.keywords)) {
         keywordMap.set(translation.primary, {
           native: translation.primary,
-          normalized: translation.normalized || normalized
+          normalized: translation.normalized || normalized2
         });
         if (translation.alternatives) {
           for (const alt of translation.alternatives) {
             keywordMap.set(alt, {
               native: alt,
-              normalized: translation.normalized || normalized
+              normalized: translation.normalized || normalized2
             });
           }
         }
       }
     }
     if (profile.references) {
-      for (const [normalized, native] of Object.entries(profile.references)) {
-        keywordMap.set(native, { native, normalized });
+      for (const [normalized2, native] of Object.entries(profile.references)) {
+        keywordMap.set(native, { native, normalized: normalized2 });
+      }
+      for (const canonical of Object.keys(profile.references)) {
+        if (!keywordMap.has(canonical)) {
+          keywordMap.set(canonical, { native: canonical, normalized: canonical });
+        }
       }
     }
     if (profile.roleMarkers) {
@@ -432,9 +468,9 @@ var _BaseTokenizer = class _BaseTokenizer {
     this.profileKeywordMap = /* @__PURE__ */ new Map();
     for (const keyword of this.profileKeywords) {
       this.profileKeywordMap.set(keyword.native.toLowerCase(), keyword);
-      const normalized = this.removeDiacritics(keyword.native);
-      if (normalized !== keyword.native && !this.profileKeywordMap.has(normalized.toLowerCase())) {
-        this.profileKeywordMap.set(normalized.toLowerCase(), keyword);
+      const normalized2 = this.removeDiacritics(keyword.native);
+      if (normalized2 !== keyword.native && !this.profileKeywordMap.has(normalized2.toLowerCase())) {
+        this.profileKeywordMap.set(normalized2.toLowerCase(), keyword);
       }
     }
   }
@@ -1013,6 +1049,12 @@ var polishProfile = {
     behavior: { primary: "zachowanie", normalized: "behavior" },
     install: { primary: "zainstaluj", normalized: "install", form: "imperative" },
     measure: { primary: "zmierz", normalized: "measure", form: "imperative" },
+    beep: { primary: "sygna\u0142", normalized: "beep" },
+    break: { primary: "przerwij", normalized: "break" },
+    copy: { primary: "kopiuj", normalized: "copy" },
+    exit: { primary: "wyjd\u017A", normalized: "exit" },
+    pick: { primary: "wybierz", normalized: "pick" },
+    render: { primary: "renderuj", normalized: "render" },
     // Modifiers
     into: { primary: "do", alternatives: ["w"], normalized: "into" },
     before: { primary: "przed", normalized: "before" },
@@ -1041,6 +1083,213 @@ var polishProfile = {
   }
 };
 
+// src/tokenizers/morphology/types.ts
+function noChange(word) {
+  return { stem: word, confidence: 1 };
+}
+function normalized(stem, confidence, metadata) {
+  if (metadata) {
+    return { stem, confidence, metadata };
+  }
+  return { stem, confidence };
+}
+
+// src/tokenizers/morphology/polish-normalizer.ts
+var IMPERATIVE_TO_INFINITIVE = /* @__PURE__ */ new Map([
+  // Core commands
+  ["prze\u0142\u0105cz", "prze\u0142\u0105cza\u0107"],
+  ["przelacz", "przelaczac"],
+  ["dodaj", "dodawa\u0107"],
+  ["usu\u0144", "usuwa\u0107"],
+  ["usun", "usuwac"],
+  ["umie\u015B\u0107", "umieszcza\u0107"],
+  ["umiesc", "umieszczac"],
+  ["wstaw", "wstawia\u0107"],
+  ["ustaw", "ustawia\u0107"],
+  ["pobierz", "pobiera\u0107"],
+  ["we\u017A", "bra\u0107"],
+  ["wez", "brac"],
+  ["zwi\u0119ksz", "zwi\u0119ksza\u0107"],
+  ["zwieksz", "zwiekszac"],
+  ["zmniejsz", "zmniejsza\u0107"],
+  ["poka\u017C", "pokazywa\u0107"],
+  ["pokaz", "pokazywac"],
+  ["ukryj", "ukrywa\u0107"],
+  ["schowaj", "schowywa\u0107"],
+  ["czekaj", "czeka\u0107"],
+  ["poczekaj", "poczeka\u0107"],
+  ["id\u017A", "i\u015B\u0107"],
+  ["idz", "isc"],
+  ["przejd\u017A", "przej\u015B\u0107"],
+  ["przejdz", "przejsc"],
+  ["wywo\u0142aj", "wywo\u0142ywa\u0107"],
+  ["wywolaj", "wywolywac"],
+  ["wy\u015Blij", "wysy\u0142a\u0107"],
+  ["wyslij", "wysylac"],
+  ["loguj", "logowa\u0107"],
+  ["wypisz", "wypisywa\u0107"],
+  ["sklonuj", "sklonowa\u0107"],
+  ["kopiuj", "kopiowa\u0107"],
+  ["zamie\u0144", "zamienia\u0107"],
+  ["zamien", "zamieniac"],
+  ["utw\xF3rz", "tworzy\u0107"],
+  ["utworz", "tworzyc"],
+  ["stw\xF3rz", "stwarza\u0107"],
+  ["stworz", "stwarzac"],
+  ["skup", "skupia\u0107"],
+  ["rozmyj", "rozmywa\u0107"],
+  ["nawiguj", "nawigowa\u0107"],
+  ["za\u0142aduj", "\u0142adowa\u0107"],
+  ["zaladuj", "ladowac"],
+  ["powt\xF3rz", "powtarza\u0107"],
+  ["powtorz", "powtarzac"],
+  ["kontynuuj", "kontynuowa\u0107"],
+  ["zatrzymaj", "zatrzymywa\u0107"],
+  ["przerwij", "przerywa\u0107"],
+  ["rzu\u0107", "rzuca\u0107"],
+  ["rzuc", "rzucac"],
+  ["zwr\xF3\u0107", "zwraca\u0107"],
+  ["zwroc", "zwracac"],
+  ["inicjuj", "inicjowa\u0107"],
+  ["zainstaluj", "instalowa\u0107"],
+  ["zmierz", "mierzy\u0107"]
+]);
+var PolishMorphologicalNormalizer = class {
+  constructor() {
+    this.language = "pl";
+  }
+  /**
+   * Check if a word appears to be a Polish verb form that can be normalized.
+   */
+  isNormalizable(word) {
+    if (word.length < 3) return false;
+    return /[a-zA-ZąęćńóśźżłĄĘĆŃÓŚŹŻŁ]/.test(word);
+  }
+  /**
+   * Normalize a Polish verb to its base/infinitive form.
+   */
+  normalize(word) {
+    const lower = word.toLowerCase();
+    if (this.isInfinitive(lower)) {
+      return noChange(lower);
+    }
+    const imperativeResult = this.tryImperativeNormalization(lower);
+    if (imperativeResult) return imperativeResult;
+    const pastResult = this.tryPastTenseNormalization(lower);
+    if (pastResult) return pastResult;
+    const presentResult = this.tryPresentTenseNormalization(lower);
+    if (presentResult) return presentResult;
+    return noChange(lower);
+  }
+  /**
+   * Check if word is already in infinitive form.
+   */
+  isInfinitive(word) {
+    const infinitiveEndings = ["a\u0107", "e\u0107", "i\u0107", "y\u0107", "\u0105\u0107", "owa\u0107", "iwa\u0107", "ywa\u0107"];
+    return infinitiveEndings.some((ending) => word.endsWith(ending));
+  }
+  /**
+   * Try to normalize imperative form to infinitive.
+   *
+   * Polish imperative (2nd person singular) patterns:
+   * - pisać → pisz (write)
+   * - czytać → czytaj (read)
+   * - robić → rób (do)
+   * - mówić → mów (speak)
+   * - uczyć → ucz (teach)
+   */
+  tryImperativeNormalization(word) {
+    if (IMPERATIVE_TO_INFINITIVE.has(word)) {
+      return normalized(IMPERATIVE_TO_INFINITIVE.get(word), 0.95, {
+        removedSuffixes: ["imperative"],
+        conjugationType: "imperative"
+      });
+    }
+    if (word.endsWith("aj")) {
+      return normalized(word.slice(0, -2) + "a\u0107", 0.8, {
+        removedSuffixes: ["aj"],
+        conjugationType: "imperative"
+      });
+    }
+    if (word.endsWith("uj")) {
+      return normalized(word.slice(0, -2) + "owa\u0107", 0.8, {
+        removedSuffixes: ["uj"],
+        conjugationType: "imperative"
+      });
+    }
+    if (word.endsWith("ij")) {
+      return normalized(word.slice(0, -2) + "i\u0107", 0.75, {
+        removedSuffixes: ["ij"],
+        conjugationType: "imperative"
+      });
+    }
+    return null;
+  }
+  /**
+   * Try to normalize present tense form to infinitive.
+   */
+  tryPresentTenseNormalization(word) {
+    if (word.endsWith("uj\u0119") || word.endsWith("uje")) {
+      return normalized(word.slice(0, -3) + "owa\u0107", 0.85, {
+        removedSuffixes: ["uj\u0119"],
+        conjugationType: "present"
+      });
+    }
+    if (word.endsWith("am") && word.length > 3) {
+      return normalized(word.slice(0, -2) + "a\u0107", 0.8, {
+        removedSuffixes: ["am"],
+        conjugationType: "present"
+      });
+    }
+    if (word.endsWith("em") && word.length > 3) {
+      return normalized(word.slice(0, -2) + "e\u0107", 0.75, {
+        removedSuffixes: ["em"],
+        conjugationType: "present"
+      });
+    }
+    if (word.endsWith("\u0119") && word.length > 2) {
+      return normalized(word.slice(0, -1) + "a\u0107", 0.7, {
+        removedSuffixes: ["\u0119"],
+        conjugationType: "present"
+      });
+    }
+    return null;
+  }
+  /**
+   * Try to normalize past tense form to infinitive.
+   */
+  tryPastTenseNormalization(word) {
+    if (word.endsWith("a\u0142em") || word.endsWith("a\u0142am")) {
+      return normalized(word.slice(0, -4) + "a\u0107", 0.85, {
+        removedSuffixes: [word.slice(-4)],
+        conjugationType: "past"
+      });
+    }
+    if (word.endsWith("a\u0142") || word.endsWith("a\u0142a")) {
+      const suffixLen = word.endsWith("a\u0142a") ? 3 : 2;
+      return normalized(word.slice(0, -suffixLen) + "a\u0107", 0.8, {
+        removedSuffixes: [word.slice(-suffixLen)],
+        conjugationType: "past"
+      });
+    }
+    if (word.endsWith("i\u0142em") || word.endsWith("i\u0142am") || word.endsWith("ilem") || word.endsWith("ilam")) {
+      return normalized(word.slice(0, -4) + "i\u0107", 0.85, {
+        removedSuffixes: [word.slice(-4)],
+        conjugationType: "past"
+      });
+    }
+    if (word.endsWith("i\u0142") || word.endsWith("i\u0142a") || word.endsWith("il") || word.endsWith("ila")) {
+      const suffixLen = word.endsWith("i\u0142a") || word.endsWith("ila") ? 3 : 2;
+      return normalized(word.slice(0, -suffixLen) + "i\u0107", 0.8, {
+        removedSuffixes: [word.slice(-suffixLen)],
+        conjugationType: "past"
+      });
+    }
+    return null;
+  }
+};
+var polishMorphologicalNormalizer = new PolishMorphologicalNormalizer();
+
 // src/tokenizers/polish.ts
 var { isLetter: isPolishLetter, isIdentifierChar: isPolishIdentifierChar } = createLatinCharClassifiers(/[a-zA-ZąęćńóśźżłĄĘĆŃÓŚŹŻŁ]/);
 var PREPOSITIONS = /* @__PURE__ */ new Set([
@@ -1180,6 +1429,7 @@ var PolishTokenizer = class extends BaseTokenizer {
     this.language = "pl";
     this.direction = "ltr";
     this.initializeKeywordsFromProfile(polishProfile, POLISH_EXTRAS);
+    this.normalizer = new PolishMorphologicalNormalizer();
   }
   tokenize(input) {
     const tokens = [];
@@ -1196,6 +1446,10 @@ var PolishTokenizer = class extends BaseTokenizer {
           pos = modifierToken.position.end;
           continue;
         }
+        if (this.tryPropertyAccess(input, pos, tokens)) {
+          pos++;
+          continue;
+        }
         const selectorToken = this.trySelector(input, pos);
         if (selectorToken) {
           tokens.push(selectorToken);
@@ -1255,7 +1509,8 @@ var PolishTokenizer = class extends BaseTokenizer {
     const lower = token.toLowerCase();
     if (PREPOSITIONS.has(lower)) return "particle";
     if (this.isKeyword(lower)) return "keyword";
-    if (token.startsWith("#") || token.startsWith(".") || token.startsWith("[")) return "selector";
+    if (token.startsWith("#") || token.startsWith(".") || token.startsWith("[") || token.startsWith("*") || token.startsWith("<"))
+      return "selector";
     if (token.startsWith('"') || token.startsWith("'")) return "literal";
     if (/^\d/.test(token)) return "literal";
     return "identifier";