@lokascript/semantic 1.2.1 → 2.0.0

package/dist/languages/pt.js

@@ -8,7 +8,7 @@ function registerLanguage(code, tokenizer, profile) {
   patternCache.delete(code);
 }
 
-// src/tokenizers/base.ts
+// src/tokenizers/token-utils.ts
 var TokenStreamImpl = class {
   constructor(tokens, language) {
     this.pos = 0;
@@ -73,18 +73,15 @@ function createToken(value, kind, position, normalizedOrOptions) {
     return { value, kind, position, normalized: normalizedOrOptions };
   }
   if (normalizedOrOptions) {
-    const { normalized, stem, stemConfidence } = normalizedOrOptions;
-    const token = { value, kind, position };
-    if (normalized !== void 0) {
-      token.normalized = normalized;
-    }
-    if (stem !== void 0) {
-      token.stem = stem;
-      if (stemConfidence !== void 0) {
-        token.stemConfidence = stemConfidence;
-      }
-    }
-    return token;
+    const { normalized: normalized2, stem, stemConfidence } = normalizedOrOptions;
+    return {
+      value,
+      kind,
+      position,
+      ...normalized2 !== void 0 && { normalized: normalized2 },
+      ...stem !== void 0 && { stem },
+      ...stemConfidence !== void 0 && { stemConfidence }
+    };
   }
   return { value, kind, position };
 }
@@ -106,11 +103,15 @@ function isAsciiLetter(char) {
 function isAsciiIdentifierChar(char) {
   return /[a-zA-Z0-9_-]/.test(char);
 }
+
+// src/tokenizers/char-classifiers.ts
 function createLatinCharClassifiers(letterPattern) {
   const isLetter = (char) => letterPattern.test(char);
   const isIdentifierChar = (char) => isLetter(char) || /[0-9_-]/.test(char);
   return { isLetter, isIdentifierChar };
 }
+
+// src/tokenizers/extractors.ts
 function extractCssSelector(input, startPos) {
   if (startPos >= input.length) return null;
   const char = input[startPos];
@@ -368,6 +369,8 @@ function extractNumber(input, startPos) {
   }
   return number;
 }
+
+// src/tokenizers/base-tokenizer.ts
 var _BaseTokenizer = class _BaseTokenizer {
   constructor() {
     /** Keywords derived from profile, sorted longest-first for greedy matching */
@@ -375,6 +378,34 @@ var _BaseTokenizer = class _BaseTokenizer {
     /** Map for O(1) keyword lookups by lowercase native word */
     this.profileKeywordMap = /* @__PURE__ */ new Map();
   }
+  /**
+   * Check if current position is a property access (obj.prop) vs CSS selector (.active).
+   * Property access: no whitespace before '.', previous token is identifier/keyword/selector.
+   * Also detects standalone method calls: .identifier( pattern.
+   *
+   * Returns true if '.' was emitted as an operator token and pos should advance by 1.
+   * Returns false if this is a CSS selector and should be handled by trySelector().
+   */
+  tryPropertyAccess(input, pos, tokens) {
+    if (input[pos] !== ".") return false;
+    const lastToken = tokens[tokens.length - 1];
+    const hasWhitespaceBefore = lastToken && lastToken.position.end < pos;
+    const isPropertyAccess = lastToken && !hasWhitespaceBefore && (lastToken.kind === "identifier" || lastToken.kind === "keyword" || lastToken.kind === "selector");
+    if (isPropertyAccess) {
+      tokens.push(createToken(".", "operator", createPosition(pos, pos + 1)));
+      return true;
+    }
+    const methodStart = pos + 1;
+    let methodEnd = methodStart;
+    while (methodEnd < input.length && isAsciiIdentifierChar(input[methodEnd])) {
+      methodEnd++;
+    }
+    if (methodEnd < input.length && input[methodEnd] === "(") {
+      tokens.push(createToken(".", "operator", createPosition(pos, pos + 1)));
+      return true;
+    }
+    return false;
+  }
   /**
    * Initialize keyword mappings from a language profile.
    * Builds a list of native→english mappings from:
@@ -391,24 +422,29 @@ var _BaseTokenizer = class _BaseTokenizer {
   initializeKeywordsFromProfile(profile, extras = []) {
     const keywordMap = /* @__PURE__ */ new Map();
     if (profile.keywords) {
-      for (const [normalized, translation] of Object.entries(profile.keywords)) {
+      for (const [normalized2, translation] of Object.entries(profile.keywords)) {
         keywordMap.set(translation.primary, {
           native: translation.primary,
-          normalized: translation.normalized || normalized
+          normalized: translation.normalized || normalized2
         });
         if (translation.alternatives) {
           for (const alt of translation.alternatives) {
             keywordMap.set(alt, {
               native: alt,
-              normalized: translation.normalized || normalized
+              normalized: translation.normalized || normalized2
             });
           }
         }
       }
     }
     if (profile.references) {
-      for (const [normalized, native] of Object.entries(profile.references)) {
-        keywordMap.set(native, { native, normalized });
+      for (const [normalized2, native] of Object.entries(profile.references)) {
+        keywordMap.set(native, { native, normalized: normalized2 });
+      }
+      for (const canonical of Object.keys(profile.references)) {
+        if (!keywordMap.has(canonical)) {
+          keywordMap.set(canonical, { native: canonical, normalized: canonical });
+        }
       }
     }
     if (profile.roleMarkers) {
@@ -432,9 +468,9 @@ var _BaseTokenizer = class _BaseTokenizer {
     this.profileKeywordMap = /* @__PURE__ */ new Map();
     for (const keyword of this.profileKeywords) {
       this.profileKeywordMap.set(keyword.native.toLowerCase(), keyword);
-      const normalized = this.removeDiacritics(keyword.native);
-      if (normalized !== keyword.native && !this.profileKeywordMap.has(normalized.toLowerCase())) {
-        this.profileKeywordMap.set(normalized.toLowerCase(), keyword);
+      const normalized2 = this.removeDiacritics(keyword.native);
+      if (normalized2 !== keyword.native && !this.profileKeywordMap.has(normalized2.toLowerCase())) {
+        this.profileKeywordMap.set(normalized2.toLowerCase(), keyword);
       }
     }
   }
@@ -882,6 +918,12 @@ var portugueseProfile = {
     behavior: { primary: "comportamento", normalized: "behavior" },
     install: { primary: "instalar", normalized: "install" },
     measure: { primary: "medir", normalized: "measure" },
+    beep: { primary: "apitar", normalized: "beep" },
+    break: { primary: "parar", normalized: "break" },
+    copy: { primary: "copiar", normalized: "copy" },
+    exit: { primary: "sair", normalized: "exit" },
+    pick: { primary: "escolher", normalized: "pick" },
+    render: { primary: "renderizar", normalized: "render" },
     into: { primary: "em", alternatives: ["dentro de"], normalized: "into" },
     before: { primary: "antes", normalized: "before" },
     after: { primary: "depois", normalized: "after" },
@@ -909,6 +951,280 @@ var portugueseProfile = {
   }
 };
 
+// src/tokenizers/morphology/types.ts
+function noChange(word) {
+  return { stem: word, confidence: 1 };
+}
+function normalized(stem, confidence, metadata) {
+  if (metadata) {
+    return { stem, confidence, metadata };
+  }
+  return { stem, confidence };
+}
+
+// src/tokenizers/morphology/portuguese-normalizer.ts
+function isPortugueseSpecificLetter(char) {
+  return /[áàâãéêíóôõúüçÁÀÂÃÉÊÍÓÔÕÚÜÇ]/.test(char);
+}
+function looksLikePortugueseVerb(word) {
+  const lower = word.toLowerCase();
+  if (lower.endsWith("ar") || lower.endsWith("er") || lower.endsWith("ir")) return true;
+  if (lower.endsWith("ando") || lower.endsWith("endo") || lower.endsWith("indo")) return true;
+  if (lower.endsWith("ado") || lower.endsWith("ido")) return true;
+  if (lower.endsWith("ar-se") || lower.endsWith("er-se") || lower.endsWith("ir-se")) return true;
+  for (const char of word) {
+    if (isPortugueseSpecificLetter(char)) return true;
+  }
+  return false;
+}
+var REFLEXIVE_SUFFIXES = ["-se", "-me", "-te", "-nos", "-vos"];
+var AR_ENDINGS = [
+  // Gerund (-ando)
+  { ending: "ando", stem: "ar", confidence: 0.88, type: "gerund" },
+  // Past participle (-ado)
+  { ending: "ado", stem: "ar", confidence: 0.88, type: "participle" },
+  { ending: "ada", stem: "ar", confidence: 0.88, type: "participle" },
+  { ending: "ados", stem: "ar", confidence: 0.88, type: "participle" },
+  { ending: "adas", stem: "ar", confidence: 0.88, type: "participle" },
+  // Present indicative
+  { ending: "o", stem: "ar", confidence: 0.75, type: "present" },
+  // eu
+  { ending: "as", stem: "ar", confidence: 0.82, type: "present" },
+  // tu
+  { ending: "a", stem: "ar", confidence: 0.75, type: "present" },
+  // ele/ela/você
+  { ending: "amos", stem: "ar", confidence: 0.85, type: "present" },
+  // nós
+  { ending: "ais", stem: "ar", confidence: 0.85, type: "present" },
+  // vós
+  { ending: "am", stem: "ar", confidence: 0.8, type: "present" },
+  // eles/elas/vocês
+  // Preterite (past)
+  { ending: "ei", stem: "ar", confidence: 0.88, type: "past" },
+  // eu
+  { ending: "aste", stem: "ar", confidence: 0.88, type: "past" },
+  // tu
+  { ending: "ou", stem: "ar", confidence: 0.88, type: "past" },
+  // ele/ela/você
+  { ending: "\xE1mos", stem: "ar", confidence: 0.88, type: "past" },
+  // nós (with accent)
+  { ending: "amos", stem: "ar", confidence: 0.85, type: "past" },
+  // nós (Brazilian)
+  { ending: "astes", stem: "ar", confidence: 0.88, type: "past" },
+  // vós
+  { ending: "aram", stem: "ar", confidence: 0.88, type: "past" },
+  // eles/elas/vocês
+  // Imperfect
+  { ending: "ava", stem: "ar", confidence: 0.88, type: "past" },
+  // eu/ele
+  { ending: "avas", stem: "ar", confidence: 0.88, type: "past" },
+  // tu
+  { ending: "\xE1vamos", stem: "ar", confidence: 0.88, type: "past" },
+  // nós
+  { ending: "avamos", stem: "ar", confidence: 0.85, type: "past" },
+  // nós (no accent)
+  { ending: "\xE1veis", stem: "ar", confidence: 0.88, type: "past" },
+  // vós
+  { ending: "aveis", stem: "ar", confidence: 0.85, type: "past" },
+  // vós (no accent)
+  { ending: "avam", stem: "ar", confidence: 0.88, type: "past" },
+  // eles
+  // Subjunctive
+  { ending: "e", stem: "ar", confidence: 0.72, type: "subjunctive" },
+  // eu/ele (ambiguous)
+  { ending: "es", stem: "ar", confidence: 0.78, type: "subjunctive" },
+  // tu
+  { ending: "emos", stem: "ar", confidence: 0.82, type: "subjunctive" },
+  // nós
+  { ending: "eis", stem: "ar", confidence: 0.82, type: "subjunctive" },
+  // vós
+  { ending: "em", stem: "ar", confidence: 0.78, type: "subjunctive" },
+  // eles
+  // Imperative
+  { ending: "a", stem: "ar", confidence: 0.75, type: "imperative" },
+  // tu/você
+  { ending: "ai", stem: "ar", confidence: 0.85, type: "imperative" },
+  // vós
+  // Infinitive
+  { ending: "ar", stem: "ar", confidence: 0.92, type: "dictionary" }
+];
+var ER_ENDINGS = [
+  // Gerund (-endo)
+  { ending: "endo", stem: "er", confidence: 0.88, type: "gerund" },
+  // Past participle (-ido)
+  { ending: "ido", stem: "er", confidence: 0.85, type: "participle" },
+  { ending: "ida", stem: "er", confidence: 0.85, type: "participle" },
+  { ending: "idos", stem: "er", confidence: 0.85, type: "participle" },
+  { ending: "idas", stem: "er", confidence: 0.85, type: "participle" },
+  // Present indicative
+  { ending: "o", stem: "er", confidence: 0.72, type: "present" },
+  // eu
+  { ending: "es", stem: "er", confidence: 0.78, type: "present" },
+  // tu
+  { ending: "e", stem: "er", confidence: 0.72, type: "present" },
+  // ele
+  { ending: "emos", stem: "er", confidence: 0.85, type: "present" },
+  // nós
+  { ending: "eis", stem: "er", confidence: 0.82, type: "present" },
+  // vós
+  { ending: "em", stem: "er", confidence: 0.78, type: "present" },
+  // eles
+  // Preterite
+  { ending: "i", stem: "er", confidence: 0.85, type: "past" },
+  // eu
+  { ending: "este", stem: "er", confidence: 0.88, type: "past" },
+  // tu
+  { ending: "eu", stem: "er", confidence: 0.88, type: "past" },
+  // ele
+  { ending: "emos", stem: "er", confidence: 0.85, type: "past" },
+  // nós
+  { ending: "estes", stem: "er", confidence: 0.88, type: "past" },
+  // vós
+  { ending: "eram", stem: "er", confidence: 0.88, type: "past" },
+  // eles
+  // Imperfect
+  { ending: "ia", stem: "er", confidence: 0.85, type: "past" },
+  // eu/ele
+  { ending: "ias", stem: "er", confidence: 0.85, type: "past" },
+  // tu
+  { ending: "\xEDamos", stem: "er", confidence: 0.88, type: "past" },
+  // nós
+  { ending: "iamos", stem: "er", confidence: 0.85, type: "past" },
+  // nós (no accent)
+  { ending: "\xEDeis", stem: "er", confidence: 0.88, type: "past" },
+  // vós
+  { ending: "ieis", stem: "er", confidence: 0.85, type: "past" },
+  // vós (no accent)
+  { ending: "iam", stem: "er", confidence: 0.85, type: "past" },
+  // eles
+  // Infinitive
+  { ending: "er", stem: "er", confidence: 0.92, type: "dictionary" }
+];
+var IR_ENDINGS = [
+  // Gerund (-indo)
+  { ending: "indo", stem: "ir", confidence: 0.88, type: "gerund" },
+  // Past participle (-ido)
+  { ending: "ido", stem: "ir", confidence: 0.85, type: "participle" },
+  { ending: "ida", stem: "ir", confidence: 0.85, type: "participle" },
+  { ending: "idos", stem: "ir", confidence: 0.85, type: "participle" },
+  { ending: "idas", stem: "ir", confidence: 0.85, type: "participle" },
+  // Present indicative
+  { ending: "o", stem: "ir", confidence: 0.72, type: "present" },
+  // eu
+  { ending: "es", stem: "ir", confidence: 0.78, type: "present" },
+  // tu
+  { ending: "e", stem: "ir", confidence: 0.72, type: "present" },
+  // ele
+  { ending: "imos", stem: "ir", confidence: 0.85, type: "present" },
+  // nós
+  { ending: "is", stem: "ir", confidence: 0.82, type: "present" },
+  // vós
+  { ending: "em", stem: "ir", confidence: 0.78, type: "present" },
+  // eles
+  // Preterite (same as -er)
+  { ending: "i", stem: "ir", confidence: 0.85, type: "past" },
+  // eu
+  { ending: "iste", stem: "ir", confidence: 0.88, type: "past" },
+  // tu
+  { ending: "iu", stem: "ir", confidence: 0.88, type: "past" },
+  // ele
+  { ending: "imos", stem: "ir", confidence: 0.85, type: "past" },
+  // nós
+  { ending: "istes", stem: "ir", confidence: 0.88, type: "past" },
+  // vós
+  { ending: "iram", stem: "ir", confidence: 0.88, type: "past" },
+  // eles
+  // Imperfect (same as -er)
+  { ending: "ia", stem: "ir", confidence: 0.85, type: "past" },
+  { ending: "ias", stem: "ir", confidence: 0.85, type: "past" },
+  { ending: "\xEDamos", stem: "ir", confidence: 0.88, type: "past" },
+  { ending: "iamos", stem: "ir", confidence: 0.85, type: "past" },
+  { ending: "\xEDeis", stem: "ir", confidence: 0.88, type: "past" },
+  { ending: "ieis", stem: "ir", confidence: 0.85, type: "past" },
+  { ending: "iam", stem: "ir", confidence: 0.85, type: "past" },
+  // Infinitive
+  { ending: "ir", stem: "ir", confidence: 0.92, type: "dictionary" }
+];
+var ALL_ENDINGS = [...AR_ENDINGS, ...ER_ENDINGS, ...IR_ENDINGS].sort(
+  (a, b) => b.ending.length - a.ending.length
+);
+var PortugueseMorphologicalNormalizer = class {
+  constructor() {
+    this.language = "pt";
+  }
+  /**
+   * Check if a word might be a Portuguese verb that can be normalized.
+   */
+  isNormalizable(word) {
+    if (word.length < 3) return false;
+    return looksLikePortugueseVerb(word);
+  }
+  /**
+   * Normalize a Portuguese word to its infinitive form.
+   */
+  normalize(word) {
+    const lower = word.toLowerCase();
+    if (lower.endsWith("ar") || lower.endsWith("er") || lower.endsWith("ir")) {
+      if (!REFLEXIVE_SUFFIXES.some((s) => lower.endsWith(s))) {
+        return noChange(word);
+      }
+    }
+    const reflexiveResult = this.tryReflexiveNormalization(lower);
+    if (reflexiveResult) return reflexiveResult;
+    const conjugationResult = this.tryConjugationNormalization(lower);
+    if (conjugationResult) return conjugationResult;
+    return noChange(word);
+  }
+  /**
+   * Try to normalize a reflexive verb.
+   * Portuguese reflexive verbs use hyphenated pronouns: mostrar-se, esconder-me
+   *
+   * Examples:
+   *   mostrar-se → mostrar
+   *   esconder-se → esconder
+   *   exibir-se → exibir
+   */
+  tryReflexiveNormalization(word) {
+    for (const suffix of REFLEXIVE_SUFFIXES) {
+      if (word.endsWith(suffix)) {
+        const withoutReflexive = word.slice(0, -suffix.length);
+        if (withoutReflexive.endsWith("ar") || withoutReflexive.endsWith("er") || withoutReflexive.endsWith("ir")) {
+          return normalized(withoutReflexive, 0.88, {
+            removedSuffixes: [suffix],
+            conjugationType: "reflexive"
+          });
+        }
+        const innerResult = this.tryConjugationNormalization(withoutReflexive);
+        if (innerResult && innerResult.stem !== withoutReflexive) {
+          return normalized(innerResult.stem, innerResult.confidence * 0.95, {
+            removedSuffixes: [suffix, ...innerResult.metadata?.removedSuffixes || []],
+            conjugationType: "reflexive"
+          });
+        }
+      }
+    }
+    return null;
+  }
+  /**
+   * Try to normalize a conjugated verb to its infinitive.
+   */
+  tryConjugationNormalization(word) {
+    for (const rule of ALL_ENDINGS) {
+      if (word.endsWith(rule.ending)) {
+        const stemBase = word.slice(0, -rule.ending.length);
+        if (stemBase.length < 2) continue;
+        const infinitive = stemBase + rule.stem;
+        return normalized(infinitive, rule.confidence, {
+          removedSuffixes: [rule.ending],
+          conjugationType: rule.type
+        });
+      }
+    }
+    return null;
+  }
+};
+var portugueseMorphologicalNormalizer = new PortugueseMorphologicalNormalizer();
+
 // src/tokenizers/portuguese.ts
 var { isLetter: isPortugueseLetter, isIdentifierChar: isPortugueseIdentifierChar } = createLatinCharClassifiers(/[a-zA-ZáàâãéêíóôõúüçÁÀÂÃÉÊÍÓÔÕÚÜÇ]/);
 var PREPOSITIONS = /* @__PURE__ */ new Set([
@@ -1026,6 +1342,7 @@ var PortugueseTokenizer = class extends BaseTokenizer {
     this.language = "pt";
     this.direction = "ltr";
     this.initializeKeywordsFromProfile(portugueseProfile, PORTUGUESE_EXTRAS);
+    this.normalizer = new PortugueseMorphologicalNormalizer();
   }
   tokenize(input) {
     const tokens = [];
@@ -1042,6 +1359,10 @@ var PortugueseTokenizer = class extends BaseTokenizer {
           pos = modifierToken.position.end;
           continue;
         }
+        if (this.tryPropertyAccess(input, pos, tokens)) {
+          pos++;
+          continue;
+        }
         const selectorToken = this.trySelector(input, pos);
         if (selectorToken) {
           tokens.push(selectorToken);
@@ -1101,7 +1422,8 @@ var PortugueseTokenizer = class extends BaseTokenizer {
     const lower = token.toLowerCase();
     if (PREPOSITIONS.has(lower)) return "particle";
     if (this.isKeyword(lower)) return "keyword";
-    if (token.startsWith("#") || token.startsWith(".") || token.startsWith("[")) return "selector";
+    if (token.startsWith("#") || token.startsWith(".") || token.startsWith("[") || token.startsWith("*") || token.startsWith("<"))
+      return "selector";
     if (token.startsWith('"') || token.startsWith("'")) return "literal";
     if (/^\d/.test(token)) return "literal";
     return "identifier";