@lokascript/semantic 1.2.1 → 2.0.0

package/dist/languages/de.js

@@ -8,7 +8,7 @@ function registerLanguage(code, tokenizer, profile) {
   patternCache.delete(code);
 }
 
-// src/tokenizers/base.ts
+// src/tokenizers/token-utils.ts
 var TokenStreamImpl = class {
   constructor(tokens, language) {
     this.pos = 0;
@@ -73,18 +73,15 @@ function createToken(value, kind, position, normalizedOrOptions) {
     return { value, kind, position, normalized: normalizedOrOptions };
   }
   if (normalizedOrOptions) {
-    const { normalized, stem, stemConfidence } = normalizedOrOptions;
-    const token = { value, kind, position };
-    if (normalized !== void 0) {
-      token.normalized = normalized;
-    }
-    if (stem !== void 0) {
-      token.stem = stem;
-      if (stemConfidence !== void 0) {
-        token.stemConfidence = stemConfidence;
-      }
-    }
-    return token;
+    const { normalized: normalized2, stem, stemConfidence } = normalizedOrOptions;
+    return {
+      value,
+      kind,
+      position,
+      ...normalized2 !== void 0 && { normalized: normalized2 },
+      ...stem !== void 0 && { stem },
+      ...stemConfidence !== void 0 && { stemConfidence }
+    };
   }
   return { value, kind, position };
 }
@@ -106,11 +103,15 @@ function isAsciiLetter(char) {
 function isAsciiIdentifierChar(char) {
   return /[a-zA-Z0-9_-]/.test(char);
 }
+
+// src/tokenizers/char-classifiers.ts
 function createLatinCharClassifiers(letterPattern) {
   const isLetter = (char) => letterPattern.test(char);
   const isIdentifierChar = (char) => isLetter(char) || /[0-9_-]/.test(char);
   return { isLetter, isIdentifierChar };
 }
+
+// src/tokenizers/extractors.ts
 function extractCssSelector(input, startPos) {
   if (startPos >= input.length) return null;
   const char = input[startPos];
@@ -368,6 +369,8 @@ function extractNumber(input, startPos) {
   }
   return number;
 }
+
+// src/tokenizers/base-tokenizer.ts
 var _BaseTokenizer = class _BaseTokenizer {
   constructor() {
     /** Keywords derived from profile, sorted longest-first for greedy matching */
@@ -375,6 +378,34 @@ var _BaseTokenizer = class _BaseTokenizer {
     /** Map for O(1) keyword lookups by lowercase native word */
     this.profileKeywordMap = /* @__PURE__ */ new Map();
   }
+  /**
+   * Check if current position is a property access (obj.prop) vs CSS selector (.active).
+   * Property access: no whitespace before '.', previous token is identifier/keyword/selector.
+   * Also detects standalone method calls: .identifier( pattern.
+   *
+   * Returns true if '.' was emitted as an operator token and pos should advance by 1.
+   * Returns false if this is a CSS selector and should be handled by trySelector().
+   */
+  tryPropertyAccess(input, pos, tokens) {
+    if (input[pos] !== ".") return false;
+    const lastToken = tokens[tokens.length - 1];
+    const hasWhitespaceBefore = lastToken && lastToken.position.end < pos;
+    const isPropertyAccess = lastToken && !hasWhitespaceBefore && (lastToken.kind === "identifier" || lastToken.kind === "keyword" || lastToken.kind === "selector");
+    if (isPropertyAccess) {
+      tokens.push(createToken(".", "operator", createPosition(pos, pos + 1)));
+      return true;
+    }
+    const methodStart = pos + 1;
+    let methodEnd = methodStart;
+    while (methodEnd < input.length && isAsciiIdentifierChar(input[methodEnd])) {
+      methodEnd++;
+    }
+    if (methodEnd < input.length && input[methodEnd] === "(") {
+      tokens.push(createToken(".", "operator", createPosition(pos, pos + 1)));
+      return true;
+    }
+    return false;
+  }
   /**
    * Initialize keyword mappings from a language profile.
    * Builds a list of native→english mappings from:
@@ -391,24 +422,29 @@ var _BaseTokenizer = class _BaseTokenizer {
   initializeKeywordsFromProfile(profile, extras = []) {
     const keywordMap = /* @__PURE__ */ new Map();
     if (profile.keywords) {
-      for (const [normalized, translation] of Object.entries(profile.keywords)) {
+      for (const [normalized2, translation] of Object.entries(profile.keywords)) {
         keywordMap.set(translation.primary, {
           native: translation.primary,
-          normalized: translation.normalized || normalized
+          normalized: translation.normalized || normalized2
         });
         if (translation.alternatives) {
           for (const alt of translation.alternatives) {
             keywordMap.set(alt, {
               native: alt,
-              normalized: translation.normalized || normalized
+              normalized: translation.normalized || normalized2
             });
           }
         }
       }
     }
     if (profile.references) {
-      for (const [normalized, native] of Object.entries(profile.references)) {
-        keywordMap.set(native, { native, normalized });
+      for (const [normalized2, native] of Object.entries(profile.references)) {
+        keywordMap.set(native, { native, normalized: normalized2 });
+      }
+      for (const canonical of Object.keys(profile.references)) {
+        if (!keywordMap.has(canonical)) {
+          keywordMap.set(canonical, { native: canonical, normalized: canonical });
+        }
       }
     }
     if (profile.roleMarkers) {
@@ -432,9 +468,9 @@ var _BaseTokenizer = class _BaseTokenizer {
     this.profileKeywordMap = /* @__PURE__ */ new Map();
     for (const keyword of this.profileKeywords) {
       this.profileKeywordMap.set(keyword.native.toLowerCase(), keyword);
-      const normalized = this.removeDiacritics(keyword.native);
-      if (normalized !== keyword.native && !this.profileKeywordMap.has(normalized.toLowerCase())) {
-        this.profileKeywordMap.set(normalized.toLowerCase(), keyword);
+      const normalized2 = this.removeDiacritics(keyword.native);
+      if (normalized2 !== keyword.native && !this.profileKeywordMap.has(normalized2.toLowerCase())) {
+        this.profileKeywordMap.set(normalized2.toLowerCase(), keyword);
       }
     }
   }
@@ -888,6 +924,12 @@ var germanProfile = {
     behavior: { primary: "verhalten", normalized: "behavior" },
     install: { primary: "installieren", normalized: "install" },
     measure: { primary: "messen", normalized: "measure" },
+    beep: { primary: "piepton", normalized: "beep" },
+    break: { primary: "unterbrechen", normalized: "break" },
+    copy: { primary: "kopieren", normalized: "copy" },
+    exit: { primary: "beenden", normalized: "exit" },
+    pick: { primary: "ausw\xE4hlen", normalized: "pick" },
+    render: { primary: "rendern", normalized: "render" },
     into: { primary: "hinein", normalized: "into" },
     before: { primary: "vor", normalized: "before" },
     after: { primary: "nach", normalized: "after" },
@@ -915,6 +957,196 @@ var germanProfile = {
   }
 };
 
+// src/tokenizers/morphology/types.ts
+function noChange(word) {
+  return { stem: word, confidence: 1 };
+}
+function normalized(stem, confidence, metadata) {
+  if (metadata) {
+    return { stem, confidence, metadata };
+  }
+  return { stem, confidence };
+}
+
+// src/tokenizers/morphology/german-normalizer.ts
+var SEPARABLE_PREFIXES = [
+  "an",
+  "auf",
+  "aus",
+  "ein",
+  "mit",
+  "vor",
+  "zu",
+  "ab",
+  "bei",
+  "nach",
+  "weg",
+  "um",
+  "her",
+  "hin"
+];
+function looksLikeGermanVerb(word) {
+  const lower = word.toLowerCase();
+  if (lower.endsWith("en") || lower.endsWith("eln") || lower.endsWith("ern")) return true;
+  if (lower.startsWith("ge") && lower.endsWith("t")) return true;
+  if (lower.startsWith("ge") && lower.endsWith("en")) return true;
+  if (/[äöüß]/i.test(word)) return true;
+  return false;
+}
+var VERB_ENDINGS = [
+  // Present participle
+  { ending: "end", stem: "en", confidence: 0.88, type: "gerund" },
+  // Present indicative (regular weak verbs)
+  { ending: "e", stem: "en", confidence: 0.75, type: "present" },
+  // ich
+  { ending: "st", stem: "en", confidence: 0.8, type: "present" },
+  // du
+  { ending: "t", stem: "en", confidence: 0.78, type: "present" },
+  // er/sie/es, ihr
+  { ending: "en", stem: "en", confidence: 0.85, type: "dictionary" },
+  // wir/sie/Sie, infinitive
+  // Past tense (weak verbs: -te, -test, -te, -ten, -tet, -ten)
+  { ending: "test", stem: "en", confidence: 0.85, type: "past" },
+  // du
+  { ending: "ten", stem: "en", confidence: 0.82, type: "past" },
+  // wir/sie/Sie
+  { ending: "tet", stem: "en", confidence: 0.85, type: "past" },
+  // ihr
+  { ending: "te", stem: "en", confidence: 0.82, type: "past" },
+  // ich/er/sie/es
+  // Subjunctive II (weak verbs)
+  { ending: "test", stem: "en", confidence: 0.8, type: "subjunctive" },
+  { ending: "ten", stem: "en", confidence: 0.78, type: "subjunctive" },
+  { ending: "tet", stem: "en", confidence: 0.8, type: "subjunctive" },
+  { ending: "te", stem: "en", confidence: 0.78, type: "subjunctive" },
+  // Imperative
+  { ending: "e", stem: "en", confidence: 0.72, type: "imperative" },
+  // du (informal singular)
+  { ending: "t", stem: "en", confidence: 0.72, type: "imperative" },
+  // ihr (informal plural)
+  { ending: "en", stem: "en", confidence: 0.75, type: "imperative" }
+  // Sie (formal)
+];
+var ELN_ERN_ENDINGS = [
+  // Present
+  { ending: "le", stem: "eln", confidence: 0.82, type: "present" },
+  // ich sammle
+  { ending: "elst", stem: "eln", confidence: 0.85, type: "present" },
+  // du sammelst
+  { ending: "elt", stem: "eln", confidence: 0.85, type: "present" },
+  // er/sie/es sammelt
+  { ending: "eln", stem: "eln", confidence: 0.88, type: "dictionary" },
+  // infinitive
+  { ending: "re", stem: "ern", confidence: 0.82, type: "present" },
+  // ich wandre
+  { ending: "erst", stem: "ern", confidence: 0.85, type: "present" },
+  // du wanderst
+  { ending: "ert", stem: "ern", confidence: 0.85, type: "present" },
+  // er/sie/es wandert
+  { ending: "ern", stem: "ern", confidence: 0.88, type: "dictionary" }
+  // infinitive
+];
+var ALL_ENDINGS = [...VERB_ENDINGS, ...ELN_ERN_ENDINGS].sort(
+  (a, b) => b.ending.length - a.ending.length
+);
+var GermanMorphologicalNormalizer = class {
+  constructor() {
+    this.language = "de";
+  }
+  /**
+   * Check if a word might be a German verb that can be normalized.
+   */
+  isNormalizable(word) {
+    if (word.length < 3) return false;
+    return looksLikeGermanVerb(word);
+  }
+  /**
+   * Normalize a German word to its infinitive form.
+   */
+  normalize(word) {
+    const lower = word.toLowerCase();
+    if (lower.endsWith("en") && lower.length >= 4) {
+      return noChange(word);
+    }
+    if ((lower.endsWith("eln") || lower.endsWith("ern")) && lower.length >= 5) {
+      return noChange(word);
+    }
+    const participleResult = this.tryParticipleNormalization(lower);
+    if (participleResult) return participleResult;
+    const conjugationResult = this.tryConjugationNormalization(lower);
+    if (conjugationResult) return conjugationResult;
+    return noChange(word);
+  }
+  /**
+   * Try to normalize a past participle.
+   * German past participles often have ge- prefix and -t or -en suffix.
+   *
+   * Examples:
+   *   gemacht → machen (weak verb)
+   *   gegangen → gehen (strong verb)
+   *   angemacht → anmachen (separable prefix)
+   */
+  tryParticipleNormalization(word) {
+    for (const prefix of SEPARABLE_PREFIXES) {
+      if (word.startsWith(prefix + "ge")) {
+        const afterPrefix = word.slice(prefix.length);
+        const innerResult = this.trySimpleParticipleNormalization(afterPrefix);
+        if (innerResult) {
+          const metadata = {
+            removedPrefixes: ["ge"],
+            conjugationType: "participle"
+          };
+          if (innerResult.metadata?.removedSuffixes) {
+            metadata.removedSuffixes = innerResult.metadata.removedSuffixes;
+          }
+          return normalized(prefix + innerResult.stem, innerResult.confidence * 0.95, metadata);
+        }
+      }
+    }
+    return this.trySimpleParticipleNormalization(word);
+  }
+  /**
+   * Try to normalize a simple ge-...-t or ge-...-en participle.
+   */
+  trySimpleParticipleNormalization(word) {
+    if (!word.startsWith("ge")) return null;
+    const withoutGe = word.slice(2);
+    if (withoutGe.endsWith("t") && withoutGe.length >= 3) {
+      const stem = withoutGe.slice(0, -1);
+      return normalized(stem + "en", 0.85, {
+        removedPrefixes: ["ge"],
+        removedSuffixes: ["t"],
+        conjugationType: "participle"
+      });
+    }
+    if (withoutGe.endsWith("en") && withoutGe.length >= 4) {
+      return normalized(withoutGe, 0.82, {
+        removedPrefixes: ["ge"],
+        conjugationType: "participle"
+      });
+    }
+    return null;
+  }
+  /**
+   * Try to normalize a conjugated verb to its infinitive.
+   */
+  tryConjugationNormalization(word) {
+    for (const rule of ALL_ENDINGS) {
+      if (word.endsWith(rule.ending)) {
+        const stemBase = word.slice(0, -rule.ending.length);
+        if (stemBase.length < 2) continue;
+        const infinitive = stemBase + rule.stem;
+        return normalized(infinitive, rule.confidence, {
+          removedSuffixes: [rule.ending],
+          conjugationType: rule.type
+        });
+      }
+    }
+    return null;
+  }
+};
+var germanMorphologicalNormalizer = new GermanMorphologicalNormalizer();
+
 // src/tokenizers/german.ts
 var { isLetter: isGermanLetter, isIdentifierChar: isGermanIdentifierChar } = createLatinCharClassifiers(/[a-zA-ZäöüÄÖÜß]/);
 var PREPOSITIONS = /* @__PURE__ */ new Set([
@@ -1062,6 +1294,7 @@ var GermanTokenizer = class extends BaseTokenizer {
     this.language = "de";
     this.direction = "ltr";
     this.initializeKeywordsFromProfile(germanProfile, GERMAN_EXTRAS);
+    this.normalizer = new GermanMorphologicalNormalizer();
   }
   tokenize(input) {
     const tokens = [];
@@ -1078,6 +1311,10 @@ var GermanTokenizer = class extends BaseTokenizer {
           pos = modifierToken.position.end;
           continue;
         }
+        if (this.tryPropertyAccess(input, pos, tokens)) {
+          pos++;
+          continue;
+        }
         const selectorToken = this.trySelector(input, pos);
         if (selectorToken) {
           tokens.push(selectorToken);
@@ -1137,7 +1374,8 @@ var GermanTokenizer = class extends BaseTokenizer {
     const lower = token.toLowerCase();
     if (PREPOSITIONS.has(lower)) return "particle";
     if (this.isKeyword(lower)) return "keyword";
-    if (token.startsWith("#") || token.startsWith(".") || token.startsWith("[")) return "selector";
+    if (token.startsWith("#") || token.startsWith(".") || token.startsWith("[") || token.startsWith("*") || token.startsWith("<"))
+      return "selector";
     if (token.startsWith('"') || token.startsWith("'")) return "literal";
     if (/^\d/.test(token)) return "literal";
     return "identifier";