@lokascript/semantic 1.3.0 → 2.0.0

package/dist/languages/th.js

@@ -8,7 +8,7 @@ function registerLanguage(code, tokenizer, profile) {
   patternCache.delete(code);
 }
 
-// src/tokenizers/base.ts
+// src/tokenizers/token-utils.ts
 var TokenStreamImpl = class {
   constructor(tokens, language) {
     this.pos = 0;
@@ -74,17 +74,14 @@ function createToken(value, kind, position, normalizedOrOptions) {
   }
   if (normalizedOrOptions) {
     const { normalized, stem, stemConfidence } = normalizedOrOptions;
-    const token = { value, kind, position };
-    if (normalized !== void 0) {
-      token.normalized = normalized;
-    }
-    if (stem !== void 0) {
-      token.stem = stem;
-      if (stemConfidence !== void 0) {
-        token.stemConfidence = stemConfidence;
-      }
-    }
-    return token;
+    return {
+      value,
+      kind,
+      position,
+      ...normalized !== void 0 && { normalized },
+      ...stem !== void 0 && { stem },
+      ...stemConfidence !== void 0 && { stemConfidence }
+    };
   }
   return { value, kind, position };
 }
@@ -106,12 +103,16 @@ function isAsciiLetter(char) {
 function isAsciiIdentifierChar(char) {
   return /[a-zA-Z0-9_-]/.test(char);
 }
+
+// src/tokenizers/char-classifiers.ts
 function createUnicodeRangeClassifier(ranges) {
   return (char) => {
     const code = char.charCodeAt(0);
     return ranges.some(([start, end]) => code >= start && code <= end);
   };
 }
+
+// src/tokenizers/extractors.ts
 function extractCssSelector(input, startPos) {
   if (startPos >= input.length) return null;
   const char = input[startPos];
@@ -369,6 +370,8 @@ function extractNumber(input, startPos) {
   }
   return number;
 }
+
+// src/tokenizers/base-tokenizer.ts
 var _BaseTokenizer = class _BaseTokenizer {
   constructor() {
     /** Keywords derived from profile, sorted longest-first for greedy matching */
@@ -376,6 +379,34 @@ var _BaseTokenizer = class _BaseTokenizer {
     /** Map for O(1) keyword lookups by lowercase native word */
     this.profileKeywordMap = /* @__PURE__ */ new Map();
   }
+  /**
+   * Check if current position is a property access (obj.prop) vs CSS selector (.active).
+   * Property access: no whitespace before '.', previous token is identifier/keyword/selector.
+   * Also detects standalone method calls: .identifier( pattern.
+   *
+   * Returns true if '.' was emitted as an operator token and pos should advance by 1.
+   * Returns false if this is a CSS selector and should be handled by trySelector().
+   */
+  tryPropertyAccess(input, pos, tokens) {
+    if (input[pos] !== ".") return false;
+    const lastToken = tokens[tokens.length - 1];
+    const hasWhitespaceBefore = lastToken && lastToken.position.end < pos;
+    const isPropertyAccess = lastToken && !hasWhitespaceBefore && (lastToken.kind === "identifier" || lastToken.kind === "keyword" || lastToken.kind === "selector");
+    if (isPropertyAccess) {
+      tokens.push(createToken(".", "operator", createPosition(pos, pos + 1)));
+      return true;
+    }
+    const methodStart = pos + 1;
+    let methodEnd = methodStart;
+    while (methodEnd < input.length && isAsciiIdentifierChar(input[methodEnd])) {
+      methodEnd++;
+    }
+    if (methodEnd < input.length && input[methodEnd] === "(") {
+      tokens.push(createToken(".", "operator", createPosition(pos, pos + 1)));
+      return true;
+    }
+    return false;
+  }
   /**
    * Initialize keyword mappings from a language profile.
    * Builds a list of native→english mappings from:
@@ -411,6 +442,11 @@ var _BaseTokenizer = class _BaseTokenizer {
       for (const [normalized, native] of Object.entries(profile.references)) {
         keywordMap.set(native, { native, normalized });
       }
+      for (const canonical of Object.keys(profile.references)) {
+        if (!keywordMap.has(canonical)) {
+          keywordMap.set(canonical, { native: canonical, normalized: canonical });
+        }
+      }
     }
     if (profile.roleMarkers) {
       for (const [role, marker] of Object.entries(profile.roleMarkers)) {
@@ -896,6 +932,12 @@ var thaiProfile = {
     behavior: { primary: "\u0E1E\u0E24\u0E15\u0E34\u0E01\u0E23\u0E23\u0E21", alternatives: [], normalized: "behavior" },
     install: { primary: "\u0E15\u0E34\u0E14\u0E15\u0E31\u0E49\u0E07", alternatives: [], normalized: "install" },
     measure: { primary: "\u0E27\u0E31\u0E14", alternatives: [], normalized: "measure" },
+    beep: { primary: "\u0E1A\u0E35\u0E4A\u0E1A", alternatives: [], normalized: "beep" },
+    break: { primary: "\u0E2B\u0E22\u0E38\u0E14", alternatives: [], normalized: "break" },
+    copy: { primary: "\u0E04\u0E31\u0E14\u0E25\u0E2D\u0E01", alternatives: [], normalized: "copy" },
+    exit: { primary: "\u0E2D\u0E2D\u0E01", alternatives: [], normalized: "exit" },
+    pick: { primary: "\u0E40\u0E25\u0E37\u0E2D\u0E01", alternatives: [], normalized: "pick" },
+    render: { primary: "\u0E41\u0E2A\u0E14\u0E07\u0E1C\u0E25", alternatives: [], normalized: "render" },
     // Modifiers
     into: { primary: "\u0E43\u0E19", alternatives: [], normalized: "into" },
     before: { primary: "\u0E01\u0E48\u0E2D\u0E19", alternatives: [], normalized: "before" },
@@ -970,6 +1012,10 @@ var ThaiTokenizer = class extends BaseTokenizer {
           pos = modifierToken.position.end;
           continue;
         }
+        if (this.tryPropertyAccess(input, pos, tokens)) {
+          pos++;
+          continue;
+        }
         const selectorToken = this.trySelector(input, pos);
         if (selectorToken) {
           tokens.push(selectorToken);
@@ -1058,7 +1104,8 @@ var ThaiTokenizer = class extends BaseTokenizer {
   }
   classifyToken(value) {
     if (this.isKeyword(value)) return "keyword";
-    if (value.startsWith(".") || value.startsWith("#") || value.startsWith("[")) return "selector";
+    if (value.startsWith(".") || value.startsWith("#") || value.startsWith("[") || value.startsWith("*") || value.startsWith("<"))
+      return "selector";
     if (value.startsWith(":")) return "identifier";
     if (value.startsWith('"') || value.startsWith("'")) return "literal";
     if (/^-?\d/.test(value)) return "literal";