@lokascript/semantic 1.3.0 → 2.0.0

package/dist/languages/ms.d.ts

@@ -1,32 +1,10 @@
-import { B as BaseTokenizer, T as TokenStream, a as TokenKind, L as LanguageProfile } from '../types-BY3Id07j.js';
-
 /**
- * Malay Tokenizer
+ * Malay Language Registration
  *
- * Tokenizes Malay hyperscript input.
- * Word order: SVO
- * Direction: ltr
- * Uses spaces: true
- *
- * This tokenizer derives keywords from the Malay profile (single source of truth)
- * with extras for literals, positional words, and event names.
- */
-
-declare class MalayTokenizer extends BaseTokenizer {
-    readonly language = "ms";
-    readonly direction: "ltr";
-    constructor();
-    tokenize(input: string): TokenStream;
-    classifyToken(token: string): TokenKind;
-}
-declare const malayTokenizer: MalayTokenizer;
-
-/**
- * Malay Language Profile
- *
- * SVO word order, prepositions, space-separated.
+ * Self-registering language module for Malay.
+ * Import this module to enable Malay language support.
  */
-
-declare const malayProfile: LanguageProfile;
-
-export { malayProfile, malayTokenizer };
+import { malayTokenizer } from '../tokenizers/ms';
+import { malayProfile } from '../generators/profiles/ms';
+export { malayTokenizer, malayProfile };
+//# sourceMappingURL=ms.d.ts.map

package/dist/languages/ms.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ms.d.ts","sourceRoot":"","sources":["../../src/languages/ms.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAMzD,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,CAAC"}

package/dist/languages/ms.js

@@ -8,7 +8,7 @@ function registerLanguage(code, tokenizer, profile) {
   patternCache.delete(code);
 }
 
-// src/tokenizers/base.ts
+// src/tokenizers/token-utils.ts
 var TokenStreamImpl = class {
   constructor(tokens, language) {
     this.pos = 0;
@@ -74,17 +74,14 @@ function createToken(value, kind, position, normalizedOrOptions) {
   }
   if (normalizedOrOptions) {
     const { normalized, stem, stemConfidence } = normalizedOrOptions;
-    const token = { value, kind, position };
-    if (normalized !== void 0) {
-      token.normalized = normalized;
-    }
-    if (stem !== void 0) {
-      token.stem = stem;
-      if (stemConfidence !== void 0) {
-        token.stemConfidence = stemConfidence;
-      }
-    }
-    return token;
+    return {
+      value,
+      kind,
+      position,
+      ...normalized !== void 0 && { normalized },
+      ...stem !== void 0 && { stem },
+      ...stemConfidence !== void 0 && { stemConfidence }
+    };
   }
   return { value, kind, position };
 }
@@ -106,6 +103,8 @@ function isAsciiLetter(char) {
 function isAsciiIdentifierChar(char) {
   return /[a-zA-Z0-9_-]/.test(char);
 }
+
+// src/tokenizers/extractors.ts
 function extractCssSelector(input, startPos) {
   if (startPos >= input.length) return null;
   const char = input[startPos];
@@ -363,6 +362,8 @@ function extractNumber(input, startPos) {
   }
   return number;
 }
+
+// src/tokenizers/base-tokenizer.ts
 var _BaseTokenizer = class _BaseTokenizer {
   constructor() {
     /** Keywords derived from profile, sorted longest-first for greedy matching */
@@ -370,6 +371,34 @@ var _BaseTokenizer = class _BaseTokenizer {
     /** Map for O(1) keyword lookups by lowercase native word */
     this.profileKeywordMap = /* @__PURE__ */ new Map();
   }
+  /**
+   * Check if current position is a property access (obj.prop) vs CSS selector (.active).
+   * Property access: no whitespace before '.', previous token is identifier/keyword/selector.
+   * Also detects standalone method calls: .identifier( pattern.
+   *
+   * Returns true if '.' was emitted as an operator token and pos should advance by 1.
+   * Returns false if this is a CSS selector and should be handled by trySelector().
+   */
+  tryPropertyAccess(input, pos, tokens) {
+    if (input[pos] !== ".") return false;
+    const lastToken = tokens[tokens.length - 1];
+    const hasWhitespaceBefore = lastToken && lastToken.position.end < pos;
+    const isPropertyAccess = lastToken && !hasWhitespaceBefore && (lastToken.kind === "identifier" || lastToken.kind === "keyword" || lastToken.kind === "selector");
+    if (isPropertyAccess) {
+      tokens.push(createToken(".", "operator", createPosition(pos, pos + 1)));
+      return true;
+    }
+    const methodStart = pos + 1;
+    let methodEnd = methodStart;
+    while (methodEnd < input.length && isAsciiIdentifierChar(input[methodEnd])) {
+      methodEnd++;
+    }
+    if (methodEnd < input.length && input[methodEnd] === "(") {
+      tokens.push(createToken(".", "operator", createPosition(pos, pos + 1)));
+      return true;
+    }
+    return false;
+  }
   /**
    * Initialize keyword mappings from a language profile.
    * Builds a list of native→english mappings from:
@@ -405,6 +434,11 @@ var _BaseTokenizer = class _BaseTokenizer {
       for (const [normalized, native] of Object.entries(profile.references)) {
         keywordMap.set(native, { native, normalized });
       }
+      for (const canonical of Object.keys(profile.references)) {
+        if (!keywordMap.has(canonical)) {
+          keywordMap.set(canonical, { native: canonical, normalized: canonical });
+        }
+      }
     }
     if (profile.roleMarkers) {
       for (const [role, marker] of Object.entries(profile.roleMarkers)) {
@@ -892,6 +926,12 @@ var malayProfile = {
     behavior: { primary: "kelakuan", normalized: "behavior" },
     install: { primary: "pasang", normalized: "install" },
     measure: { primary: "ukur", normalized: "measure" },
+    beep: { primary: "bunyi", normalized: "beep" },
+    break: { primary: "henti", normalized: "break" },
+    copy: { primary: "salin", normalized: "copy" },
+    exit: { primary: "keluar", normalized: "exit" },
+    pick: { primary: "pilih", normalized: "pick" },
+    render: { primary: "papar", normalized: "render" },
     // Modifiers
     into: { primary: "ke_dalam", normalized: "into" },
     before: { primary: "sebelum", normalized: "before" },
@@ -957,6 +997,10 @@ var MalayTokenizer = class extends BaseTokenizer {
           pos = modifierToken.position.end;
           continue;
         }
+        if (this.tryPropertyAccess(input, pos, tokens)) {
+          pos++;
+          continue;
+        }
         const selectorToken = this.trySelector(input, pos);
         if (selectorToken) {
           tokens.push(selectorToken);
@@ -1025,7 +1069,8 @@ var MalayTokenizer = class extends BaseTokenizer {
   }
   classifyToken(token) {
     if (this.isKeyword(token)) return "keyword";
-    if (token.startsWith(".") || token.startsWith("#") || token.startsWith("[")) return "selector";
+    if (token.startsWith(".") || token.startsWith("#") || token.startsWith("[") || token.startsWith("*") || token.startsWith("<"))
+      return "selector";
     if (token.startsWith(":")) return "identifier";
     if (token.startsWith('"') || token.startsWith("'")) return "literal";
     if (/^-?\d/.test(token)) return "literal";