@lokascript/semantic 1.3.0 → 2.0.0

package/src/tokenizers/bengali.ts

@@ -40,7 +40,18 @@ const isBengali = createUnicodeRangeClassifier([[0x0980, 0x09ff]]);
 /**
  * Single-word postpositions.
  */
-const SINGLE_POSTPOSITIONS = new Set(['কে', 'তে', 'থেকে', 'র', 'এর', 'দিয়ে', 'জন্য', 'পর্যন্ত']);
+const SINGLE_POSTPOSITIONS = new Set([
+  'কে',
+  'তে',
+  'থেকে',
+  'র',
+  'এর',
+  'দিয়ে',
+  'জন্য',
+  'পর্যন্ত',
+  'এ', // locative "at/on/in" — marks events (ক্লিক এ = "on click")
+  'মধ্যে', // "in/within"
+]);
 
 // =============================================================================
 // Bengali Extras (not in profile)
@@ -112,6 +123,12 @@ export class BengaliTokenizer extends BaseTokenizer {
           continue;
         }
 
+        // Check for property access (obj.prop) vs CSS selector (.active)
+        if (this.tryPropertyAccess(input, pos, tokens)) {
+          pos++;
+          continue;
+        }
+
         const selectorToken = this.trySelector(input, pos);
         if (selectorToken) {
           tokens.push(selectorToken);
@@ -182,16 +199,19 @@ export class BengaliTokenizer extends BaseTokenizer {
           pos++;
         }
 
-        // Check if it's a keyword (O(1) Map lookup)
-        const keywordEntry = this.lookupKeyword(word);
-        if (keywordEntry) {
-          tokens.push(
-            createToken(word, 'keyword', createPosition(startPos, pos), keywordEntry.normalized)
-          );
-        } else if (SINGLE_POSTPOSITIONS.has(word)) {
+        // Postpositions take priority — they are grammatical particles, not keywords
+        if (SINGLE_POSTPOSITIONS.has(word)) {
           tokens.push(createToken(word, 'particle', createPosition(startPos, pos)));
         } else {
-          tokens.push(createToken(word, 'identifier', createPosition(startPos, pos)));
+          // Check if it's a keyword (O(1) Map lookup)
+          const keywordEntry = this.lookupKeyword(word);
+          if (keywordEntry) {
+            tokens.push(
+              createToken(word, 'keyword', createPosition(startPos, pos), keywordEntry.normalized)
+            );
+          } else {
+            tokens.push(createToken(word, 'identifier', createPosition(startPos, pos)));
+          }
         }
         continue;
       }
@@ -234,7 +254,14 @@ export class BengaliTokenizer extends BaseTokenizer {
       return 'keyword';
     }
     if (SINGLE_POSTPOSITIONS.has(value)) return 'particle';
-    if (value.startsWith('.') || value.startsWith('#') || value.startsWith('[')) return 'selector';
+    if (
+      value.startsWith('.') ||
+      value.startsWith('#') ||
+      value.startsWith('[') ||
+      value.startsWith('*') ||
+      value.startsWith('<')
+    )
+      return 'selector';
     if (value.startsWith(':')) return 'identifier';
     if (value.startsWith('"') || value.startsWith("'")) return 'literal';
     if (/^-?\d/.test(value)) return 'literal';

package/src/tokenizers/char-classifiers.ts

@@ -0,0 +1,79 @@
+/**
+ * Character Classifiers
+ *
+ * Unicode range classification and Latin character classifier factories.
+ * Used by language-specific tokenizers to define character sets.
+ */
+
+// =============================================================================
+// Unicode Range Classification
+// =============================================================================
+
+/**
+ * Unicode range tuple: [start, end] (inclusive).
+ */
+export type UnicodeRange = readonly [number, number];
+
+/**
+ * Create a character classifier for Unicode ranges.
+ * Returns a function that checks if a character's code point falls within any of the ranges.
+ *
+ * @example
+ * // Japanese Hiragana
+ * const isHiragana = createUnicodeRangeClassifier([[0x3040, 0x309f]]);
+ *
+ * // Korean (Hangul syllables + Jamo)
+ * const isKorean = createUnicodeRangeClassifier([
+ *   [0xac00, 0xd7a3],  // Hangul syllables
+ *   [0x1100, 0x11ff],  // Hangul Jamo
+ *   [0x3130, 0x318f],  // Hangul Compatibility Jamo
+ * ]);
+ */
+export function createUnicodeRangeClassifier(
+  ranges: readonly UnicodeRange[]
+): (char: string) => boolean {
+  return (char: string): boolean => {
+    const code = char.charCodeAt(0);
+    return ranges.some(([start, end]) => code >= start && code <= end);
+  };
+}
+
+/**
+ * Combine multiple character classifiers into one.
+ * Returns true if any of the classifiers return true.
+ *
+ * @example
+ * const isJapanese = combineClassifiers(isHiragana, isKatakana, isKanji);
+ */
+export function combineClassifiers(
+  ...classifiers: Array<(char: string) => boolean>
+): (char: string) => boolean {
+  return (char: string): boolean => classifiers.some(fn => fn(char));
+}
+
+/**
+ * Character classifiers for a Latin-based language.
+ */
+export interface LatinCharClassifiers {
+  /** Check if character is a letter in this language (including accented chars). */
+  isLetter: (char: string) => boolean;
+  /** Check if character is part of an identifier (letter, digit, underscore, hyphen). */
+  isIdentifierChar: (char: string) => boolean;
+}
+
+/**
+ * Create character classifiers for a Latin-based language.
+ * Returns isLetter and isIdentifierChar functions based on the provided regex.
+ *
+ * @example
+ * // Spanish letters
+ * const { isLetter, isIdentifierChar } = createLatinCharClassifiers(/[a-zA-ZáéíóúüñÁÉÍÓÚÜÑ]/);
+ *
+ * // German letters
+ * const { isLetter, isIdentifierChar } = createLatinCharClassifiers(/[a-zA-ZäöüÄÖÜß]/);
+ */
+export function createLatinCharClassifiers(letterPattern: RegExp): LatinCharClassifiers {
+  const isLetter = (char: string): boolean => letterPattern.test(char);
+  const isIdentifierChar = (char: string): boolean => isLetter(char) || /[0-9_-]/.test(char);
+  return { isLetter, isIdentifierChar };
+}

package/src/tokenizers/chinese.ts

@@ -232,6 +232,12 @@ export class ChineseTokenizer extends BaseTokenizer {
           continue;
         }
 
+        // Check for property access (obj.prop) vs CSS selector (.active)
+        if (this.tryPropertyAccess(input, pos, tokens)) {
+          pos++;
+          continue;
+        }
+
         const selectorToken = this.trySelector(input, pos);
         if (selectorToken) {
           tokens.push(selectorToken);
@@ -328,6 +334,7 @@ export class ChineseTokenizer extends BaseTokenizer {
       token.startsWith('#') ||
       token.startsWith('.') ||
       token.startsWith('[') ||
+      token.startsWith('*') ||
       token.startsWith('<')
     )
       return 'selector';

package/src/tokenizers/english.ts

@@ -303,7 +303,14 @@ export class EnglishTokenizer extends BaseTokenizer {
   classifyToken(token: string): TokenKind {
     // O(1) Map lookup instead of O(n) array search
     if (this.isKeyword(token)) return 'keyword';
-    if (token.startsWith('#') || token.startsWith('.') || token.startsWith('[')) return 'selector';
+    if (
+      token.startsWith('#') ||
+      token.startsWith('.') ||
+      token.startsWith('[') ||
+      token.startsWith('*') ||
+      token.startsWith('<')
+    )
+      return 'selector';
     if (token.startsWith('"') || token.startsWith("'")) return 'literal';
     if (/^\d/.test(token)) return 'literal';
     if (['==', '!=', '<=', '>=', '<', '>', '&&', '||', '!'].includes(token)) return 'operator';

package/src/tokenizers/extractors.ts

@@ -0,0 +1,452 @@
+/**
+ * Extraction Utilities
+ *
+ * Pure functions for extracting CSS selectors, string literals, URLs, and numbers
+ * from input strings. These are language-independent and used by all tokenizers.
+ */
+
+import {
+  isSelectorStart,
+  isWhitespace,
+  isAsciiIdentifierChar,
+  isAsciiLetter,
+  isQuote,
+  isDigit,
+} from './token-utils';
+
+// =============================================================================
+// CSS Selector Tokenization
+// =============================================================================
+
+/**
+ * Extract a CSS selector from the input string starting at pos.
+ * CSS selectors are universal across languages.
+ *
+ * Supported formats:
+ * - #id
+ * - .class
+ * - [attribute]
+ * - [attribute=value]
+ * - @attribute (shorthand)
+ * - *property (CSS property shorthand)
+ * - Complex selectors with combinators (limited)
+ *
+ * Method call handling:
+ * - #dialog.showModal() → stops after #dialog (method call, not compound selector)
+ * - #box.active → compound selector (no parens)
+ */
+export function extractCssSelector(input: string, startPos: number): string | null {
+  if (startPos >= input.length) return null;
+
+  const char = input[startPos];
+  if (!isSelectorStart(char)) return null;
+
+  let pos = startPos;
+  let selector = '';
+
+  // Handle different selector types
+  if (char === '#' || char === '.') {
+    // ID or class selector: #id, .class
+    selector += input[pos++];
+    while (pos < input.length && isAsciiIdentifierChar(input[pos])) {
+      selector += input[pos++];
+    }
+    // Must have at least one character after prefix
+    if (selector.length <= 1) return null;
+
+    // Check for method call pattern: #id.method() or .class.method()
+    // If we see .identifier followed by (, don't consume it - it's a method call
+    if (pos < input.length && input[pos] === '.' && char === '#') {
+      // Look ahead to see if this is a method call
+      const methodStart = pos + 1;
+      let methodEnd = methodStart;
+      while (methodEnd < input.length && isAsciiIdentifierChar(input[methodEnd])) {
+        methodEnd++;
+      }
+      // If followed by (, it's a method call - stop here
+      if (methodEnd < input.length && input[methodEnd] === '(') {
+        return selector;
+      }
+    }
+  } else if (char === '[') {
+    // Attribute selector: [attr] or [attr=value] or [attr="value"]
+    // Need to track quote state to avoid counting brackets inside quotes
+    let depth = 1;
+    let inQuote = false;
+    let quoteChar: string | null = null;
+    let escaped = false;
+
+    selector += input[pos++]; // [
+
+    while (pos < input.length && depth > 0) {
+      const c = input[pos];
+      selector += c;
+
+      if (escaped) {
+        // Skip escaped character
+        escaped = false;
+      } else if (c === '\\') {
+        // Next character is escaped
+        escaped = true;
+      } else if (inQuote) {
+        // Inside a quoted string
+        if (c === quoteChar) {
+          inQuote = false;
+          quoteChar = null;
+        }
+      } else {
+        // Not inside a quoted string
+        if (c === '"' || c === "'" || c === '`') {
+          inQuote = true;
+          quoteChar = c;
+        } else if (c === '[') {
+          depth++;
+        } else if (c === ']') {
+          depth--;
+        }
+      }
+      pos++;
+    }
+    if (depth !== 0) return null;
+  } else if (char === '@') {
+    // Attribute shorthand: @disabled
+    selector += input[pos++];
+    while (pos < input.length && isAsciiIdentifierChar(input[pos])) {
+      selector += input[pos++];
+    }
+    if (selector.length <= 1) return null;
+  } else if (char === '*') {
+    // CSS property shorthand: *display
+    selector += input[pos++];
+    while (pos < input.length && isAsciiIdentifierChar(input[pos])) {
+      selector += input[pos++];
+    }
+    if (selector.length <= 1) return null;
+  } else if (char === '<') {
+    // HTML literal selector with optional modifiers and attributes:
+    // - <div>
+    // - <div.class>
+    // - <div#id>
+    // - <div.class#id>
+    // - <button[disabled]/>
+    // - <div.card/>
+    // - <div.class#id[attr="value"]/>
+    selector += input[pos++]; // <
+
+    // Must be followed by an identifier (tag name)
+    if (pos >= input.length || !isAsciiLetter(input[pos])) return null;
+
+    // Extract tag name
+    while (pos < input.length && isAsciiIdentifierChar(input[pos])) {
+      selector += input[pos++];
+    }
+
+    // Process modifiers and attributes
+    // Can have multiple .class, one #id, and multiple [attr] in any order
+    while (pos < input.length) {
+      const modChar = input[pos];
+
+      if (modChar === '.') {
+        // Class modifier
+        selector += input[pos++]; // .
+        if (pos >= input.length || !isAsciiIdentifierChar(input[pos])) {
+          return null; // Invalid - class name required after .
+        }
+        while (pos < input.length && isAsciiIdentifierChar(input[pos])) {
+          selector += input[pos++];
+        }
+      } else if (modChar === '#') {
+        // ID modifier
+        selector += input[pos++]; // #
+        if (pos >= input.length || !isAsciiIdentifierChar(input[pos])) {
+          return null; // Invalid - ID required after #
+        }
+        while (pos < input.length && isAsciiIdentifierChar(input[pos])) {
+          selector += input[pos++];
+        }
+      } else if (modChar === '[') {
+        // Attribute modifier: [disabled] or [type="button"]
+        // Need to track quote state to avoid counting brackets inside quotes
+        let depth = 1;
+        let inQuote = false;
+        let quoteChar: string | null = null;
+        let escaped = false;
+
+        selector += input[pos++]; // [
+
+        while (pos < input.length && depth > 0) {
+          const c = input[pos];
+          selector += c;
+
+          if (escaped) {
+            escaped = false;
+          } else if (c === '\\') {
+            escaped = true;
+          } else if (inQuote) {
+            if (c === quoteChar) {
+              inQuote = false;
+              quoteChar = null;
+            }
+          } else {
+            if (c === '"' || c === "'" || c === '`') {
+              inQuote = true;
+              quoteChar = c;
+            } else if (c === '[') {
+              depth++;
+            } else if (c === ']') {
+              depth--;
+            }
+          }
+          pos++;
+        }
+        if (depth !== 0) return null; // Unclosed bracket
+      } else {
+        // No more modifiers
+        break;
+      }
+    }
+
+    // Skip whitespace before optional self-closing /
+    while (pos < input.length && isWhitespace(input[pos])) {
+      selector += input[pos++];
+    }
+
+    // Optional self-closing /
+    if (pos < input.length && input[pos] === '/') {
+      selector += input[pos++];
+      // Skip whitespace after /
+      while (pos < input.length && isWhitespace(input[pos])) {
+        selector += input[pos++];
+      }
+    }
+
+    // Must end with >
+    if (pos >= input.length || input[pos] !== '>') return null;
+    selector += input[pos++]; // >
+  }
+
+  return selector || null;
+}
+
+// =============================================================================
+// String Literal Tokenization
+// =============================================================================
+
+/**
+ * Check if a single quote at pos is a possessive marker ('s).
+ * Returns true if this looks like possessive, not a string start.
+ *
+ * Examples:
+ * - #element's *opacity → possessive (returns true)
+ * - 'hello' → string (returns false)
+ * - it's value → possessive (returns true)
+ */
+export function isPossessiveMarker(input: string, pos: number): boolean {
+  if (pos >= input.length || input[pos] !== "'") return false;
+
+  // Check if followed by 's' or 'S'
+  if (pos + 1 >= input.length) return false;
+  const nextChar = input[pos + 1].toLowerCase();
+  if (nextChar !== 's') return false;
+
+  // After 's, should be end, whitespace, or special char (not alphanumeric)
+  if (pos + 2 >= input.length) return true; // end of input
+  const afterS = input[pos + 2];
+  return isWhitespace(afterS) || afterS === '*' || !isAsciiIdentifierChar(afterS);
+}
+
+/**
+ * Extract a string literal from the input starting at pos.
+ * Handles both ASCII quotes and Unicode quotes.
+ *
+ * Note: Single quotes that look like possessive markers ('s) are skipped.
+ */
+export function extractStringLiteral(input: string, startPos: number): string | null {
+  if (startPos >= input.length) return null;
+
+  const openQuote = input[startPos];
+  if (!isQuote(openQuote)) return null;
+
+  // Check for possessive marker - don't treat as string
+  if (openQuote === "'" && isPossessiveMarker(input, startPos)) {
+    return null;
+  }
+
+  // Map opening quotes to closing quotes
+  const closeQuoteMap: Record<string, string> = {
+    '"': '"',
+    "'": "'",
+    '`': '`',
+    '「': '」',
+  };
+
+  const closeQuote = closeQuoteMap[openQuote];
+  if (!closeQuote) return null;
+
+  let pos = startPos + 1;
+  let literal = openQuote;
+  let escaped = false;
+
+  while (pos < input.length) {
+    const char = input[pos];
+    literal += char;
+
+    if (escaped) {
+      escaped = false;
+    } else if (char === '\\') {
+      escaped = true;
+    } else if (char === closeQuote) {
+      // Found closing quote
+      return literal;
+    }
+    pos++;
+  }
+
+  // Unclosed string - return what we have
+  return literal;
+}
+
+// =============================================================================
+// URL Tokenization
+// =============================================================================
+
+/**
+ * Check if the input at position starts a URL.
+ * Detects: /path, ./path, ../path, //domain.com, http://, https://
+ */
+export function isUrlStart(input: string, pos: number): boolean {
+  if (pos >= input.length) return false;
+
+  const char = input[pos];
+  const next = input[pos + 1] || '';
+  const third = input[pos + 2] || '';
+
+  // Absolute path: /something (but not just /)
+  // Must be followed by alphanumeric or path char, not another / (that's protocol-relative)
+  if (char === '/' && next !== '/' && /[a-zA-Z0-9._-]/.test(next)) {
+    return true;
+  }
+
+  // Protocol-relative: //domain.com
+  if (char === '/' && next === '/' && /[a-zA-Z]/.test(third)) {
+    return true;
+  }
+
+  // Relative path: ./ or ../
+  if (char === '.' && (next === '/' || (next === '.' && third === '/'))) {
+    return true;
+  }
+
+  // Full URL: http:// or https://
+  const slice = input.slice(pos, pos + 8).toLowerCase();
+  if (slice.startsWith('http://') || slice.startsWith('https://')) {
+    return true;
+  }
+
+  return false;
+}
+
+/**
+ * Extract a URL from the input starting at pos.
+ * Handles paths, query strings, and fragments.
+ *
+ * Fragment (#) handling:
+ * - /page#section → includes fragment as part of URL
+ * - #id alone → not a URL (CSS selector)
+ */
+export function extractUrl(input: string, startPos: number): string | null {
+  if (!isUrlStart(input, startPos)) return null;
+
+  let pos = startPos;
+  let url = '';
+
+  // Core URL characters (RFC 3986 unreserved + sub-delims + path/query chars)
+  // Includes: letters, digits, and - . _ ~ : / ? # [ ] @ ! $ & ' ( ) * + , ; = %
+  const urlChars = /[a-zA-Z0-9/:._\-?&=%@+~!$'()*,;[\]]/;
+
+  while (pos < input.length) {
+    const char = input[pos];
+
+    // Special handling for #
+    if (char === '#') {
+      // Only include # if we have path content before it (it's a fragment)
+      // If # appears at URL start or after certain chars, stop (might be CSS selector)
+      if (url.length > 0 && /[a-zA-Z0-9/.]$/.test(url)) {
+        // Include fragment
+        url += char;
+        pos++;
+        // Consume fragment identifier (letters, digits, underscore, hyphen)
+        while (pos < input.length && /[a-zA-Z0-9_-]/.test(input[pos])) {
+          url += input[pos++];
+        }
+      }
+      // Stop either way - fragment consumed or # is separate token
+      break;
+    }
+
+    if (urlChars.test(char)) {
+      url += char;
+      pos++;
+    } else {
+      break;
+    }
+  }
+
+  // Minimum length validation
+  if (url.length < 2) return null;
+
+  return url;
+}
+
+// =============================================================================
+// Number Tokenization
+// =============================================================================
+
+/**
+ * Extract a number from the input starting at pos.
+ * Handles integers and decimals.
+ */
+export function extractNumber(input: string, startPos: number): string | null {
+  if (startPos >= input.length) return null;
+
+  const char = input[startPos];
+  if (!isDigit(char) && char !== '-' && char !== '+') return null;
+
+  let pos = startPos;
+  let number = '';
+
+  // Optional sign
+  if (input[pos] === '-' || input[pos] === '+') {
+    number += input[pos++];
+  }
+
+  // Must have at least one digit
+  if (pos >= input.length || !isDigit(input[pos])) {
+    return null;
+  }
+
+  // Integer part
+  while (pos < input.length && isDigit(input[pos])) {
+    number += input[pos++];
+  }
+
+  // Optional decimal part
+  if (pos < input.length && input[pos] === '.') {
+    number += input[pos++];
+    while (pos < input.length && isDigit(input[pos])) {
+      number += input[pos++];
+    }
+  }
+
+  // Optional duration suffix (s, ms, m, h)
+  if (pos < input.length) {
+    const suffix = input.slice(pos, pos + 2);
+    if (suffix === 'ms') {
+      number += 'ms';
+    } else if (input[pos] === 's' || input[pos] === 'm' || input[pos] === 'h') {
+      number += input[pos];
+    }
+  }
+
+  return number;
+}

package/src/tokenizers/french.ts

@@ -25,6 +25,7 @@ import {
   type TimeUnitMapping,
 } from './base';
 import { frenchProfile } from '../generators/profiles/french';
+import { FrenchMorphologicalNormalizer } from './morphology/french-normalizer';
 
 // =============================================================================
 // French Character Classification
@@ -188,6 +189,7 @@ export class FrenchTokenizer extends BaseTokenizer {
   constructor() {
     super();
     this.initializeKeywordsFromProfile(frenchProfile, FRENCH_EXTRAS);
+    this.normalizer = new FrenchMorphologicalNormalizer();
   }
 
   tokenize(input: string): TokenStream {
@@ -209,6 +211,12 @@ export class FrenchTokenizer extends BaseTokenizer {
           continue;
         }
 
+        // Check for property access (obj.prop) vs CSS selector (.active)
+        if (this.tryPropertyAccess(input, pos, tokens)) {
+          pos++;
+          continue;
+        }
+
         const selectorToken = this.trySelector(input, pos);
         if (selectorToken) {
           tokens.push(selectorToken);
@@ -281,7 +289,14 @@ export class FrenchTokenizer extends BaseTokenizer {
     if (PREPOSITIONS.has(lower)) return 'particle';
     // O(1) Map lookup instead of O(n) array search
     if (this.isKeyword(lower)) return 'keyword';
-    if (token.startsWith('#') || token.startsWith('.') || token.startsWith('[')) return 'selector';
+    if (
+      token.startsWith('#') ||
+      token.startsWith('.') ||
+      token.startsWith('[') ||
+      token.startsWith('*') ||
+      token.startsWith('<')
+    )
+      return 'selector';
     if (token.startsWith('"') || token.startsWith("'")) return 'literal';
     if (/^\d/.test(token)) return 'literal';
     return 'identifier';