@lokascript/semantic 1.0.0

package/src/tokenizers/hindi.ts

@@ -0,0 +1,319 @@
+/**
+ * Hindi Tokenizer
+ *
+ * Tokenizes Hindi hyperscript input.
+ * Hindi is a fusional SOV language with:
+ * - Devanagari script (U+0900-U+097F)
+ * - Postposition markers (को, में, पर, से, etc.)
+ * - Verb conjugations with stem + suffix patterns
+ * - CSS selectors are embedded ASCII
+ */
+
+import type { LanguageToken, TokenKind, TokenStream } from '../types';
+import {
+  BaseTokenizer,
+  TokenStreamImpl,
+  createToken,
+  createPosition,
+  createUnicodeRangeClassifier,
+  isWhitespace,
+  isSelectorStart,
+  isQuote,
+  isDigit,
+  isAsciiIdentifierChar,
+  isUrlStart,
+  type KeywordEntry,
+} from './base';
+import { hindiProfile } from '../generators/profiles/hindi';
+
+// =============================================================================
+// Hindi Character Classification
+// =============================================================================
+
+/** Check if character is in the Devanagari script range (U+0900-U+097F, U+A8E0-U+A8FF). */
+const isDevanagari = createUnicodeRangeClassifier([
+  [0x0900, 0x097f], // Devanagari
+  [0xa8e0, 0xa8ff], // Devanagari Extended
+]);
+
+/** Check if character is Hindi (Devanagari). */
+const isHindi = isDevanagari;
+
+// =============================================================================
+// Hindi Postpositions
+// =============================================================================
+
+/**
+ * Single-word postpositions.
+ */
+const SINGLE_POSTPOSITIONS = new Set(['को', 'में', 'पर', 'से', 'का', 'की', 'के', 'तक', 'ने']);
+
+// =============================================================================
+// Hindi Extras (not in profile)
+// =============================================================================
+
+/**
+ * Extra keywords not covered by the Hindi profile.
+ * Profile provides: commands, references, possessives, roleMarkers
+ * Extras provide: values, positional, events, modifiers
+ */
+const HINDI_EXTRAS: KeywordEntry[] = [
+  // Values
+  { native: 'सच', normalized: 'true' },
+  { native: 'सत्य', normalized: 'true' },
+  { native: 'झूठ', normalized: 'false' },
+  { native: 'असत्य', normalized: 'false' },
+  { native: 'खाली', normalized: 'null' },
+  { native: 'अपरिभाषित', normalized: 'undefined' },
+
+  // Positional
+  { native: 'पहला', normalized: 'first' },
+  { native: 'अंतिम', normalized: 'last' },
+  { native: 'अगला', normalized: 'next' },
+  { native: 'पिछला', normalized: 'previous' },
+  { native: 'निकटतम', normalized: 'closest' },
+  { native: 'मूल', normalized: 'parent' },
+
+  // Events
+  { native: 'क्लिक', normalized: 'click' },
+  { native: 'परिवर्तन', normalized: 'change' },
+  { native: 'जमा', normalized: 'submit' },
+  { native: 'इनपुट', normalized: 'input' },
+  { native: 'लोड', normalized: 'load' },
+  { native: 'स्क्रॉल', normalized: 'scroll' },
+
+  // Additional modifiers not in profile
+  { native: 'को', normalized: 'to' },
+  { native: 'के साथ', normalized: 'with' },
+];
+
+// =============================================================================
+// Hindi Tokenizer Class
+// =============================================================================
+
+export class HindiTokenizer extends BaseTokenizer {
+  readonly language = 'hi';
+  readonly direction = 'ltr' as const;
+
+  constructor() {
+    super();
+    this.initializeKeywordsFromProfile(hindiProfile, HINDI_EXTRAS);
+  }
+
+  tokenize(input: string): TokenStream {
+    const tokens: LanguageToken[] = [];
+    let pos = 0;
+
+    while (pos < input.length) {
+      // Skip whitespace
+      if (isWhitespace(input[pos])) {
+        pos++;
+        continue;
+      }
+
+      // Try CSS selector first (ASCII-based, highest priority)
+      if (isSelectorStart(input[pos])) {
+        // Check for event modifier first (.once, .debounce(), etc.)
+        const modifierToken = this.tryEventModifier(input, pos);
+        if (modifierToken) {
+          tokens.push(modifierToken);
+          pos = modifierToken.position.end;
+          continue;
+        }
+
+        const selectorToken = this.trySelector(input, pos);
+        if (selectorToken) {
+          tokens.push(selectorToken);
+          pos = selectorToken.position.end;
+          continue;
+        }
+      }
+
+      // Try string literal
+      if (isQuote(input[pos])) {
+        const stringToken = this.tryString(input, pos);
+        if (stringToken) {
+          tokens.push(stringToken);
+          pos = stringToken.position.end;
+          continue;
+        }
+      }
+
+      // Try URL
+      if (isUrlStart(input, pos)) {
+        const urlToken = this.tryUrl(input, pos);
+        if (urlToken) {
+          tokens.push(urlToken);
+          pos = urlToken.position.end;
+          continue;
+        }
+      }
+
+      // Try number
+      if (
+        isDigit(input[pos]) ||
+        (input[pos] === '-' && pos + 1 < input.length && isDigit(input[pos + 1]))
+      ) {
+        const numberToken = this.extractNumber(input, pos);
+        if (numberToken) {
+          tokens.push(numberToken);
+          pos = numberToken.position.end;
+          continue;
+        }
+      }
+
+      // Variable references (:name)
+      if (input[pos] === ':') {
+        const startPos = pos;
+        pos++; // Skip :
+        let varName = '';
+        while (pos < input.length && (isAsciiIdentifierChar(input[pos]) || isHindi(input[pos]))) {
+          varName += input[pos];
+          pos++;
+        }
+        if (varName) {
+          tokens.push(
+            createToken(':' + varName, 'identifier', createPosition(startPos, pos), ':' + varName)
+          );
+          continue;
+        }
+        // Lone colon - put back
+        pos = startPos;
+      }
+
+      // Devanagari words
+      if (isHindi(input[pos])) {
+        const startPos = pos;
+        let word = '';
+
+        while (pos < input.length && (isHindi(input[pos]) || input[pos] === ' ')) {
+          // Allow spaces for compound words but stop at double spaces
+          if (input[pos] === ' ') {
+            // Check if next char is Hindi (compound postposition)
+            if (pos + 1 < input.length && isHindi(input[pos + 1])) {
+              // Check if it forms a known compound
+              const rest = input.slice(pos);
+              const compound = [
+                ' के लिए',
+                ' के साथ',
+                ' के बाद',
+                ' से पहले',
+                ' नहीं तो',
+                ' जब तक',
+                ' के बारे में',
+              ].find(c => rest.startsWith(c));
+              if (compound) {
+                word += compound;
+                pos += compound.length;
+                continue;
+              }
+            }
+            break;
+          }
+          word += input[pos];
+          pos++;
+        }
+
+        // Check if it's a keyword (O(1) Map lookup)
+        const keywordEntry = this.lookupKeyword(word);
+        if (keywordEntry) {
+          tokens.push(
+            createToken(word, 'keyword', createPosition(startPos, pos), keywordEntry.normalized)
+          );
+        } else if (SINGLE_POSTPOSITIONS.has(word)) {
+          // It's a particle
+          tokens.push(createToken(word, 'particle', createPosition(startPos, pos)));
+        } else {
+          // Unknown Hindi word - treat as identifier
+          tokens.push(createToken(word, 'identifier', createPosition(startPos, pos)));
+        }
+        continue;
+      }
+
+      // ASCII identifiers (for mixed content)
+      if (isAsciiIdentifierChar(input[pos])) {
+        const startPos = pos;
+        let word = '';
+        while (pos < input.length && isAsciiIdentifierChar(input[pos])) {
+          word += input[pos];
+          pos++;
+        }
+
+        // Check if it's a known keyword (O(1) Map lookup)
+        const keywordEntry = this.lookupKeyword(word);
+        const kind: TokenKind = keywordEntry ? 'keyword' : 'identifier';
+        tokens.push(
+          createToken(
+            word,
+            kind,
+            createPosition(startPos, pos),
+            keywordEntry?.normalized || word.toLowerCase()
+          )
+        );
+        continue;
+      }
+
+      // Operators and punctuation
+      const startPos = pos;
+      tokens.push(createToken(input[pos], 'operator', createPosition(startPos, pos + 1)));
+      pos++;
+    }
+
+    return new TokenStreamImpl(tokens, this.language);
+  }
+
+  classifyToken(value: string): TokenKind {
+    // O(1) Map lookup instead of O(n) array search
+    if (this.isKeyword(value)) {
+      return 'keyword';
+    }
+    if (SINGLE_POSTPOSITIONS.has(value)) return 'particle';
+    if (value.startsWith('.') || value.startsWith('#') || value.startsWith('[')) return 'selector';
+    if (value.startsWith(':')) return 'identifier';
+    if (value.startsWith('"') || value.startsWith("'")) return 'literal';
+    if (/^-?\d/.test(value)) return 'literal';
+    return 'identifier';
+  }
+
+  /**
+   * Extract a number from the input.
+   */
+  private extractNumber(input: string, start: number): LanguageToken | null {
+    let pos = start;
+    let num = '';
+
+    // Handle negative sign
+    if (input[pos] === '-') {
+      num += '-';
+      pos++;
+    }
+
+    // Integer part
+    while (pos < input.length && isDigit(input[pos])) {
+      num += input[pos];
+      pos++;
+    }
+
+    // Decimal part
+    if (pos < input.length && input[pos] === '.') {
+      num += '.';
+      pos++;
+      while (pos < input.length && isDigit(input[pos])) {
+        num += input[pos];
+        pos++;
+      }
+    }
+
+    if (num === '-') {
+      return null;
+    }
+
+    return createToken(num, 'literal', createPosition(start, pos));
+  }
+}
+
+// =============================================================================
+// Export
+// =============================================================================
+
+export const hindiTokenizer = new HindiTokenizer();

package/src/tokenizers/index.ts

@@ -0,0 +1,127 @@
+/**
+ * Tokenizer Registry
+ *
+ * Provides a unified interface for tokenization.
+ * Delegates to the central registry for language lookups.
+ *
+ * For tree-shaking, import specific tokenizers directly:
+ *   import { englishTokenizer } from './tokenizers/english';
+ *
+ * To register languages, import the language modules:
+ *   import '@lokascript/semantic/languages/en';
+ */
+
+import type { LanguageTokenizer, TokenStream } from '../types';
+import {
+  tryGetTokenizer,
+  getRegisteredLanguages,
+  isLanguageRegistered,
+  tokenize as registryTokenize,
+  registerTokenizer as registryRegisterTokenizer,
+} from '../registry';
+
+// =============================================================================
+// Registry Delegation
+// =============================================================================
+
+/**
+ * Get a tokenizer for the specified language.
+ * Returns undefined if language is not registered.
+ */
+export function getTokenizer(language: string): LanguageTokenizer | undefined {
+  return tryGetTokenizer(language);
+}
+
+/**
+ * Tokenize input in the specified language.
+ * @throws Error if language is not registered
+ */
+export function tokenize(input: string, language: string): TokenStream {
+  return registryTokenize(input, language);
+}
+
+/**
+ * Get all supported languages.
+ * Returns only languages that have been registered.
+ */
+export function getSupportedLanguages(): string[] {
+  return getRegisteredLanguages();
+}
+
+/**
+ * Check if a language is supported.
+ */
+export function isLanguageSupported(language: string): boolean {
+  return isLanguageRegistered(language);
+}
+
+/**
+ * Register a custom tokenizer.
+ * Note: For full language support, use registerLanguage() from registry instead.
+ */
+export function registerTokenizer(tokenizer: LanguageTokenizer): void {
+  registryRegisterTokenizer(tokenizer);
+}
+
+// =============================================================================
+// Re-exports (tree-shakeable - only included if imported)
+// =============================================================================
+
+export { englishTokenizer } from './english';
+export { japaneseTokenizer } from './japanese';
+export { koreanTokenizer } from './korean';
+export { arabicTokenizer } from './arabic';
+export { spanishTokenizer } from './spanish';
+export { turkishTokenizer } from './turkish';
+export { chineseTokenizer } from './chinese';
+export { portugueseTokenizer } from './portuguese';
+export { frenchTokenizer } from './french';
+export { germanTokenizer } from './german';
+export { indonesianTokenizer } from './indonesian';
+export { quechuaTokenizer } from './quechua';
+export { swahiliTokenizer } from './swahili';
+export { italianTokenizer } from './italian';
+export { vietnameseTokenizer } from './vietnamese';
+export { polishTokenizer } from './polish';
+export { russianTokenizer } from './russian';
+export { ukrainianTokenizer } from './ukrainian';
+export { hindiTokenizer } from './hindi';
+export { bengaliTokenizer } from './bengali';
+export { thaiTokenizer } from './thai';
+
+export { malayTokenizer } from './ms';
+
+export { tagalogTokenizer } from './tl';
+
+export {
+  BaseTokenizer,
+  TokenStreamImpl,
+  createToken,
+  createPosition,
+  extractCssSelector,
+  extractStringLiteral,
+  extractNumber,
+  isUrlStart,
+  extractUrl,
+  // Character classification utilities
+  isWhitespace,
+  isSelectorStart,
+  isQuote,
+  isDigit,
+  isAsciiLetter,
+  isAsciiIdentifierChar,
+  // Factory functions for custom tokenizers
+  createUnicodeRangeClassifier,
+  combineClassifiers,
+  createLatinCharClassifiers,
+} from './base';
+
+// Types for profile-derived keyword initialization and custom tokenizers
+export type {
+  KeywordEntry,
+  TokenizerProfile,
+  TimeUnitMapping,
+  UnicodeRange,
+  LatinCharClassifiers,
+  CreateTokenOptions,
+} from './base';