@lokascript/semantic 1.3.0 → 2.0.0

package/dist/tokenizers/base-tokenizer.d.ts

@@ -0,0 +1,240 @@
+/**
+ * Base Tokenizer Class
+ *
+ * Abstract base class for language-specific tokenizers.
+ * Provides keyword management, morphological normalization,
+ * and high-level token extraction methods.
+ */
+import type { LanguageToken, TokenKind, TokenStream, LanguageTokenizer } from '../types';
+import type { MorphologicalNormalizer, NormalizationResult } from './morphology/types';
+import { type TimeUnitMapping } from './token-utils';
+/**
+ * Keyword entry for tokenizer - maps native word to normalized English form.
+ */
+export interface KeywordEntry {
+    readonly native: string;
+    readonly normalized: string;
+}
+/**
+ * Profile interface for keyword derivation.
+ * Matches the structure of LanguageProfile but only includes fields needed for tokenization.
+ */
+export interface TokenizerProfile {
+    readonly keywords?: Record<string, {
+        primary: string;
+        alternatives?: string[];
+        normalized?: string;
+    }>;
+    readonly references?: Record<string, string>;
+    readonly roleMarkers?: Record<string, {
+        primary: string;
+        alternatives?: string[];
+        position?: string;
+    }>;
+}
+/**
+ * Abstract base class for language-specific tokenizers.
+ * Provides common functionality for CSS selectors, strings, and numbers.
+ */
+export declare abstract class BaseTokenizer implements LanguageTokenizer {
+    abstract readonly language: string;
+    abstract readonly direction: 'ltr' | 'rtl';
+    /** Optional morphological normalizer for this language */
+    protected normalizer?: MorphologicalNormalizer;
+    /** Keywords derived from profile, sorted longest-first for greedy matching */
+    protected profileKeywords: KeywordEntry[];
+    /** Map for O(1) keyword lookups by lowercase native word */
+    protected profileKeywordMap: Map<string, KeywordEntry>;
+    abstract tokenize(input: string): TokenStream;
+    abstract classifyToken(token: string): TokenKind;
+    /**
+     * Check if current position is a property access (obj.prop) vs CSS selector (.active).
+     * Property access: no whitespace before '.', previous token is identifier/keyword/selector.
+     * Also detects standalone method calls: .identifier( pattern.
+     *
+     * Returns true if '.' was emitted as an operator token and pos should advance by 1.
+     * Returns false if this is a CSS selector and should be handled by trySelector().
+     */
+    protected tryPropertyAccess(input: string, pos: number, tokens: LanguageToken[]): boolean;
+    /**
+     * Initialize keyword mappings from a language profile.
+     * Builds a list of native→english mappings from:
+     * - profile.keywords (primary + alternatives)
+     * - profile.references (me, it, you, etc.)
+     * - profile.roleMarkers (into, from, with, etc.)
+     *
+     * Results are sorted longest-first for greedy matching (important for non-space languages).
+     * Extras take precedence over profile entries when there are duplicates.
+     *
+     * @param profile - Language profile containing keyword translations
+     * @param extras - Additional keyword entries to include (literals, positional, events)
+     */
+    protected initializeKeywordsFromProfile(profile: TokenizerProfile, extras?: KeywordEntry[]): void;
+    /**
+     * Remove diacritical marks from a word for normalization.
+     * Primarily for Arabic (shadda, fatha, kasra, damma, sukun, etc.)
+     * but could be extended for other languages.
+     *
+     * @param word - Word to normalize
+     * @returns Word without diacritics
+     */
+    protected removeDiacritics(word: string): string;
+    /**
+     * Try to match a keyword from profile at the current position.
+     * Uses longest-first greedy matching (important for non-space languages).
+     *
+     * @param input - Input string
+     * @param pos - Current position
+     * @returns Token if matched, null otherwise
+     */
+    protected tryProfileKeyword(input: string, pos: number): LanguageToken | null;
+    /**
+     * Check if the remaining input starts with any known keyword.
+     * Useful for non-space languages to detect word boundaries.
+     *
+     * @param input - Input string
+     * @param pos - Current position
+     * @returns true if a keyword starts at this position
+     */
+    protected isKeywordStart(input: string, pos: number): boolean;
+    /**
+     * Look up a keyword by native word (case-insensitive).
+     * O(1) lookup using the keyword map.
+     *
+     * @param native - Native word to look up
+     * @returns KeywordEntry if found, undefined otherwise
+     */
+    protected lookupKeyword(native: string): KeywordEntry | undefined;
+    /**
+     * Check if a word is a known keyword (case-insensitive).
+     * O(1) lookup using the keyword map.
+     *
+     * @param native - Native word to check
+     * @returns true if the word is a keyword
+     */
+    protected isKeyword(native: string): boolean;
+    /**
+     * Set the morphological normalizer for this tokenizer.
+     */
+    setNormalizer(normalizer: MorphologicalNormalizer): void;
+    /**
+     * Try to normalize a word using the morphological normalizer.
+     * Returns null if no normalizer is set or normalization fails.
+     *
+     * Note: We don't check isNormalizable() here because the individual tokenizers
+     * historically called normalize() directly without that check. The normalize()
+     * method itself handles returning noChange() for words that can't be normalized.
+     */
+    protected tryNormalize(word: string): NormalizationResult | null;
+    /**
+     * Try morphological normalization and keyword lookup.
+     *
+     * If the word can be normalized to a stem that matches a known keyword,
+     * returns a keyword token with morphological metadata (stem, stemConfidence).
+     *
+     * This is the common pattern for handling conjugated verbs across languages:
+     * 1. Normalize the word (e.g., "toggled" → "toggle")
+     * 2. Look up the stem in the keyword map
+     * 3. Create a token with both the original form and stem metadata
+     *
+     * @param word - The word to normalize and look up
+     * @param startPos - Start position for the token
+     * @param endPos - End position for the token
+     * @returns Token if stem matches a keyword, null otherwise
+     */
+    protected tryMorphKeywordMatch(word: string, startPos: number, endPos: number): LanguageToken | null;
+    /**
+     * Try to extract a CSS selector at the current position.
+     */
+    protected trySelector(input: string, pos: number): LanguageToken | null;
+    /**
+     * Try to extract an event modifier at the current position.
+     * Event modifiers are .once, .debounce(N), .throttle(N), .queue(strategy)
+     */
+    protected tryEventModifier(input: string, pos: number): LanguageToken | null;
+    /**
+     * Try to extract a string literal at the current position.
+     */
+    protected tryString(input: string, pos: number): LanguageToken | null;
+    /**
+     * Try to extract a number at the current position.
+     */
+    protected tryNumber(input: string, pos: number): LanguageToken | null;
+    /**
+     * Configuration for native language time units.
+     * Maps patterns to their standard suffix (ms, s, m, h).
+     */
+    protected static readonly STANDARD_TIME_UNITS: readonly TimeUnitMapping[];
+    /**
+     * Try to match a time unit from a list of patterns.
+     *
+     * @param input - Input string
+     * @param pos - Position after the number
+     * @param timeUnits - Array of time unit mappings (native pattern → standard suffix)
+     * @param skipWhitespace - Whether to skip whitespace before time unit (default: false)
+     * @returns Object with matched suffix and new position, or null if no match
+     */
+    protected tryMatchTimeUnit(input: string, pos: number, timeUnits: readonly TimeUnitMapping[], skipWhitespace?: boolean): {
+        suffix: string;
+        endPos: number;
+    } | null;
+    /**
+     * Parse a base number (sign, integer, decimal) without time units.
+     * Returns the number string and end position.
+     *
+     * @param input - Input string
+     * @param startPos - Start position
+     * @param allowSign - Whether to allow +/- sign (default: true)
+     * @returns Object with number string and end position, or null
+     */
+    protected parseBaseNumber(input: string, startPos: number, allowSign?: boolean): {
+        number: string;
+        endPos: number;
+    } | null;
+    /**
+     * Try to extract a number with native language time units.
+     *
+     * This is a template method that handles the common pattern:
+     * 1. Parse the base number (sign, integer, decimal)
+     * 2. Try to match native language time units
+     * 3. Fall back to standard time units (ms, s, m, h)
+     *
+     * @param input - Input string
+     * @param pos - Start position
+     * @param nativeTimeUnits - Language-specific time unit mappings
+     * @param options - Configuration options
+     * @returns Token if number found, null otherwise
+     */
+    protected tryNumberWithTimeUnits(input: string, pos: number, nativeTimeUnits: readonly TimeUnitMapping[], options?: {
+        allowSign?: boolean;
+        skipWhitespace?: boolean;
+    }): LanguageToken | null;
+    /**
+     * Try to extract a URL at the current position.
+     * Handles /path, ./path, ../path, //domain.com, http://, https://
+     */
+    protected tryUrl(input: string, pos: number): LanguageToken | null;
+    /**
+     * Try to extract a variable reference (:varname) at the current position.
+     * In hyperscript, :x refers to a local variable named x.
+     */
+    protected tryVariableRef(input: string, pos: number): LanguageToken | null;
+    /**
+     * Try to extract an operator or punctuation token at the current position.
+     * Handles two-character operators (==, !=, etc.) and single-character operators.
+     */
+    protected tryOperator(input: string, pos: number): LanguageToken | null;
+    /**
+     * Try to match a multi-character particle from a list.
+     *
+     * Used by languages like Japanese, Korean, and Chinese that have
+     * multi-character particles (e.g., Japanese から, まで, より).
+     *
+     * @param input - Input string
+     * @param pos - Current position
+     * @param particles - Array of multi-character particles to match
+     * @returns Token if matched, null otherwise
+     */
+    protected tryMultiCharParticle(input: string, pos: number, particles: readonly string[]): LanguageToken | null;
+}
+//# sourceMappingURL=base-tokenizer.d.ts.map

package/dist/tokenizers/base-tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"base-tokenizer.d.ts","sourceRoot":"","sources":["../../src/tokenizers/base-tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AACzF,OAAO,KAAK,EAAE,uBAAuB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACvF,OAAO,EAML,KAAK,eAAe,EAErB,MAAM,eAAe,CAAC;AAOvB;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CACxB,MAAM,EACN;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAClE,CAAC;IACF,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAC3B,MAAM,EACN;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAChE,CAAC;CACH;AAMD;;;GAGG;AACH,8BAAsB,aAAc,YAAW,iBAAiB;IAC9D,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,QAAQ,CAAC,SAAS,EAAE,KAAK,GAAG,KAAK,CAAC;IAE3C,0DAA0D;IAC1D,SAAS,CAAC,UAAU,CAAC,EAAE,uBAAuB,CAAC;IAE/C,8EAA8E;IAC9E,SAAS,CAAC,eAAe,EAAE,YAAY,EAAE,CAAM;IAE/C,4DAA4D;IAC5D,SAAS,CAAC,iBAAiB,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,CAAa;IAEnE,QAAQ,CAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IAC7C,QAAQ,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS;IAEhD;;;;;;;OAOG;IACH,SAAS,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO;IAgCzF;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,6BAA6B,CACrC,OAAO,EAAE,gBAAgB,EACzB,MAAM,GAAE,YAAY,EAAO,GAC1B,IAAI;IAgFP;;;;;;;OAOG;IACH,SAAS,CAAC,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAMhD;;;;;;;OAOG;IACH,SAAS,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAc7E;;;;;;;OAOG;IACH,SAAS,CAAC,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO;IAK7D;;;;;;OAMG;IACH,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,GAAG,SAAS;IAIjE;;;;;;OAMG;IACH,SAAS,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO;IAI5C;;OAEG;IACH,aAAa,CAAC,UAAU,EAAE,uBAAuB,GAAG,IAAI;IAIxD;;;;;;;OAOG;IACH,SAAS,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,GAAG,IAAI;IAahE;;;;;;;;;;;;;;;OAeG;IACH,SAAS,CAAC,oBAAoB,CAC5B,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,GACb,aAAa,GAAG,IAAI;IAgBvB;;OAEG;IACH,SAAS,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAQvE;;;OAGG;IACH,SAAS,CAAC,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAmC5E;;OAEG;IACH,SAAS,CAAC,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAQrE;;OAEG;IACH,SAAS,CAAC,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAQrE;;;OAGG;IACH,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,EAAE,SAAS,eAAe,EAAE,CAKvE;IAEF;;;;;;;;OAQG;IACH,SAAS,CAAC,gBAAgB,CACxB,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,SAAS,eAAe,EAAE,EACrC,cAAc,UAAQ,GACrB;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAuC5C;;;;;;;;OAQG;IACH,SAAS,CAAC,eAAe,CACvB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,SAAS,UAAO,GACf;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAgC5C;;;;;;;;;;;;;OAaG;IACH,SAAS,CAAC,sBAAsB,CAC9B,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,eAAe,EAAE,SAAS,eAAe,EAAE,EAC3C,OAAO,GAAE;QAAE,SAAS,CAAC,EAAE,OAAO,CAAC;QAAC,cAAc,CAAC,EAAE,OAAO,CAAA;KAAO,GAC9D,aAAa,GAAG,IAAI;IAqBvB;;;OAGG;IACH,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAQlE;;;OAGG;IACH,SAAS,CAAC,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAc1E;;;OAGG;IACH,SAAS,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAqBvE;;;;;;;;;;OAUG;IACH,SAAS,CAAC,oBAAoB,CAC5B,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,SAAS,MAAM,EAAE,GAC3B,aAAa,GAAG,IAAI;CAQxB"}

package/dist/tokenizers/base.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Base Tokenizer
+ *
+ * Re-exports all tokenizer utilities and the BaseTokenizer class.
+ * Language-specific tokenizers import from this file.
+ */
+export { TokenStreamImpl, createToken, createPosition, isWhitespace, isSelectorStart, isQuote, isDigit, isAsciiLetter, isAsciiIdentifierChar, } from './token-utils';
+export type { TimeUnitMapping, CreateTokenOptions } from './token-utils';
+export { createUnicodeRangeClassifier, combineClassifiers, createLatinCharClassifiers, } from './char-classifiers';
+export type { UnicodeRange, LatinCharClassifiers } from './char-classifiers';
+export { extractCssSelector, isPossessiveMarker, extractStringLiteral, isUrlStart, extractUrl, extractNumber, } from './extractors';
+export { BaseTokenizer } from './base-tokenizer';
+export type { KeywordEntry, TokenizerProfile } from './base-tokenizer';
+//# sourceMappingURL=base.d.ts.map

package/dist/tokenizers/base.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"base.d.ts","sourceRoot":"","sources":["../../src/tokenizers/base.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EACL,eAAe,EACf,WAAW,EACX,cAAc,EACd,YAAY,EACZ,eAAe,EACf,OAAO,EACP,OAAO,EACP,aAAa,EACb,qBAAqB,GACtB,MAAM,eAAe,CAAC;AACvB,YAAY,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAGzE,OAAO,EACL,4BAA4B,EAC5B,kBAAkB,EAClB,0BAA0B,GAC3B,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EAAE,YAAY,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAG7E,OAAO,EACL,kBAAkB,EAClB,kBAAkB,EAClB,oBAAoB,EACpB,UAAU,EACV,UAAU,EACV,aAAa,GACd,MAAM,cAAc,CAAC;AAGtB,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,YAAY,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC"}

package/dist/tokenizers/bengali.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Bengali Tokenizer
+ *
+ * Tokenizes Bengali hyperscript input.
+ * Bengali is an agglutinative SOV language with:
+ * - Bengali script (U+0980-U+09FF)
+ * - Postposition markers (কে, তে, থেকে, etc.)
+ * - Similar grammatical structure to Hindi
+ * - CSS selectors are embedded ASCII
+ */
+import type { TokenKind, TokenStream } from '../types';
+import { BaseTokenizer } from './base';
+export declare class BengaliTokenizer extends BaseTokenizer {
+    readonly language = "bn";
+    readonly direction: "ltr";
+    constructor();
+    tokenize(input: string): TokenStream;
+    classifyToken(value: string): TokenKind;
+    private extractNumber;
+}
+export declare const bengaliTokenizer: BengaliTokenizer;
+//# sourceMappingURL=bengali.d.ts.map

package/dist/tokenizers/bengali.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bengali.d.ts","sourceRoot":"","sources":["../../src/tokenizers/bengali.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAiB,SAAS,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACtE,OAAO,EACL,aAAa,EAYd,MAAM,QAAQ,CAAC;AAsEhB,qBAAa,gBAAiB,SAAQ,aAAa;IACjD,QAAQ,CAAC,QAAQ,QAAQ;IACzB,QAAQ,CAAC,SAAS,EAAG,KAAK,CAAU;;IAOpC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IAkJpC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS;IAoBvC,OAAO,CAAC,aAAa;CA6BtB;AAED,eAAO,MAAM,gBAAgB,kBAAyB,CAAC"}

package/dist/tokenizers/char-classifiers.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Character Classifiers
+ *
+ * Unicode range classification and Latin character classifier factories.
+ * Used by language-specific tokenizers to define character sets.
+ */
+/**
+ * Unicode range tuple: [start, end] (inclusive).
+ */
+export type UnicodeRange = readonly [number, number];
+/**
+ * Create a character classifier for Unicode ranges.
+ * Returns a function that checks if a character's code point falls within any of the ranges.
+ *
+ * @example
+ * // Japanese Hiragana
+ * const isHiragana = createUnicodeRangeClassifier([[0x3040, 0x309f]]);
+ *
+ * // Korean (Hangul syllables + Jamo)
+ * const isKorean = createUnicodeRangeClassifier([
+ *   [0xac00, 0xd7a3],  // Hangul syllables
+ *   [0x1100, 0x11ff],  // Hangul Jamo
+ *   [0x3130, 0x318f],  // Hangul Compatibility Jamo
+ * ]);
+ */
+export declare function createUnicodeRangeClassifier(ranges: readonly UnicodeRange[]): (char: string) => boolean;
+/**
+ * Combine multiple character classifiers into one.
+ * Returns true if any of the classifiers return true.
+ *
+ * @example
+ * const isJapanese = combineClassifiers(isHiragana, isKatakana, isKanji);
+ */
+export declare function combineClassifiers(...classifiers: Array<(char: string) => boolean>): (char: string) => boolean;
+/**
+ * Character classifiers for a Latin-based language.
+ */
+export interface LatinCharClassifiers {
+    /** Check if character is a letter in this language (including accented chars). */
+    isLetter: (char: string) => boolean;
+    /** Check if character is part of an identifier (letter, digit, underscore, hyphen). */
+    isIdentifierChar: (char: string) => boolean;
+}
+/**
+ * Create character classifiers for a Latin-based language.
+ * Returns isLetter and isIdentifierChar functions based on the provided regex.
+ *
+ * @example
+ * // Spanish letters
+ * const { isLetter, isIdentifierChar } = createLatinCharClassifiers(/[a-zA-ZáéíóúüñÁÉÍÓÚÜÑ]/);
+ *
+ * // German letters
+ * const { isLetter, isIdentifierChar } = createLatinCharClassifiers(/[a-zA-ZäöüÄÖÜß]/);
+ */
+export declare function createLatinCharClassifiers(letterPattern: RegExp): LatinCharClassifiers;
+//# sourceMappingURL=char-classifiers.d.ts.map

package/dist/tokenizers/char-classifiers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"char-classifiers.d.ts","sourceRoot":"","sources":["../../src/tokenizers/char-classifiers.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAMH;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAErD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,SAAS,YAAY,EAAE,GAC9B,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAK3B;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAChC,GAAG,WAAW,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,GAC/C,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAE3B;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,kFAAkF;IAClF,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC;IACpC,uFAAuF;IACvF,gBAAgB,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC;CAC7C;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,0BAA0B,CAAC,aAAa,EAAE,MAAM,GAAG,oBAAoB,CAItF"}

package/dist/tokenizers/chinese.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Chinese Tokenizer
+ *
+ * Tokenizes Chinese hyperscript input.
+ * Chinese is challenging because:
+ * - No spaces between words (like Japanese)
+ * - Uses CJK characters (shared with Japanese Kanji)
+ * - SVO word order (like English)
+ * - Uses prepositions (把, 在, 从, etc.) for grammatical roles
+ * - No conjugation (unlike Japanese/Korean)
+ * - CSS selectors are embedded ASCII
+ */
+import type { TokenKind, TokenStream } from '../types';
+import { BaseTokenizer } from './base';
+export declare class ChineseTokenizer extends BaseTokenizer {
+    readonly language = "zh";
+    readonly direction: "ltr";
+    constructor();
+    tokenize(input: string): TokenStream;
+    classifyToken(token: string): TokenKind;
+    /**
+     * Extract a Chinese word.
+     * Uses greedy matching to find the longest known keyword.
+     * Chinese doesn't have inflection, so we don't need morphological normalization.
+     * profileKeywords is already sorted longest-first, enabling greedy matching.
+     */
+    private extractChineseWord;
+    /**
+     * Extract an ASCII word (for mixed Chinese/English content).
+     */
+    private extractAsciiWord;
+    /**
+     * Try to extract a string literal, including Chinese quotes.
+     * Chinese quotes: \u201C " (open) \u201D " (close) \u2018 ' (open) \u2019 ' (close)
+     */
+    private tryChineseString;
+    /**
+     * Extract a number, including Chinese time unit suffixes.
+     * Chinese time units attach directly without whitespace.
+     */
+    private extractChineseNumber;
+}
+/**
+ * Singleton instance.
+ */
+export declare const chineseTokenizer: ChineseTokenizer;
+//# sourceMappingURL=chinese.d.ts.map

package/dist/tokenizers/chinese.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chinese.d.ts","sourceRoot":"","sources":["../../src/tokenizers/chinese.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAiB,SAAS,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACtE,OAAO,EACL,aAAa,EAad,MAAM,QAAQ,CAAC;AAgLhB,qBAAa,gBAAiB,SAAQ,aAAa;IACjD,QAAQ,CAAC,QAAQ,QAAQ;IACzB,QAAQ,CAAC,SAAS,EAAG,KAAK,CAAU;;IAOpC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IAmHpC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS;IAwBvC;;;;;OAKG;IACH,OAAO,CAAC,kBAAkB;IA2D1B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAaxB;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAmCxB;;;OAGG;IACH,OAAO,CAAC,oBAAoB;CAM7B;AAED;;GAEG;AACH,eAAO,MAAM,gBAAgB,kBAAyB,CAAC"}

package/dist/tokenizers/english.d.ts

@@ -0,0 +1,34 @@
+/**
+ * English Tokenizer
+ *
+ * Tokenizes English hyperscript input.
+ * English uses space-separated words with prepositions.
+ */
+import type { TokenKind, TokenStream } from '../types';
+import { BaseTokenizer } from './base';
+export declare class EnglishTokenizer extends BaseTokenizer {
+    readonly language = "en";
+    readonly direction: "ltr";
+    constructor();
+    tokenize(input: string): TokenStream;
+    classifyToken(token: string): TokenKind;
+    /**
+     * Extract a word (identifier or keyword) from the input.
+     * Handles namespaced event names like "draggable:start".
+     */
+    private extractWord;
+    /**
+     * Try to convert an identifier followed by "class" to a class selector.
+     * E.g., "active class" → ".active"
+     *
+     * This enables natural English syntax like:
+     * - "toggle the active class" → "toggle .active"
+     * - "add the visible class" → "add .visible"
+     */
+    private tryConvertToClassSelector;
+}
+/**
+ * Singleton instance.
+ */
+export declare const englishTokenizer: EnglishTokenizer;
+//# sourceMappingURL=english.d.ts.map

package/dist/tokenizers/english.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"english.d.ts","sourceRoot":"","sources":["../../src/tokenizers/english.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAiB,SAAS,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACtE,OAAO,EACL,aAAa,EAYd,MAAM,QAAQ,CAAC;AAmIhB,qBAAa,gBAAiB,SAAQ,aAAa;IACjD,QAAQ,CAAC,QAAQ,QAAQ;IACzB,QAAQ,CAAC,SAAS,EAAG,KAAK,CAAU;;IAQpC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IA4IpC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS;IAmBvC;;;OAGG;IACH,OAAO,CAAC,WAAW;IAsDnB;;;;;;;OAOG;IACH,OAAO,CAAC,yBAAyB;CA8BlC;AAED;;GAEG;AACH,eAAO,MAAM,gBAAgB,kBAAyB,CAAC"}

package/dist/tokenizers/extractors.d.ts

@@ -0,0 +1,61 @@
+/**
+ * Extraction Utilities
+ *
+ * Pure functions for extracting CSS selectors, string literals, URLs, and numbers
+ * from input strings. These are language-independent and used by all tokenizers.
+ */
+/**
+ * Extract a CSS selector from the input string starting at pos.
+ * CSS selectors are universal across languages.
+ *
+ * Supported formats:
+ * - #id
+ * - .class
+ * - [attribute]
+ * - [attribute=value]
+ * - @attribute (shorthand)
+ * - *property (CSS property shorthand)
+ * - Complex selectors with combinators (limited)
+ *
+ * Method call handling:
+ * - #dialog.showModal() → stops after #dialog (method call, not compound selector)
+ * - #box.active → compound selector (no parens)
+ */
+export declare function extractCssSelector(input: string, startPos: number): string | null;
+/**
+ * Check if a single quote at pos is a possessive marker ('s).
+ * Returns true if this looks like possessive, not a string start.
+ *
+ * Examples:
+ * - #element's *opacity → possessive (returns true)
+ * - 'hello' → string (returns false)
+ * - it's value → possessive (returns true)
+ */
+export declare function isPossessiveMarker(input: string, pos: number): boolean;
+/**
+ * Extract a string literal from the input starting at pos.
+ * Handles both ASCII quotes and Unicode quotes.
+ *
+ * Note: Single quotes that look like possessive markers ('s) are skipped.
+ */
+export declare function extractStringLiteral(input: string, startPos: number): string | null;
+/**
+ * Check if the input at position starts a URL.
+ * Detects: /path, ./path, ../path, //domain.com, http://, https://
+ */
+export declare function isUrlStart(input: string, pos: number): boolean;
+/**
+ * Extract a URL from the input starting at pos.
+ * Handles paths, query strings, and fragments.
+ *
+ * Fragment (#) handling:
+ * - /page#section → includes fragment as part of URL
+ * - #id alone → not a URL (CSS selector)
+ */
+export declare function extractUrl(input: string, startPos: number): string | null;
+/**
+ * Extract a number from the input starting at pos.
+ * Handles integers and decimals.
+ */
+export declare function extractNumber(input: string, startPos: number): string | null;
+//# sourceMappingURL=extractors.d.ts.map

package/dist/tokenizers/extractors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extractors.d.ts","sourceRoot":"","sources":["../../src/tokenizers/extractors.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAeH;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CA+LjF;AAMD;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAYtE;AAED;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CA2CnF;AAMD;;;GAGG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CA8B9D;AAED;;;;;;;GAOG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CA0CzE;AAMD;;;GAGG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CA2C5E"}

package/dist/tokenizers/french.d.ts

@@ -0,0 +1,26 @@
+/**
+ * French Tokenizer
+ *
+ * Tokenizes French hyperscript input.
+ * French characteristics:
+ * - SVO word order
+ * - Space-separated words
+ * - Prepositions
+ * - Accent marks (é, è, ê, ë, à, â, ù, û, ô, î, ï, ç, œ, æ)
+ */
+import type { TokenKind, TokenStream } from '../types';
+import { BaseTokenizer } from './base';
+export declare class FrenchTokenizer extends BaseTokenizer {
+    readonly language = "fr";
+    readonly direction: "ltr";
+    constructor();
+    tokenize(input: string): TokenStream;
+    classifyToken(token: string): TokenKind;
+    private extractWord;
+    /**
+     * Extract a number, including French time unit suffixes.
+     */
+    private extractNumber;
+}
+export declare const frenchTokenizer: FrenchTokenizer;
+//# sourceMappingURL=french.d.ts.map

package/dist/tokenizers/french.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"french.d.ts","sourceRoot":"","sources":["../../src/tokenizers/french.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAiB,SAAS,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACtE,OAAO,EACL,aAAa,EAYd,MAAM,QAAQ,CAAC;AA+JhB,qBAAa,eAAgB,SAAQ,aAAa;IAChD,QAAQ,CAAC,QAAQ,QAAQ;IACzB,QAAQ,CAAC,SAAS,EAAG,KAAK,CAAU;;IAQpC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IA4FpC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS;IAkBvC,OAAO,CAAC,WAAW;IAyBnB;;OAEG;IACH,OAAO,CAAC,aAAa;CAMtB;AAED,eAAO,MAAM,eAAe,iBAAwB,CAAC"}

package/dist/tokenizers/german.d.ts

@@ -0,0 +1,27 @@
+/**
+ * German Tokenizer
+ *
+ * Tokenizes German hyperscript input.
+ * German characteristics:
+ * - SVO word order (V2 in main clauses, but SVO for our purposes)
+ * - Space-separated words
+ * - Prepositions
+ * - Umlauts (ä, ö, ü) and ß
+ * - Compound nouns
+ */
+import type { TokenKind, TokenStream } from '../types';
+import { BaseTokenizer } from './base';
+export declare class GermanTokenizer extends BaseTokenizer {
+    readonly language = "de";
+    readonly direction: "ltr";
+    constructor();
+    tokenize(input: string): TokenStream;
+    classifyToken(token: string): TokenKind;
+    private extractWord;
+    /**
+     * Extract a number, including German time unit suffixes.
+     */
+    private extractNumber;
+}
+export declare const germanTokenizer: GermanTokenizer;
+//# sourceMappingURL=german.d.ts.map

package/dist/tokenizers/german.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"german.d.ts","sourceRoot":"","sources":["../../src/tokenizers/german.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAiB,SAAS,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACtE,OAAO,EACL,aAAa,EAYd,MAAM,QAAQ,CAAC;AA4JhB,qBAAa,eAAgB,SAAQ,aAAa;IAChD,QAAQ,CAAC,QAAQ,QAAQ;IACzB,QAAQ,CAAC,SAAS,EAAG,KAAK,CAAU;;IAQpC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IA4FpC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS;IAkBvC,OAAO,CAAC,WAAW;IAyBnB;;OAEG;IACH,OAAO,CAAC,aAAa;CAMtB;AAED,eAAO,MAAM,eAAe,iBAAwB,CAAC"}

package/dist/tokenizers/he.d.ts

@@ -0,0 +1,58 @@
+/**
+ * Hebrew Tokenizer
+ *
+ * Tokenizes Hebrew hyperscript input.
+ * Hebrew is challenging because:
+ * - Right-to-left (RTL) text direction
+ * - Prefix prepositions that attach to words (ב, ל, מ, כ, ה, ו, ש)
+ * - Optional vowel points (nikkud) typically omitted in modern text
+ * - CSS selectors are LTR islands within RTL text
+ */
+import type { TokenKind, TokenStream } from '../types';
+import { BaseTokenizer } from './base';
+export declare class HebrewTokenizer extends BaseTokenizer {
+    readonly language = "he";
+    readonly direction: "rtl";
+    constructor();
+    tokenize(input: string): TokenStream;
+    classifyToken(token: string): TokenKind;
+    /**
+     * Try to match a Hebrew preposition.
+     */
+    private tryPreposition;
+    /**
+     * Try to extract a prefix conjunction (ו - "and") that's attached to the following word.
+     *
+     * Hebrew conjunction prefix attaches directly to words without space:
+     * - והחלף → ו + החלף (and + toggle)
+     * - ולחיצה → ו + לחיצה (and + click)
+     */
+    private tryPrefixConjunction;
+    /**
+     * Try to extract an event marker prefix (ב, כ) attached to an event name.
+     *
+     * Hebrew event markers attach directly to event names without space:
+     * - בלחיצה → ב + לחיצה (on + click)
+     * - כשינוי → כ + שינוי (when + change)
+     *
+     * Returns both the marker token and the event name token if successful.
+     */
+    private tryEventMarkerPrefix;
+    /**
+     * Extract a Hebrew word.
+     */
+    private extractHebrewWord;
+    /**
+     * Extract an ASCII word.
+     */
+    private extractAsciiWord;
+    /**
+     * Extract a number, including Hebrew time unit suffixes.
+     */
+    private extractHebrewNumber;
+}
+/**
+ * Singleton instance.
+ */
+export declare const hebrewTokenizer: HebrewTokenizer;
+//# sourceMappingURL=he.d.ts.map

package/dist/tokenizers/he.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"he.d.ts","sourceRoot":"","sources":["../../src/tokenizers/he.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAiB,SAAS,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACtE,OAAO,EACL,aAAa,EAad,MAAM,QAAQ,CAAC;AAkKhB,qBAAa,eAAgB,SAAQ,aAAa;IAChD,QAAQ,CAAC,QAAQ,QAAQ;IACzB,QAAQ,CAAC,SAAS,EAAG,KAAK,CAAU;;IAOpC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IA8HpC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS;IAiBvC;;OAEG;IACH,OAAO,CAAC,cAAc;IAsBtB;;;;;;OAMG;IACH,OAAO,CAAC,oBAAoB;IAuD5B;;;;;;;;OAQG;IACH,OAAO,CAAC,oBAAoB;IAiD5B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAoCzB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAaxB;;OAEG;IACH,OAAO,CAAC,mBAAmB;CAM5B;AAED;;GAEG;AACH,eAAO,MAAM,eAAe,iBAAwB,CAAC"}

package/dist/tokenizers/hindi.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Hindi Tokenizer
+ *
+ * Tokenizes Hindi hyperscript input.
+ * Hindi is a fusional SOV language with:
+ * - Devanagari script (U+0900-U+097F)
+ * - Postposition markers (को, में, पर, से, etc.)
+ * - Verb conjugations with stem + suffix patterns
+ * - CSS selectors are embedded ASCII
+ */
+import type { TokenKind, TokenStream } from '../types';
+import { BaseTokenizer } from './base';
+export declare class HindiTokenizer extends BaseTokenizer {
+    readonly language = "hi";
+    readonly direction: "ltr";
+    constructor();
+    tokenize(input: string): TokenStream;
+    classifyToken(value: string): TokenKind;
+    /**
+     * Extract a number from the input.
+     */
+    private extractNumber;
+}
+export declare const hindiTokenizer: HindiTokenizer;
+//# sourceMappingURL=hindi.d.ts.map