@lokascript/framework 2.0.0

package/dist/core/tokenization/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/core/tokenization/token-utils.ts","../../../src/core/tokenization/extractors.ts","../../../src/core/tokenization/extractors/operator.ts","../../../src/core/tokenization/extractors/punctuation.ts","../../../src/interfaces/value-extractor.ts","../../../src/core/tokenization/default-extractors.ts","../../../src/core/tokenization/char-classifiers.ts","../../../src/core/tokenization/base-tokenizer.ts","../../../src/core/tokenization/morphology/types.ts"],"sourcesContent":["/**\n * Token Utilities\n *\n * Core token creation, stream implementation, and character classification.\n * These are the foundational building blocks used by all tokenizers.\n */\n\nimport type { LanguageToken, TokenKind, TokenStream, StreamMark, SourcePosition } from '../types';\n\n// =============================================================================\n// Time Unit Configuration\n// =============================================================================\n\n/**\n * Configuration for a native language time unit pattern.\n * Used by tryNumberWithTimeUnits() to match language-specific time units.\n */\nexport interface TimeUnitMapping {\n  /** The pattern to match (e.g., 'segundos', 'ミリ秒') */\n  readonly pattern: string;\n  /** The standard suffix to use (ms, s, m, h) */\n  readonly suffix: string;\n  /** Length of the pattern (for optimization) */\n  readonly length: number;\n  /** Whether to check for word boundary after the pattern */\n  readonly checkBoundary?: boolean;\n  /** Character that cannot follow the pattern (e.g., 's' for 'm' to avoid 'ms') */\n  readonly notFollowedBy?: string;\n  /** Whether to do case-insensitive matching */\n  readonly caseInsensitive?: boolean;\n}\n\n// =============================================================================\n// Token Stream Implementation\n// =============================================================================\n\n/**\n * Concrete implementation of TokenStream.\n */\nexport class TokenStreamImpl implements TokenStream {\n  readonly tokens: readonly LanguageToken[];\n  readonly language: string;\n  private pos: number = 0;\n\n  constructor(tokens: LanguageToken[], language: string) {\n    this.tokens = tokens;\n    this.language = language;\n  }\n\n  peek(offset: number = 0): LanguageToken | null {\n    const index = this.pos + offset;\n    if (index < 0 || index >= this.tokens.length) {\n      return null;\n    }\n    return this.tokens[index];\n  }\n\n  advance(): LanguageToken {\n    if (this.isAtEnd()) {\n      throw new Error('Unexpected end of token stream');\n    }\n    return this.tokens[this.pos++];\n  }\n\n  isAtEnd(): boolean {\n    return this.pos >= this.tokens.length;\n  }\n\n  mark(): StreamMark {\n    return { position: this.pos };\n  }\n\n  reset(mark: StreamMark): void {\n    this.pos = mark.position;\n  }\n\n  position(): number {\n    return this.pos;\n  }\n\n  /**\n   * Get remaining tokens as an array.\n   */\n  remaining(): LanguageToken[] {\n    return this.tokens.slice(this.pos);\n  }\n\n  /**\n   * Consume tokens while predicate is true.\n   */\n  takeWhile(predicate: (token: LanguageToken) => boolean): LanguageToken[] {\n    const result: LanguageToken[] = [];\n    while (!this.isAtEnd() && predicate(this.peek()!)) {\n      result.push(this.advance());\n    }\n    return result;\n  }\n\n  /**\n   * Skip tokens while predicate is true.\n   */\n  skipWhile(predicate: (token: LanguageToken) => boolean): void {\n    while (!this.isAtEnd() && predicate(this.peek()!)) {\n      this.advance();\n    }\n  }\n}\n\n// =============================================================================\n// Shared Tokenization Utilities\n// =============================================================================\n\n/**\n * Create a source position from start and end offsets.\n */\nexport function createPosition(start: number, end: number): SourcePosition {\n  return { start, end };\n}\n\n/**\n * Options for creating a token with optional morphological data.\n */\nexport interface CreateTokenOptions {\n  /** Explicitly normalized form from keyword map */\n  normalized?: string;\n  /** Morphologically normalized stem */\n  stem?: string;\n  /** Confidence in the stem (0.0-1.0) */\n  stemConfidence?: number;\n  /** Additional metadata for specific token types (e.g., event modifier data) */\n  metadata?: Record<string, unknown>;\n}\n\n/**\n * Token creation options (object style).\n */\nexport interface TokenCreationParams extends CreateTokenOptions {\n  value: string;\n  kind: TokenKind;\n  position: SourcePosition;\n}\n\n/**\n * Create a language token (object style).\n */\nexport function createToken(params: TokenCreationParams): LanguageToken;\n\n/**\n * Create a language token (separate parameters style).\n */\nexport function createToken(\n  value: string,\n  kind: TokenKind,\n  position: SourcePosition,\n  normalizedOrOptions?: string | CreateTokenOptions\n): LanguageToken;\n\n/**\n * Create a language token.\n * Supports both object style and separate parameters style.\n */\nexport function createToken(\n  valueOrParams: string | TokenCreationParams,\n  kind?: TokenKind,\n  position?: SourcePosition,\n  normalizedOrOptions?: string | CreateTokenOptions\n): LanguageToken {\n  // Handle object style\n  if (typeof valueOrParams === 'object') {\n    const { value, kind, position, normalized, stem, stemConfidence, metadata } = valueOrParams;\n    return {\n      value,\n      kind,\n      position,\n      ...(normalized !== undefined && { normalized }),\n      ...(stem !== undefined && { stem }),\n      ...(stemConfidence !== undefined && { stemConfidence }),\n      ...(metadata !== undefined && { metadata }),\n    };\n  }\n\n  // Handle separate parameters style\n  const value = valueOrParams;\n  if (!kind || !position) {\n    throw new Error('createToken requires kind and position parameters');\n  }\n  // Handle legacy string argument for backward compatibility\n  if (typeof normalizedOrOptions === 'string') {\n    return { value, kind, position, normalized: normalizedOrOptions };\n  }\n\n  // Handle options object\n  if (normalizedOrOptions) {\n    const { normalized, stem, stemConfidence, metadata } = normalizedOrOptions;\n    return {\n      value,\n      kind,\n      position,\n      ...(normalized !== undefined && { normalized }),\n      ...(stem !== undefined && { stem }),\n      ...(stemConfidence !== undefined && { stemConfidence }),\n      ...(metadata !== undefined && { metadata }),\n    };\n  }\n\n  return { value, kind, position };\n}\n\n/**\n * Check if a character is whitespace.\n */\nexport function isWhitespace(char: string): boolean {\n  return /\\s/.test(char);\n}\n\n/**\n * Check if a string starts with a CSS selector prefix.\n * Includes JSX-style element selectors: <form />, <div>\n */\nexport function isSelectorStart(char: string): boolean {\n  return (\n    char === '#' || char === '.' || char === '[' || char === '@' || char === '*' || char === '<'\n  );\n}\n\n/**\n * Check if a character is a quote (string delimiter).\n */\nexport function isQuote(char: string): boolean {\n  return char === '\"' || char === \"'\" || char === '`' || char === '「' || char === '」';\n}\n\n/**\n * Check if a character is a digit.\n */\nexport function isDigit(char: string): boolean {\n  return /\\d/.test(char);\n}\n\n/**\n * Check if a character is an ASCII letter.\n */\nexport function isAsciiLetter(char: string): boolean {\n  return /[a-zA-Z]/.test(char);\n}\n\n/**\n * Check if a character is part of an ASCII identifier.\n */\nexport function isAsciiIdentifierChar(char: string): boolean {\n  return /[a-zA-Z0-9_-]/.test(char);\n}\n","/**\n * Extraction Utilities\n *\n * Pure functions for extracting CSS selectors, string literals, URLs, and numbers\n * from input strings. These are language-independent and used by all tokenizers.\n */\n\nimport {\n  isSelectorStart,\n  isWhitespace,\n  isAsciiIdentifierChar,\n  isAsciiLetter,\n  isQuote,\n  isDigit,\n} from './token-utils';\n\n// =============================================================================\n// CSS Selector Tokenization\n// =============================================================================\n\n/**\n * Extract a CSS selector from the input string starting at pos.\n * CSS selectors are universal across languages.\n *\n * Supported formats:\n * - #id\n * - .class\n * - [attribute]\n * - [attribute=value]\n * - @attribute (shorthand)\n * - *property (CSS property shorthand)\n * - Complex selectors with combinators (limited)\n *\n * Method call handling:\n * - #dialog.showModal() → stops after #dialog (method call, not compound selector)\n * - #box.active → compound selector (no parens)\n */\nexport function extractCssSelector(input: string, startPos: number): string | null {\n  if (startPos >= input.length) return null;\n\n  const char = input[startPos];\n  if (!isSelectorStart(char)) return null;\n\n  let pos = startPos;\n  let selector = '';\n\n  // Handle different selector types\n  if (char === '#' || char === '.') {\n    // ID or class selector: #id, .class\n    selector += input[pos++];\n    while (pos < input.length && isAsciiIdentifierChar(input[pos])) {\n      selector += input[pos++];\n    }\n    // Must have at least one character after prefix\n    if (selector.length <= 1) return null;\n\n    // Check for method call pattern: #id.method() or .class.method()\n    // If we see .identifier followed by (, don't consume it - it's a method call\n    if (pos < input.length && input[pos] === '.' && char === '#') {\n      // Look ahead to see if this is a method call\n      const methodStart = pos + 1;\n      let methodEnd = methodStart;\n      while (methodEnd < input.length && isAsciiIdentifierChar(input[methodEnd])) {\n        methodEnd++;\n      }\n      // If followed by (, it's a method call - stop here\n      if (methodEnd < input.length && input[methodEnd] === '(') {\n        return selector;\n      }\n    }\n  } else if (char === '[') {\n    // Attribute selector: [attr] or [attr=value] or [attr=\"value\"]\n    // Need to track quote state to avoid counting brackets inside quotes\n    let depth = 1;\n    let inQuote = false;\n    let quoteChar: string | null = null;\n    let escaped = false;\n\n    selector += input[pos++]; // [\n\n    while (pos < input.length && depth > 0) {\n      const c = input[pos];\n      selector += c;\n\n      if (escaped) {\n        // Skip escaped character\n        escaped = false;\n      } else if (c === '\\\\') {\n        // Next character is escaped\n        escaped = true;\n      } else if (inQuote) {\n        // Inside a quoted string\n        if (c === quoteChar) {\n          inQuote = false;\n          quoteChar = null;\n        }\n      } else {\n        // Not inside a quoted string\n        if (c === '\"' || c === \"'\" || c === '`') {\n          inQuote = true;\n          quoteChar = c;\n        } else if (c === '[') {\n          depth++;\n        } else if (c === ']') {\n          depth--;\n        }\n      }\n      pos++;\n    }\n    if (depth !== 0) return null;\n  } else if (char === '@') {\n    // Attribute shorthand: @disabled\n    selector += input[pos++];\n    while (pos < input.length && isAsciiIdentifierChar(input[pos])) {\n      selector += input[pos++];\n    }\n    if (selector.length <= 1) return null;\n  } else if (char === '*') {\n    // CSS property shorthand: *display\n    selector += input[pos++];\n    while (pos < input.length && isAsciiIdentifierChar(input[pos])) {\n      selector += input[pos++];\n    }\n    if (selector.length <= 1) return null;\n  } else if (char === '<') {\n    // HTML literal selector with optional modifiers and attributes:\n    // - <div>\n    // - <div.class>\n    // - <div#id>\n    // - <div.class#id>\n    // - <button[disabled]/>\n    // - <div.card/>\n    // - <div.class#id[attr=\"value\"]/>\n    selector += input[pos++]; // <\n\n    // Must be followed by an identifier (tag name)\n    if (pos >= input.length || !isAsciiLetter(input[pos])) return null;\n\n    // Extract tag name\n    while (pos < input.length && isAsciiIdentifierChar(input[pos])) {\n      selector += input[pos++];\n    }\n\n    // Process modifiers and attributes\n    // Can have multiple .class, one #id, and multiple [attr] in any order\n    while (pos < input.length) {\n      const modChar = input[pos];\n\n      if (modChar === '.') {\n        // Class modifier\n        selector += input[pos++]; // .\n        if (pos >= input.length || !isAsciiIdentifierChar(input[pos])) {\n          return null; // Invalid - class name required after .\n        }\n        while (pos < input.length && isAsciiIdentifierChar(input[pos])) {\n          selector += input[pos++];\n        }\n      } else if (modChar === '#') {\n        // ID modifier\n        selector += input[pos++]; // #\n        if (pos >= input.length || !isAsciiIdentifierChar(input[pos])) {\n          return null; // Invalid - ID required after #\n        }\n        while (pos < input.length && isAsciiIdentifierChar(input[pos])) {\n          selector += input[pos++];\n        }\n      } else if (modChar === '[') {\n        // Attribute modifier: [disabled] or [type=\"button\"]\n        // Need to track quote state to avoid counting brackets inside quotes\n        let depth = 1;\n        let inQuote = false;\n        let quoteChar: string | null = null;\n        let escaped = false;\n\n        selector += input[pos++]; // [\n\n        while (pos < input.length && depth > 0) {\n          const c = input[pos];\n          selector += c;\n\n          if (escaped) {\n            escaped = false;\n          } else if (c === '\\\\') {\n            escaped = true;\n          } else if (inQuote) {\n            if (c === quoteChar) {\n              inQuote = false;\n              quoteChar = null;\n            }\n          } else {\n            if (c === '\"' || c === \"'\" || c === '`') {\n              inQuote = true;\n              quoteChar = c;\n            } else if (c === '[') {\n              depth++;\n            } else if (c === ']') {\n              depth--;\n            }\n          }\n          pos++;\n        }\n        if (depth !== 0) return null; // Unclosed bracket\n      } else {\n        // No more modifiers\n        break;\n      }\n    }\n\n    // Skip whitespace before optional self-closing /\n    while (pos < input.length && isWhitespace(input[pos])) {\n      selector += input[pos++];\n    }\n\n    // Optional self-closing /\n    if (pos < input.length && input[pos] === '/') {\n      selector += input[pos++];\n      // Skip whitespace after /\n      while (pos < input.length && isWhitespace(input[pos])) {\n        selector += input[pos++];\n      }\n    }\n\n    // Must end with >\n    if (pos >= input.length || input[pos] !== '>') return null;\n    selector += input[pos++]; // >\n  }\n\n  return selector || null;\n}\n\n// =============================================================================\n// String Literal Tokenization\n// =============================================================================\n\n/**\n * Check if a single quote at pos is a possessive marker ('s).\n * Returns true if this looks like possessive, not a string start.\n *\n * Examples:\n * - #element's *opacity → possessive (returns true)\n * - 'hello' → string (returns false)\n * - it's value → possessive (returns true)\n */\nexport function isPossessiveMarker(input: string, pos: number): boolean {\n  if (pos >= input.length || input[pos] !== \"'\") return false;\n\n  // Check if followed by 's' or 'S'\n  if (pos + 1 >= input.length) return false;\n  const nextChar = input[pos + 1].toLowerCase();\n  if (nextChar !== 's') return false;\n\n  // After 's, should be end, whitespace, or special char (not alphanumeric)\n  if (pos + 2 >= input.length) return true; // end of input\n  const afterS = input[pos + 2];\n  return isWhitespace(afterS) || afterS === '*' || !isAsciiIdentifierChar(afterS);\n}\n\n/**\n * Extract a string literal from the input starting at pos.\n * Handles both ASCII quotes and Unicode quotes.\n *\n * Note: Single quotes that look like possessive markers ('s) are skipped.\n */\nexport function extractStringLiteral(input: string, startPos: number): string | null {\n  if (startPos >= input.length) return null;\n\n  const openQuote = input[startPos];\n  if (!isQuote(openQuote)) return null;\n\n  // Check for possessive marker - don't treat as string\n  if (openQuote === \"'\" && isPossessiveMarker(input, startPos)) {\n    return null;\n  }\n\n  // Map opening quotes to closing quotes\n  const closeQuoteMap: Record<string, string> = {\n    '\"': '\"',\n    \"'\": \"'\",\n    '`': '`',\n    '「': '」',\n  };\n\n  const closeQuote = closeQuoteMap[openQuote];\n  if (!closeQuote) return null;\n\n  let pos = startPos + 1;\n  let literal = openQuote;\n  let escaped = false;\n\n  while (pos < input.length) {\n    const char = input[pos];\n    literal += char;\n\n    if (escaped) {\n      escaped = false;\n    } else if (char === '\\\\') {\n      escaped = true;\n    } else if (char === closeQuote) {\n      // Found closing quote\n      return literal;\n    }\n    pos++;\n  }\n\n  // Unclosed string - return what we have\n  return literal;\n}\n\n// =============================================================================\n// URL Tokenization\n// =============================================================================\n\n/**\n * Check if the input at position starts a URL.\n * Detects: /path, ./path, ../path, //domain.com, http://, https://\n */\nexport function isUrlStart(input: string, pos: number): boolean {\n  if (pos >= input.length) return false;\n\n  const char = input[pos];\n  const next = input[pos + 1] || '';\n  const third = input[pos + 2] || '';\n\n  // Absolute path: /something (but not just /)\n  // Must be followed by alphanumeric or path char, not another / (that's protocol-relative)\n  if (char === '/' && next !== '/' && /[a-zA-Z0-9._-]/.test(next)) {\n    return true;\n  }\n\n  // Protocol-relative: //domain.com\n  if (char === '/' && next === '/' && /[a-zA-Z]/.test(third)) {\n    return true;\n  }\n\n  // Relative path: ./ or ../\n  if (char === '.' && (next === '/' || (next === '.' && third === '/'))) {\n    return true;\n  }\n\n  // Full URL: http:// or https://\n  const slice = input.slice(pos, pos + 8).toLowerCase();\n  if (slice.startsWith('http://') || slice.startsWith('https://')) {\n    return true;\n  }\n\n  return false;\n}\n\n/**\n * Extract a URL from the input starting at pos.\n * Handles paths, query strings, and fragments.\n *\n * Fragment (#) handling:\n * - /page#section → includes fragment as part of URL\n * - #id alone → not a URL (CSS selector)\n */\nexport function extractUrl(input: string, startPos: number): string | null {\n  if (!isUrlStart(input, startPos)) return null;\n\n  let pos = startPos;\n  let url = '';\n\n  // Core URL characters (RFC 3986 unreserved + sub-delims + path/query chars)\n  // Includes: letters, digits, and - . _ ~ : / ? # [ ] @ ! $ & ' ( ) * + , ; = %\n  const urlChars = /[a-zA-Z0-9/:._\\-?&=%@+~!$'()*,;[\\]]/;\n\n  while (pos < input.length) {\n    const char = input[pos];\n\n    // Special handling for #\n    if (char === '#') {\n      // Only include # if we have path content before it (it's a fragment)\n      // If # appears at URL start or after certain chars, stop (might be CSS selector)\n      if (url.length > 0 && /[a-zA-Z0-9/.]$/.test(url)) {\n        // Include fragment\n        url += char;\n        pos++;\n        // Consume fragment identifier (letters, digits, underscore, hyphen)\n        while (pos < input.length && /[a-zA-Z0-9_-]/.test(input[pos])) {\n          url += input[pos++];\n        }\n      }\n      // Stop either way - fragment consumed or # is separate token\n      break;\n    }\n\n    if (urlChars.test(char)) {\n      url += char;\n      pos++;\n    } else {\n      break;\n    }\n  }\n\n  // Minimum length validation\n  if (url.length < 2) return null;\n\n  return url;\n}\n\n// =============================================================================\n// Number Tokenization\n// =============================================================================\n\n/**\n * Extract a number from the input starting at pos.\n * Handles integers and decimals.\n */\nexport function extractNumber(input: string, startPos: number): string | null {\n  if (startPos >= input.length) return null;\n\n  const char = input[startPos];\n  if (!isDigit(char) && char !== '-' && char !== '+') return null;\n\n  let pos = startPos;\n  let number = '';\n\n  // Optional sign\n  if (input[pos] === '-' || input[pos] === '+') {\n    number += input[pos++];\n  }\n\n  // Must have at least one digit\n  if (pos >= input.length || !isDigit(input[pos])) {\n    return null;\n  }\n\n  // Integer part\n  while (pos < input.length && isDigit(input[pos])) {\n    number += input[pos++];\n  }\n\n  // Optional decimal part\n  if (pos < input.length && input[pos] === '.') {\n    number += input[pos++];\n    while (pos < input.length && isDigit(input[pos])) {\n      number += input[pos++];\n    }\n  }\n\n  // Optional duration suffix (s, ms, m, h)\n  if (pos < input.length) {\n    const suffix = input.slice(pos, pos + 2);\n    if (suffix === 'ms') {\n      number += 'ms';\n    } else if (input[pos] === 's' || input[pos] === 'm' || input[pos] === 'h') {\n      number += input[pos];\n    }\n  }\n\n  return number;\n}\n","/**\n * Operator Extractor - Handles programming language operators\n *\n * Extracts operators like +, -, *, /, =, >, <, >=, <=, !=, ===, etc.\n * Supports multi-character operators with longest-match priority.\n */\n\nimport type { ValueExtractor, ExtractionResult } from '../../../interfaces/value-extractor';\n\n/**\n * Default operators for most programming languages.\n * Sorted longest-first for greedy matching.\n */\nexport const DEFAULT_OPERATORS = [\n  // Three-character operators\n  '===',\n  '!==',\n  '->',\n  // Two-character operators\n  '==',\n  '!=',\n  '<=',\n  '>=',\n  '&&',\n  '||',\n  '**',\n  '+=',\n  '-=',\n  '*=',\n  '/=',\n  // Single-character operators\n  '+',\n  '-',\n  '*',\n  '/',\n  '=',\n  '>',\n  '<',\n  '!',\n  '&',\n  '|',\n  '%',\n  '^',\n  '~',\n];\n\n/**\n * OperatorExtractor - Extracts programming language operators.\n */\nexport class OperatorExtractor implements ValueExtractor {\n  readonly name = 'operator';\n\n  constructor(private operators: string[] = DEFAULT_OPERATORS) {\n    // Sort operators longest-first for greedy matching\n    this.operators = [...operators].sort((a, b) => b.length - a.length);\n  }\n\n  canExtract(input: string, position: number): boolean {\n    return this.operators.some(op => input.startsWith(op, position));\n  }\n\n  extract(input: string, position: number): ExtractionResult | null {\n    // Find longest matching operator\n    for (const op of this.operators) {\n      if (input.startsWith(op, position)) {\n        return {\n          value: op,\n          length: op.length,\n        };\n      }\n    }\n\n    return null;\n  }\n}\n","/**\n * Punctuation Extractor - Handles punctuation characters\n *\n * Extracts punctuation like parentheses, brackets, braces, commas, colons, semicolons.\n * Each character is extracted individually (no multi-character punctuation).\n */\n\nimport type { ValueExtractor, ExtractionResult } from '../../../interfaces/value-extractor';\n\n/**\n * Default punctuation characters for most programming languages.\n */\nexport const DEFAULT_PUNCTUATION = '()[]{},:;';\n\n/**\n * PunctuationExtractor - Extracts punctuation characters.\n */\nexport class PunctuationExtractor implements ValueExtractor {\n  readonly name = 'punctuation';\n\n  constructor(private punctuation: string = DEFAULT_PUNCTUATION) {}\n\n  canExtract(input: string, position: number): boolean {\n    return this.punctuation.includes(input[position]);\n  }\n\n  extract(input: string, position: number): ExtractionResult | null {\n    const char = input[position];\n\n    if (this.punctuation.includes(char)) {\n      return {\n        value: char,\n        length: 1,\n      };\n    }\n\n    return null;\n  }\n}\n","/**\n * Value Extractor Interface - Pluggable Tokenization\n *\n * Extracts typed values from input strings.\n * DSLs can provide custom extractors for their domain-specific syntax.\n *\n * Includes the ContextAwareExtractor extension for extractors that need\n * access to tokenizer state (keyword maps, morphological normalizers, etc.).\n */\n\nimport type { MorphologicalNormalizer } from '../core/tokenization/morphology/types';\n\n// =============================================================================\n// Keyword Entry (needed by TokenizerContext)\n// =============================================================================\n\n/**\n * Keyword entry for tokenizer - maps native word to normalized English form.\n * Re-exported here so ContextAwareExtractor consumers can use it without\n * depending on the base-tokenizer module directly.\n */\nexport interface KeywordEntry {\n  readonly native: string;\n  readonly normalized: string;\n}\n\n// =============================================================================\n// Core Extractor Types\n// =============================================================================\n\n/**\n * Extraction result with value and consumed length.\n */\nexport interface ExtractionResult {\n  /** The extracted value */\n  readonly value: string;\n\n  /** Number of characters consumed */\n  readonly length: number;\n\n  /** Optional metadata about the extraction */\n  readonly metadata?: Record<string, unknown>;\n}\n\n/**\n * Value extractor - identifies and extracts typed values from input.\n */\nexport interface ValueExtractor {\n  /** Name of this extractor (for debugging) */\n  readonly name: string;\n\n  /**\n   * Check if this extractor can handle input at position.\n   *\n   * @param input - Full input string\n   * @param position - Current position\n   * @returns True if this extractor should try\n   *\n   * @example\n   * // CSS selector extractor\n   * canExtract('#button', 0) // → true (starts with #)\n   * canExtract('button', 0)  // → false\n   */\n  canExtract(input: string, position: number): boolean;\n\n  /**\n   * Extract value from input at position.\n   *\n   * @param input - Full input string\n   * @param position - Start position\n   * @returns Extraction result or null if extraction failed\n   *\n   * @example\n   * extract('#button', 0) // → { value: '#button', length: 7 }\n   * extract('button', 0)  // → null (can't extract CSS selector)\n   */\n  extract(input: string, position: number): ExtractionResult | null;\n}\n\n/**\n * String literal extractor - handles quoted strings.\n */\nexport class StringLiteralExtractor implements ValueExtractor {\n  readonly name = 'string-literal';\n\n  canExtract(input: string, position: number): boolean {\n    const char = input[position];\n    return (\n      char === '\"' ||\n      char === \"'\" ||\n      char === '`' ||\n      char === '\\u201C' || // Chinese double quote open \"\n      char === '\\u2018' // Chinese single quote open '\n    );\n  }\n\n  extract(input: string, position: number): ExtractionResult | null {\n    const quote = input[position];\n\n    // Chinese double quotes \" ... \"\n    if (quote === '\\u201C') {\n      let length = 1;\n      while (position + length < input.length) {\n        if (input[position + length] === '\\u201D') {\n          length++;\n          return { value: input.substring(position, position + length), length };\n        }\n        length++;\n      }\n      return null;\n    }\n\n    // Chinese single quotes ' ... '\n    if (quote === '\\u2018') {\n      let length = 1;\n      while (position + length < input.length) {\n        if (input[position + length] === '\\u2019') {\n          length++;\n          return { value: input.substring(position, position + length), length };\n        }\n        length++;\n      }\n      return null;\n    }\n\n    // ASCII quotes (same open/close, support escaping)\n    let length = 1;\n    let escaped = false;\n\n    while (position + length < input.length) {\n      const char = input[position + length];\n\n      if (escaped) {\n        escaped = false;\n        length++;\n        continue;\n      }\n\n      if (char === '\\\\') {\n        escaped = true;\n        length++;\n        continue;\n      }\n\n      if (char === quote) {\n        length++; // Include closing quote\n        return {\n          value: input.substring(position, position + length),\n          length,\n        };\n      }\n\n      length++;\n    }\n\n    // Unterminated string\n    return null;\n  }\n}\n\n/**\n * Number extractor - handles integers and floats.\n */\nexport class NumberExtractor implements ValueExtractor {\n  readonly name = 'number';\n\n  canExtract(input: string, position: number): boolean {\n    return /\\d/.test(input[position]);\n  }\n\n  extract(input: string, position: number): ExtractionResult | null {\n    let length = 0;\n    let hasDecimal = false;\n\n    while (position + length < input.length) {\n      const char = input[position + length];\n\n      if (/\\d/.test(char)) {\n        length++;\n      } else if (char === '.' && !hasDecimal) {\n        hasDecimal = true;\n        length++;\n      } else {\n        break;\n      }\n    }\n\n    if (length === 0) return null;\n\n    const numValue = input.substring(position, position + length);\n    const afterNum = position + length;\n\n    // Check for time unit suffixes\n    if (afterNum < input.length) {\n      const remaining = input.slice(afterNum);\n\n      // CJK multi-char time units (longest first)\n      const cjkMultiUnits: { pattern: string; suffix: string }[] = [\n        { pattern: '毫秒', suffix: 'ms' }, // Chinese milliseconds\n        { pattern: '分钟', suffix: 'm' }, // Chinese minutes\n        { pattern: '小时', suffix: 'h' }, // Chinese hours\n        { pattern: 'ミリ秒', suffix: 'ms' }, // Japanese milliseconds\n        { pattern: '時間', suffix: 'h' }, // Japanese hours\n      ];\n      for (const unit of cjkMultiUnits) {\n        if (remaining.startsWith(unit.pattern)) {\n          return {\n            value: numValue + unit.suffix,\n            length: length + unit.pattern.length,\n            metadata: { hasTimeUnit: true },\n          };\n        }\n      }\n\n      // ASCII 'ms' (2 chars, must check before single-char)\n      if (remaining.startsWith('ms')) {\n        return {\n          value: numValue + 'ms',\n          length: length + 2,\n          metadata: { hasTimeUnit: true },\n        };\n      }\n\n      // CJK single-char time units\n      const cjkSingleUnits: { pattern: string; suffix: string }[] = [\n        { pattern: '秒', suffix: 's' }, // CJK seconds\n        { pattern: '分', suffix: 'm' }, // CJK minutes\n      ];\n      for (const unit of cjkSingleUnits) {\n        if (remaining.startsWith(unit.pattern)) {\n          return {\n            value: numValue + unit.suffix,\n            length: length + 1,\n            metadata: { hasTimeUnit: true },\n          };\n        }\n      }\n\n      // ASCII single-char units: s, m, h (with word boundary check)\n      if (/^[smh](?![a-zA-Z])/.test(remaining)) {\n        return {\n          value: numValue + remaining[0],\n          length: length + 1,\n          metadata: { hasTimeUnit: true },\n        };\n      }\n    }\n\n    return { value: numValue, length };\n  }\n}\n\n/**\n * Identifier extractor - handles variable/property names.\n */\nexport class IdentifierExtractor implements ValueExtractor {\n  readonly name = 'identifier';\n\n  canExtract(input: string, position: number): boolean {\n    return /[a-zA-Z_]/.test(input[position]);\n  }\n\n  extract(input: string, position: number): ExtractionResult | null {\n    let length = 0;\n\n    while (position + length < input.length) {\n      const char = input[position + length];\n      if (/[a-zA-Z0-9_]/.test(char)) {\n        length++;\n      } else {\n        break;\n      }\n    }\n\n    return length > 0\n      ? {\n          value: input.substring(position, position + length),\n          length,\n        }\n      : null;\n  }\n}\n\n/**\n * Unicode identifier extractor - handles non-Latin scripts.\n *\n * Matches contiguous runs of Unicode letters, numbers, and combining marks\n * that aren't ASCII (ASCII identifiers are handled by IdentifierExtractor).\n * Essential for DSLs supporting CJK, Arabic, Cyrillic, Devanagari, etc.\n */\nexport class UnicodeIdentifierExtractor implements ValueExtractor {\n  readonly name = 'unicode-identifier';\n\n  canExtract(input: string, position: number): boolean {\n    const code = input.charCodeAt(position);\n    // Skip ASCII range (handled by IdentifierExtractor)\n    if (code < 0x80) return false;\n    // Match any Unicode letter\n    return /\\p{L}/u.test(input[position]);\n  }\n\n  extract(input: string, position: number): ExtractionResult | null {\n    let length = 0;\n\n    while (position + length < input.length) {\n      const char = input[position + length];\n      // Match Unicode letters, numbers, and combining marks (e.g., Arabic diacritics)\n      if (/[\\p{L}\\p{N}\\p{M}]/u.test(char)) {\n        length++;\n      } else {\n        break;\n      }\n    }\n\n    return length > 0 ? { value: input.substring(position, position + length), length } : null;\n  }\n}\n\n/**\n * Whitespace extractor - handles spaces, tabs, newlines.\n */\nexport class WhitespaceExtractor implements ValueExtractor {\n  readonly name = 'whitespace';\n\n  canExtract(input: string, position: number): boolean {\n    return /\\s/.test(input[position]);\n  }\n\n  extract(input: string, position: number): ExtractionResult | null {\n    let length = 0;\n\n    while (position + length < input.length && /\\s/.test(input[position + length])) {\n      length++;\n    }\n\n    return length > 0\n      ? {\n          value: input.substring(position, position + length),\n          length,\n        }\n      : null;\n  }\n}\n\n// =============================================================================\n// Context-Aware Extractor System\n// =============================================================================\n\n/**\n * Tokenizer context provided to context-aware extractors.\n * Gives extractors access to tokenizer state without tight coupling.\n */\nexport interface TokenizerContext {\n  /** ISO 639-1 language code */\n  readonly language: string;\n\n  /** Text direction */\n  readonly direction: 'ltr' | 'rtl';\n\n  /**\n   * Look up a keyword by its native form.\n   * Returns keyword entry with normalized form, or undefined if not found.\n   */\n  lookupKeyword(native: string): KeywordEntry | undefined;\n\n  /**\n   * Check if a word is a known keyword.\n   */\n  isKeyword(native: string): boolean;\n\n  /**\n   * Check if a known keyword starts at the given position.\n   * Useful for word boundary detection in non-space languages.\n   */\n  isKeywordStart(input: string, position: number): boolean;\n\n  /**\n   * Optional morphological normalizer for this language.\n   */\n  readonly normalizer?: MorphologicalNormalizer;\n}\n\n/**\n * Context-aware extractor - has access to tokenizer state.\n *\n * Use this for extractors that need:\n * - Keyword lookup (for normalization)\n * - Morphological analysis (for conjugation handling)\n * - Language-specific rules\n *\n * For stateless extractors (strings, numbers, operators), use ValueExtractor.\n */\nexport interface ContextAwareExtractor extends ValueExtractor {\n  /**\n   * Set the tokenizer context.\n   * Called once by the tokenizer during registration.\n   */\n  setContext(context: TokenizerContext): void;\n}\n\n/**\n * Type guard to check if an extractor is context-aware.\n */\nexport function isContextAwareExtractor(\n  extractor: ValueExtractor | ContextAwareExtractor\n): extractor is ContextAwareExtractor {\n  return 'setContext' in extractor && typeof extractor.setContext === 'function';\n}\n\n/**\n * Create a TokenizerContext from a tokenizer instance.\n * Works with any object that exposes the required methods.\n */\nexport function createTokenizerContext(tokenizer: {\n  language: string;\n  direction: 'ltr' | 'rtl';\n  lookupKeyword(native: string): KeywordEntry | undefined;\n  isKeyword(native: string): boolean;\n  isKeywordStart(input: string, position: number): boolean;\n  normalizer?: MorphologicalNormalizer;\n}): TokenizerContext {\n  const ctx: TokenizerContext = {\n    language: tokenizer.language,\n    direction: tokenizer.direction,\n    lookupKeyword: tokenizer.lookupKeyword.bind(tokenizer),\n    isKeyword: tokenizer.isKeyword.bind(tokenizer),\n    isKeywordStart: tokenizer.isKeywordStart.bind(tokenizer),\n  };\n\n  if (tokenizer.normalizer) {\n    return { ...ctx, normalizer: tokenizer.normalizer };\n  }\n\n  return ctx;\n}\n","/**\n * Default Extractor Sets\n *\n * Provides pre-configured sets of extractors for common use cases.\n * DSLs can use these as a starting point and add domain-specific extractors.\n */\n\nimport type { ValueExtractor } from '../../interfaces/value-extractor';\nimport {\n  StringLiteralExtractor,\n  NumberExtractor,\n  IdentifierExtractor,\n  UnicodeIdentifierExtractor,\n} from '../../interfaces/value-extractor';\nimport { OperatorExtractor, PunctuationExtractor } from './extractors/index';\n\n/**\n * Get default extractors for generic programming-language-style DSLs.\n * These work for most DSLs (SQL, config files, scripts, etc.).\n *\n * Included extractors:\n * - String literals: \"double\", 'single', `backtick`\n * - Numbers: 123, 45.67\n * - Operators: +, -, *, /, =, ==, !=, >=, <=, etc.\n * - Punctuation: ( ) [ ] { } , : ;\n * - Identifiers: variable_names, functionNames\n * - Unicode identifiers: CJK, Arabic, Cyrillic, etc.\n *\n * @returns Array of default extractors\n *\n * @example\n * ```typescript\n * class MyDSLTokenizer extends BaseTokenizer {\n *   constructor() {\n *     super();\n *     this.registerExtractors(getDefaultExtractors());\n *   }\n * }\n * ```\n */\nexport function getDefaultExtractors(): ValueExtractor[] {\n  return [\n    new StringLiteralExtractor(), // \"strings\", 'strings', `strings`\n    new NumberExtractor(), // 123, 45.67\n    new OperatorExtractor(), // +, -, *, /, =, >, <, etc.\n    new PunctuationExtractor(), // ( ) [ ] { } , : ;\n    new IdentifierExtractor(), // variable_names, functionNames (ASCII)\n    new UnicodeIdentifierExtractor(), // CJK, Arabic, Cyrillic, etc.\n  ];\n}\n\n/**\n * Auto-register default extractors in a tokenizer.\n * Convenience helper for chaining.\n *\n * @param tokenizer - Tokenizer to configure\n * @returns The same tokenizer (for chaining)\n *\n * @example\n * ```typescript\n * const tokenizer = withDefaultExtractors(new MyTokenizer());\n * ```\n */\nexport function withDefaultExtractors<\n  T extends { registerExtractors(extractors: ValueExtractor[]): void },\n>(tokenizer: T): T {\n  tokenizer.registerExtractors(getDefaultExtractors());\n  return tokenizer;\n}\n","/**\n * Character Classifiers\n *\n * Unicode range classification and Latin character classifier factories.\n * Used by language-specific tokenizers to define character sets.\n */\n\n// =============================================================================\n// Unicode Range Classification\n// =============================================================================\n\n/**\n * Unicode range tuple: [start, end] (inclusive).\n */\nexport type UnicodeRange = readonly [number, number];\n\n/**\n * Create a character classifier for Unicode ranges.\n * Returns a function that checks if a character's code point falls within any of the ranges.\n *\n * @example\n * // Japanese Hiragana\n * const isHiragana = createUnicodeRangeClassifier([[0x3040, 0x309f]]);\n *\n * // Korean (Hangul syllables + Jamo)\n * const isKorean = createUnicodeRangeClassifier([\n *   [0xac00, 0xd7a3],  // Hangul syllables\n *   [0x1100, 0x11ff],  // Hangul Jamo\n *   [0x3130, 0x318f],  // Hangul Compatibility Jamo\n * ]);\n */\nexport function createUnicodeRangeClassifier(\n  ranges: readonly UnicodeRange[]\n): (char: string) => boolean {\n  return (char: string): boolean => {\n    const code = char.charCodeAt(0);\n    return ranges.some(([start, end]) => code >= start && code <= end);\n  };\n}\n\n/**\n * Combine multiple character classifiers into one.\n * Returns true if any of the classifiers return true.\n *\n * @example\n * const isJapanese = combineClassifiers(isHiragana, isKatakana, isKanji);\n */\nexport function combineClassifiers(\n  ...classifiers: Array<(char: string) => boolean>\n): (char: string) => boolean {\n  return (char: string): boolean => classifiers.some(fn => fn(char));\n}\n\n/**\n * Character classifiers for a Latin-based language.\n */\nexport interface LatinCharClassifiers {\n  /** Check if character is a letter in this language (including accented chars). */\n  isLetter: (char: string) => boolean;\n  /** Check if character is part of an identifier (letter, digit, underscore, hyphen). */\n  isIdentifierChar: (char: string) => boolean;\n}\n\n/**\n * Create character classifiers for a Latin-based language.\n * Returns isLetter and isIdentifierChar functions based on the provided regex.\n *\n * @example\n * // Spanish letters\n * const { isLetter, isIdentifierChar } = createLatinCharClassifiers(/[a-zA-ZáéíóúüñÁÉÍÓÚÜÑ]/);\n *\n * // German letters\n * const { isLetter, isIdentifierChar } = createLatinCharClassifiers(/[a-zA-ZäöüÄÖÜß]/);\n */\nexport function createLatinCharClassifiers(letterPattern: RegExp): LatinCharClassifiers {\n  const isLetter = (char: string): boolean => letterPattern.test(char);\n  const isIdentifierChar = (char: string): boolean => isLetter(char) || /[0-9_-]/.test(char);\n  return { isLetter, isIdentifierChar };\n}\n","/**\n * Base Tokenizer Class\n *\n * Abstract base class for language-specific tokenizers.\n * Provides keyword management, morphological normalization,\n * and high-level token extraction methods.\n */\n\nimport type { LanguageToken, TokenKind, TokenStream, LanguageTokenizer } from '../types';\nimport type { MorphologicalNormalizer, NormalizationResult } from './morphology/types';\nimport {\n  type ValueExtractor,\n  type KeywordEntry,\n  isContextAwareExtractor,\n  createTokenizerContext,\n} from '../../interfaces/value-extractor';\nimport {\n  createToken,\n  createPosition,\n  isWhitespace,\n  isDigit,\n  isAsciiIdentifierChar,\n  TokenStreamImpl,\n  type TimeUnitMapping,\n  type CreateTokenOptions,\n} from './token-utils';\nimport { extractCssSelector, extractStringLiteral, extractNumber, extractUrl } from './extractors';\nimport { DEFAULT_OPERATORS } from './extractors/operator';\nimport { getDefaultExtractors } from './default-extractors';\n\n// Module-scope operator set for O(1) lookup in createSimpleTokenizer.\n// Uses the canonical list from OperatorExtractor to avoid duplication.\nconst SIMPLE_TOKENIZER_OPERATOR_SET = new Set(DEFAULT_OPERATORS);\n\n// =============================================================================\n// Types\n// =============================================================================\n\n// KeywordEntry is imported from interfaces/value-extractor and re-exported\n// for backward compatibility with code importing from this module.\nexport type { KeywordEntry };\n\n/**\n * Profile interface for keyword derivation.\n * Matches the structure of LanguageProfile but only includes fields needed for tokenization.\n */\nexport interface TokenizerProfile {\n  readonly keywords?: Record<\n    string,\n    { primary: string; alternatives?: string[]; normalized?: string }\n  >;\n  readonly references?: Record<string, string>;\n  readonly roleMarkers?: Record<\n    string,\n    { primary: string; alternatives?: string[]; position?: string }\n  >;\n  readonly possessive?: {\n    readonly marker: string;\n    readonly markerPosition: 'after-object' | 'between' | 'before-property';\n    readonly specialForms?: Record<string, string>;\n    readonly usePossessiveAdjectives?: boolean;\n    readonly keywords?: Record<string, string>;\n  };\n}\n\n// =============================================================================\n// Base Tokenizer Class\n// =============================================================================\n\n/**\n * Abstract base class for language-specific tokenizers.\n * Provides common functionality for CSS selectors, strings, and numbers.\n */\nexport abstract class BaseTokenizer implements LanguageTokenizer {\n  abstract readonly language: string;\n  abstract readonly direction: 'ltr' | 'rtl';\n\n  /** Optional morphological normalizer for this language */\n  protected normalizer?: MorphologicalNormalizer;\n\n  /** Keywords derived from profile, sorted longest-first for greedy matching */\n  protected profileKeywords: KeywordEntry[] = [];\n\n  /** Map for O(1) keyword lookups by lowercase native word */\n  protected profileKeywordMap: Map<string, KeywordEntry> = new Map();\n\n  /**\n   * Pluggable value extractors for domain-specific syntax.\n   * When registered, BaseTokenizer will use extractor-based tokenization instead of legacy methods.\n   */\n  protected extractors: ValueExtractor[] = [];\n\n  /**\n   * Tokenize input string to token stream.\n   * Delegates to extractor-based tokenization if extractors are registered,\n   * otherwise subclass must override this method.\n   *\n   * @param input - Input string to tokenize\n   * @returns Token stream\n   */\n  tokenize(input: string): TokenStream {\n    if (this.isUsingExtractors()) {\n      return this.tokenizeWithExtractors(input);\n    }\n\n    // If no extractors registered, subclass must provide implementation\n    throw new Error(\n      `${this.constructor.name}: tokenize() not implemented and no extractors registered. ` +\n        'Either register extractors or override tokenize() method.'\n    );\n  }\n\n  abstract classifyToken(token: string): TokenKind;\n\n  /**\n   * Register a value extractor for domain-specific syntax.\n   * Extractors are tried in registration order during tokenization.\n   * Context-aware extractors automatically receive the tokenizer context.\n   *\n   * @param extractor - Value extractor to register\n   */\n  registerExtractor(extractor: ValueExtractor): void {\n    if (isContextAwareExtractor(extractor)) {\n      extractor.setContext(createTokenizerContext(this as any));\n    }\n    this.extractors.push(extractor);\n  }\n\n  /**\n   * Register multiple value extractors at once.\n   *\n   * @param extractors - Array of value extractors to register\n   */\n  registerExtractors(extractors: ValueExtractor[]): void {\n    for (const extractor of extractors) {\n      this.registerExtractor(extractor);\n    }\n  }\n\n  /**\n   * Clear all registered extractors.\n   * Returns tokenizer to legacy mode.\n   */\n  clearExtractors(): void {\n    this.extractors = [];\n  }\n\n  /**\n   * Check if this tokenizer is using extractor-based tokenization.\n   * Returns true if any extractors are registered.\n   */\n  protected isUsingExtractors(): boolean {\n    return this.extractors.length > 0;\n  }\n\n  /**\n   * Tokenize input using registered value extractors.\n   * This is the new path - extractors handle all syntax detection.\n   *\n   * @param input - Input string to tokenize\n   * @returns Token stream\n   */\n  protected tokenizeWithExtractors(input: string): TokenStream {\n    const tokens: LanguageToken[] = [];\n    let pos = 0;\n\n    while (pos < input.length) {\n      // Skip whitespace\n      while (pos < input.length && isWhitespace(input[pos])) {\n        pos++;\n      }\n      if (pos >= input.length) break;\n\n      // Try registered extractors in order\n      let extracted = false;\n      for (const extractor of this.extractors) {\n        if (extractor.canExtract(input, pos)) {\n          const result = extractor.extract(input, pos);\n          if (result) {\n            // Promote normalized/stem/stemConfidence from metadata to top-level token options\n            const normalized = result.metadata?.normalized as string | undefined;\n            const stem = result.metadata?.stem as string | undefined;\n            const stemConfidence = result.metadata?.stemConfidence as number | undefined;\n\n            // Build clean metadata without promoted fields\n            const cleanMetadata: Record<string, unknown> = {};\n            if (result.metadata) {\n              for (const [key, value] of Object.entries(result.metadata)) {\n                if (key !== 'normalized' && key !== 'stem' && key !== 'stemConfidence') {\n                  cleanMetadata[key] = value;\n                }\n              }\n            }\n\n            const options: CreateTokenOptions = {};\n            if (normalized) options.normalized = normalized;\n            if (stem) options.stem = stem;\n            if (stemConfidence !== undefined) options.stemConfidence = stemConfidence;\n            if (Object.keys(cleanMetadata).length > 0) options.metadata = cleanMetadata;\n\n            tokens.push(\n              createToken(\n                result.value,\n                this.classifyToken(result.value),\n                createPosition(pos, pos + result.length),\n                Object.keys(options).length > 0 ? options : undefined\n              )\n            );\n            pos += result.length;\n            extracted = true;\n            break;\n          }\n        }\n      }\n\n      // Fallback: single character as operator/punctuation\n      if (!extracted) {\n        const char = input[pos];\n        const kind = this.classifyUnknownChar(char);\n        tokens.push(createToken(char, kind, createPosition(pos, pos + 1)));\n        pos++;\n      }\n    }\n\n    return new TokenStreamImpl(tokens, this.language);\n  }\n\n  /**\n   * Classify an unknown character when no extractor matches.\n   * Provides sensible defaults for common syntax.\n   *\n   * @param char - Character to classify\n   * @returns Token kind\n   */\n  protected classifyUnknownChar(char: string): TokenKind {\n    if ('()[]{},:;'.includes(char)) return 'punctuation';\n    if ('+-*/<>=!&|'.includes(char)) return 'operator';\n    return 'identifier';\n  }\n\n  /**\n   * Check if current position is a property access (obj.prop) vs CSS selector (.active).\n   * Property access: no whitespace before '.', previous token is identifier/keyword/selector.\n   * Also detects standalone method calls: .identifier( pattern.\n   *\n   * Returns true if '.' was emitted as an operator token and pos should advance by 1.\n   * Returns false if this is a CSS selector and should be handled by trySelector().\n   */\n  protected tryPropertyAccess(input: string, pos: number, tokens: LanguageToken[]): boolean {\n    if (input[pos] !== '.') return false;\n\n    const lastToken = tokens[tokens.length - 1];\n    // Property access requires NO whitespace between tokens (e.g., \"obj.prop\")\n    const hasWhitespaceBefore = lastToken && lastToken.position.end < pos;\n    const isPropertyAccess =\n      lastToken &&\n      !hasWhitespaceBefore &&\n      (lastToken.kind === 'identifier' ||\n        lastToken.kind === 'keyword' ||\n        lastToken.kind === 'selector');\n\n    if (isPropertyAccess) {\n      tokens.push(createToken('.', 'operator', createPosition(pos, pos + 1)));\n      return true;\n    }\n\n    // Check for method call pattern at start: .identifier(\n    const methodStart = pos + 1;\n    let methodEnd = methodStart;\n    while (methodEnd < input.length && isAsciiIdentifierChar(input[methodEnd])) {\n      methodEnd++;\n    }\n    if (methodEnd < input.length && input[methodEnd] === '(') {\n      tokens.push(createToken('.', 'operator', createPosition(pos, pos + 1)));\n      return true;\n    }\n\n    return false;\n  }\n\n  /**\n   * Initialize keyword mappings from a language profile.\n   * Builds a list of native→english mappings from:\n   * - profile.keywords (primary + alternatives)\n   * - profile.references (me, it, you, etc.)\n   * - profile.roleMarkers (into, from, with, etc.)\n   *\n   * Results are sorted longest-first for greedy matching (important for non-space languages).\n   * Extras take precedence over profile entries when there are duplicates.\n   *\n   * @param profile - Language profile containing keyword translations\n   * @param extras - Additional keyword entries to include (literals, positional, events)\n   */\n  protected initializeKeywordsFromProfile(\n    profile: TokenizerProfile,\n    extras: KeywordEntry[] = []\n  ): void {\n    // Use a Map to deduplicate, with extras taking precedence\n    const keywordMap = new Map<string, KeywordEntry>();\n\n    // Extract from keywords (command translations)\n    if (profile.keywords) {\n      for (const [normalized, translation] of Object.entries(profile.keywords)) {\n        // Primary translation\n        keywordMap.set(translation.primary, {\n          native: translation.primary,\n          normalized: translation.normalized || normalized,\n        });\n\n        // Alternative forms\n        if (translation.alternatives) {\n          for (const alt of translation.alternatives) {\n            keywordMap.set(alt, {\n              native: alt,\n              normalized: translation.normalized || normalized,\n            });\n          }\n        }\n      }\n    }\n\n    // Extract from references (me, it, you, etc.)\n    if (profile.references) {\n      for (const [normalized, native] of Object.entries(profile.references)) {\n        keywordMap.set(native, { native, normalized });\n      }\n      // Also register English canonical forms as universal fallbacks.\n      // Users frequently mix English references (me, it, you) into non-English\n      // hyperscript (e.g., \"alternar .active on me\"). Without this, the English\n      // word \"me\" would be unrecognized in non-English token streams.\n      for (const canonical of Object.keys(profile.references)) {\n        if (!keywordMap.has(canonical)) {\n          keywordMap.set(canonical, { native: canonical, normalized: canonical });\n        }\n      }\n    }\n\n    // Extract from roleMarkers (into, from, with, etc.)\n    if (profile.roleMarkers) {\n      for (const [role, marker] of Object.entries(profile.roleMarkers)) {\n        if (marker.primary) {\n          keywordMap.set(marker.primary, { native: marker.primary, normalized: role });\n        }\n        if (marker.alternatives) {\n          for (const alt of marker.alternatives) {\n            keywordMap.set(alt, { native: alt, normalized: role });\n          }\n        }\n      }\n    }\n\n    // Extract from possessive keywords (e.g., ñuqapa, qampa for Quechua)\n    if (profile.possessive?.keywords) {\n      for (const [native, normalized] of Object.entries(profile.possessive.keywords)) {\n        keywordMap.set(native, { native, normalized });\n      }\n    }\n\n    // Add extra entries (literals, positional, events) - these OVERRIDE profile entries\n    for (const extra of extras) {\n      keywordMap.set(extra.native, extra);\n    }\n\n    // Convert to array and sort longest-first for greedy matching\n    this.profileKeywords = Array.from(keywordMap.values()).sort(\n      (a, b) => b.native.length - a.native.length\n    );\n\n    // Build Map for O(1) lookups (case-insensitive + diacritic-insensitive)\n    // This allows matching both 'بدّل' (with shadda) and 'بدل' (without) to the same entry\n    this.profileKeywordMap = new Map();\n    for (const keyword of this.profileKeywords) {\n      // Add original form (with diacritics if present)\n      this.profileKeywordMap.set(keyword.native.toLowerCase(), keyword);\n\n      // Add diacritic-normalized form (for Arabic, Turkish, etc.)\n      const normalized = this.removeDiacritics(keyword.native);\n      if (normalized !== keyword.native && !this.profileKeywordMap.has(normalized.toLowerCase())) {\n        this.profileKeywordMap.set(normalized.toLowerCase(), keyword);\n      }\n    }\n  }\n\n  /**\n   * Remove diacritical marks from a word for normalization.\n   * Primarily for Arabic (shadda, fatha, kasra, damma, sukun, etc.)\n   * but could be extended for other languages.\n   *\n   * @param word - Word to normalize\n   * @returns Word without diacritics\n   */\n  protected removeDiacritics(word: string): string {\n    // Arabic diacritics: U+064B-U+0652 (fatha, kasra, damma, sukun, shadda, etc.)\n    // U+0670 (superscript alif)\n    return word.replace(/[\\u064B-\\u0652\\u0670]/g, '');\n  }\n\n  /**\n   * Try to match a keyword from profile at the current position.\n   * Uses longest-first greedy matching (important for non-space languages).\n   *\n   * @param input - Input string\n   * @param pos - Current position\n   * @returns Token if matched, null otherwise\n   */\n  protected tryProfileKeyword(input: string, pos: number): LanguageToken | null {\n    for (const entry of this.profileKeywords) {\n      if (input.slice(pos).startsWith(entry.native)) {\n        return createToken(\n          entry.native,\n          'keyword',\n          createPosition(pos, pos + entry.native.length),\n          entry.normalized\n        );\n      }\n    }\n    return null;\n  }\n\n  /**\n   * Check if the remaining input starts with any known keyword.\n   * Useful for non-space languages to detect word boundaries.\n   *\n   * @param input - Input string\n   * @param pos - Current position\n   * @returns true if a keyword starts at this position\n   */\n  protected isKeywordStart(input: string, pos: number): boolean {\n    const remaining = input.slice(pos);\n    return this.profileKeywords.some(entry => remaining.startsWith(entry.native));\n  }\n\n  /**\n   * Look up a keyword by native word (case-insensitive).\n   * O(1) lookup using the keyword map.\n   *\n   * @param native - Native word to look up\n   * @returns KeywordEntry if found, undefined otherwise\n   */\n  protected lookupKeyword(native: string): KeywordEntry | undefined {\n    return this.profileKeywordMap.get(native.toLowerCase());\n  }\n\n  /**\n   * Check if a word is a known keyword (case-insensitive).\n   * O(1) lookup using the keyword map.\n   *\n   * @param native - Native word to check\n   * @returns true if the word is a keyword\n   */\n  protected isKeyword(native: string): boolean {\n    return this.profileKeywordMap.has(native.toLowerCase());\n  }\n\n  /**\n   * Set the morphological normalizer for this tokenizer.\n   */\n  setNormalizer(normalizer: MorphologicalNormalizer): void {\n    this.normalizer = normalizer;\n  }\n\n  /**\n   * Try to normalize a word using the morphological normalizer.\n   * Returns null if no normalizer is set or normalization fails.\n   *\n   * Note: We don't check isNormalizable() here because the individual tokenizers\n   * historically called normalize() directly without that check. The normalize()\n   * method itself handles returning noChange() for words that can't be normalized.\n   */\n  protected tryNormalize(word: string): NormalizationResult | null {\n    if (!this.normalizer) return null;\n\n    const result = this.normalizer.normalize(word);\n\n    // Only return if actually normalized (stem differs from input)\n    if (result.stem !== word && result.confidence >= 0.7) {\n      return result;\n    }\n\n    return null;\n  }\n\n  /**\n   * Try morphological normalization and keyword lookup.\n   *\n   * If the word can be normalized to a stem that matches a known keyword,\n   * returns a keyword token with morphological metadata (stem, stemConfidence).\n   *\n   * This is the common pattern for handling conjugated verbs across languages:\n   * 1. Normalize the word (e.g., \"toggled\" → \"toggle\")\n   * 2. Look up the stem in the keyword map\n   * 3. Create a token with both the original form and stem metadata\n   *\n   * @param word - The word to normalize and look up\n   * @param startPos - Start position for the token\n   * @param endPos - End position for the token\n   * @returns Token if stem matches a keyword, null otherwise\n   */\n  protected tryMorphKeywordMatch(\n    word: string,\n    startPos: number,\n    endPos: number\n  ): LanguageToken | null {\n    const result = this.tryNormalize(word);\n    if (!result) return null;\n\n    // Check if the stem is a known keyword\n    const stemEntry = this.lookupKeyword(result.stem);\n    if (!stemEntry) return null;\n\n    const tokenOptions: CreateTokenOptions = {\n      normalized: stemEntry.normalized,\n      stem: result.stem,\n      stemConfidence: result.confidence,\n    };\n    return createToken(word, 'keyword', createPosition(startPos, endPos), tokenOptions);\n  }\n\n  /**\n   * Try to extract a CSS selector at the current position.\n   */\n  protected trySelector(input: string, pos: number): LanguageToken | null {\n    const selector = extractCssSelector(input, pos);\n    if (selector) {\n      return createToken(selector, 'selector', createPosition(pos, pos + selector.length));\n    }\n    return null;\n  }\n\n  /**\n   * Try to extract an event modifier at the current position.\n   * Event modifiers are .once, .debounce(N), .throttle(N), .queue(strategy)\n   */\n  protected tryEventModifier(input: string, pos: number): LanguageToken | null {\n    // Must start with a dot\n    if (input[pos] !== '.') {\n      return null;\n    }\n\n    // Match pattern: .(once|debounce|throttle|queue) followed by optional (value)\n    const match = input\n      .slice(pos)\n      .match(/^\\.(?:once|debounce|throttle|queue)(?:\\(([^)]+)\\))?(?:\\s|$|\\.)/);\n    if (!match) {\n      return null;\n    }\n\n    const fullMatch = match[0].replace(/(\\s|\\.)$/, ''); // Remove trailing space or dot\n    const modifierName = fullMatch.slice(1).split('(')[0]; // Extract modifier name\n    const value = match[1]; // Extract value from parentheses if present\n\n    // Create token with metadata\n    const token = createToken(\n      fullMatch,\n      'event-modifier',\n      createPosition(pos, pos + fullMatch.length)\n    );\n\n    // Add metadata for the modifier\n    return {\n      ...token,\n      metadata: {\n        modifierName,\n        value: value ? (modifierName === 'queue' ? value : parseInt(value, 10)) : undefined,\n      },\n    };\n  }\n\n  /**\n   * Try to extract a string literal at the current position.\n   */\n  protected tryString(input: string, pos: number): LanguageToken | null {\n    const literal = extractStringLiteral(input, pos);\n    if (literal) {\n      return createToken(literal, 'literal', createPosition(pos, pos + literal.length));\n    }\n    return null;\n  }\n\n  /**\n   * Try to extract a number at the current position.\n   */\n  protected tryNumber(input: string, pos: number): LanguageToken | null {\n    const number = extractNumber(input, pos);\n    if (number) {\n      return createToken(number, 'literal', createPosition(pos, pos + number.length));\n    }\n    return null;\n  }\n\n  /**\n   * Configuration for native language time units.\n   * Maps patterns to their standard suffix (ms, s, m, h).\n   */\n  protected static readonly STANDARD_TIME_UNITS: readonly TimeUnitMapping[] = [\n    { pattern: 'ms', suffix: 'ms', length: 2 },\n    { pattern: 's', suffix: 's', length: 1, checkBoundary: true },\n    { pattern: 'm', suffix: 'm', length: 1, checkBoundary: true, notFollowedBy: 's' },\n    { pattern: 'h', suffix: 'h', length: 1, checkBoundary: true },\n  ];\n\n  /**\n   * Try to match a time unit from a list of patterns.\n   *\n   * @param input - Input string\n   * @param pos - Position after the number\n   * @param timeUnits - Array of time unit mappings (native pattern → standard suffix)\n   * @param skipWhitespace - Whether to skip whitespace before time unit (default: false)\n   * @returns Object with matched suffix and new position, or null if no match\n   */\n  protected tryMatchTimeUnit(\n    input: string,\n    pos: number,\n    timeUnits: readonly TimeUnitMapping[],\n    skipWhitespace = false\n  ): { suffix: string; endPos: number } | null {\n    let unitPos = pos;\n\n    // Optionally skip whitespace before time unit\n    if (skipWhitespace) {\n      while (unitPos < input.length && isWhitespace(input[unitPos])) {\n        unitPos++;\n      }\n    }\n\n    const remaining = input.slice(unitPos);\n\n    // Check each time unit pattern\n    for (const unit of timeUnits) {\n      const candidate = remaining.slice(0, unit.length);\n      const matches = unit.caseInsensitive\n        ? candidate.toLowerCase() === unit.pattern.toLowerCase()\n        : candidate === unit.pattern;\n\n      if (matches) {\n        // Check notFollowedBy constraint (e.g., 'm' should not match 'ms')\n        if (unit.notFollowedBy) {\n          const nextChar = remaining[unit.length] || '';\n          if (nextChar === unit.notFollowedBy) continue;\n        }\n\n        // Check word boundary if required\n        if (unit.checkBoundary) {\n          const nextChar = remaining[unit.length] || '';\n          if (isAsciiIdentifierChar(nextChar)) continue;\n        }\n\n        return { suffix: unit.suffix, endPos: unitPos + unit.length };\n      }\n    }\n\n    return null;\n  }\n\n  /**\n   * Parse a base number (sign, integer, decimal) without time units.\n   * Returns the number string and end position.\n   *\n   * @param input - Input string\n   * @param startPos - Start position\n   * @param allowSign - Whether to allow +/- sign (default: true)\n   * @returns Object with number string and end position, or null\n   */\n  protected parseBaseNumber(\n    input: string,\n    startPos: number,\n    allowSign = true\n  ): { number: string; endPos: number } | null {\n    let pos = startPos;\n    let number = '';\n\n    // Optional sign\n    if (allowSign && (input[pos] === '-' || input[pos] === '+')) {\n      number += input[pos++];\n    }\n\n    // Must have at least one digit\n    if (pos >= input.length || !isDigit(input[pos])) {\n      return null;\n    }\n\n    // Integer part\n    while (pos < input.length && isDigit(input[pos])) {\n      number += input[pos++];\n    }\n\n    // Optional decimal\n    if (pos < input.length && input[pos] === '.') {\n      number += input[pos++];\n      while (pos < input.length && isDigit(input[pos])) {\n        number += input[pos++];\n      }\n    }\n\n    if (!number || number === '-' || number === '+') return null;\n\n    return { number, endPos: pos };\n  }\n\n  /**\n   * Try to extract a number with native language time units.\n   *\n   * This is a template method that handles the common pattern:\n   * 1. Parse the base number (sign, integer, decimal)\n   * 2. Try to match native language time units\n   * 3. Fall back to standard time units (ms, s, m, h)\n   *\n   * @param input - Input string\n   * @param pos - Start position\n   * @param nativeTimeUnits - Language-specific time unit mappings\n   * @param options - Configuration options\n   * @returns Token if number found, null otherwise\n   */\n  protected tryNumberWithTimeUnits(\n    input: string,\n    pos: number,\n    nativeTimeUnits: readonly TimeUnitMapping[],\n    options: { allowSign?: boolean; skipWhitespace?: boolean } = {}\n  ): LanguageToken | null {\n    const { allowSign = true, skipWhitespace = false } = options;\n\n    // Parse base number\n    const baseResult = this.parseBaseNumber(input, pos, allowSign);\n    if (!baseResult) return null;\n\n    let { number, endPos } = baseResult;\n\n    // Try native time units first, then standard\n    const allUnits = [...nativeTimeUnits, ...BaseTokenizer.STANDARD_TIME_UNITS];\n    const timeMatch = this.tryMatchTimeUnit(input, endPos, allUnits, skipWhitespace);\n\n    if (timeMatch) {\n      number += timeMatch.suffix;\n      endPos = timeMatch.endPos;\n    }\n\n    return createToken(number, 'literal', createPosition(pos, endPos));\n  }\n\n  /**\n   * Try to extract a URL at the current position.\n   * Handles /path, ./path, ../path, //domain.com, http://, https://\n   */\n  protected tryUrl(input: string, pos: number): LanguageToken | null {\n    const url = extractUrl(input, pos);\n    if (url) {\n      return createToken(url, 'url', createPosition(pos, pos + url.length));\n    }\n    return null;\n  }\n\n  /**\n   * Try to extract a variable reference (:varname) at the current position.\n   * In hyperscript, :x refers to a local variable named x.\n   */\n  protected tryVariableRef(input: string, pos: number): LanguageToken | null {\n    if (input[pos] !== ':') return null;\n    if (pos + 1 >= input.length) return null;\n    if (!isAsciiIdentifierChar(input[pos + 1])) return null;\n\n    let endPos = pos + 1;\n    while (endPos < input.length && isAsciiIdentifierChar(input[endPos])) {\n      endPos++;\n    }\n\n    const varRef = input.slice(pos, endPos);\n    return createToken(varRef, 'identifier', createPosition(pos, endPos));\n  }\n\n  /**\n   * Try to extract an operator or punctuation token at the current position.\n   * Handles two-character operators (==, !=, etc.) and single-character operators.\n   */\n  protected tryOperator(input: string, pos: number): LanguageToken | null {\n    // Two-character operators\n    const twoChar = input.slice(pos, pos + 2);\n    if (['==', '!=', '<=', '>=', '&&', '||', '->'].includes(twoChar)) {\n      return createToken(twoChar, 'operator', createPosition(pos, pos + 2));\n    }\n\n    // Single-character operators\n    const oneChar = input[pos];\n    if (['<', '>', '!', '+', '-', '*', '/', '='].includes(oneChar)) {\n      return createToken(oneChar, 'operator', createPosition(pos, pos + 1));\n    }\n\n    // Punctuation\n    if (['(', ')', '{', '}', ',', ';', ':'].includes(oneChar)) {\n      return createToken(oneChar, 'punctuation', createPosition(pos, pos + 1));\n    }\n\n    return null;\n  }\n\n  /**\n   * Try to match a multi-character particle from a list.\n   *\n   * Used by languages like Japanese, Korean, and Chinese that have\n   * multi-character particles (e.g., Japanese から, まで, より).\n   *\n   * @param input - Input string\n   * @param pos - Current position\n   * @param particles - Array of multi-character particles to match\n   * @returns Token if matched, null otherwise\n   */\n  protected tryMultiCharParticle(\n    input: string,\n    pos: number,\n    particles: readonly string[]\n  ): LanguageToken | null {\n    for (const particle of particles) {\n      if (input.slice(pos, pos + particle.length) === particle) {\n        return createToken(particle, 'particle', createPosition(pos, pos + particle.length));\n      }\n    }\n    return null;\n  }\n}\n\n// =============================================================================\n// Simple Tokenizer Factory\n// =============================================================================\n\n/**\n * Configuration for createSimpleTokenizer.\n *\n * Creates a tokenizer from declarative config instead of a class definition.\n * Covers the common pattern used by domain packages (SQL, BDD, JSX).\n *\n * **Keyword resolution** uses two additive paths:\n * 1. `keywords` — explicit list, checked first. Respects `caseInsensitive`.\n * 2. `keywordProfile` — populates BaseTokenizer's profile keyword map via\n *    `initializeKeywordsFromProfile()`. Checked second via `isKeyword()`, which\n *    always lowercases (harmless for CJK/Arabic; notable for Latin scripts\n *    with `caseInsensitive: false`). Provides normalization metadata for\n *    non-Latin scripts.\n */\nexport interface SimpleTokenizerConfig {\n  /** ISO 639-1 language code */\n  language: string;\n  /** Text direction (default: 'ltr') */\n  direction?: 'ltr' | 'rtl';\n  /** Keywords to recognize (lowercased for lookup if caseInsensitive) */\n  keywords: string[];\n  /** Extra keyword entries for non-Latin normalization */\n  keywordExtras?: KeywordEntry[];\n  /** Profile for initializeKeywordsFromProfile (for non-Latin scripts) */\n  keywordProfile?: TokenizerProfile;\n  /** Include operator classification (default: false). Uses DEFAULT_OPERATORS from OperatorExtractor. */\n  includeOperators?: boolean;\n  /** Case-insensitive keyword matching (default: true) */\n  caseInsensitive?: boolean;\n  /** Custom extractors registered BEFORE default extractors */\n  customExtractors?: ValueExtractor[];\n}\n\n/**\n * Create a tokenizer from declarative configuration.\n *\n * Eliminates the boilerplate of extending BaseTokenizer for simple domain tokenizers.\n * Handles keyword classification, optional operator support, and non-Latin keyword setup.\n *\n * @example\n * ```typescript\n * const englishSQL = createSimpleTokenizer({\n *   language: 'en',\n *   keywords: ['select', 'insert', 'update', 'delete', 'from', 'into', 'where', 'set', 'values'],\n *   includeOperators: true,\n *   caseInsensitive: true,\n * });\n * ```\n */\nexport function createSimpleTokenizer(config: SimpleTokenizerConfig): LanguageTokenizer {\n  const {\n    language,\n    direction = 'ltr',\n    keywords,\n    keywordExtras,\n    keywordProfile,\n    includeOperators = false,\n    caseInsensitive = true,\n    customExtractors,\n  } = config;\n\n  const keywordSet = new Set(caseInsensitive ? keywords.map(k => k.toLowerCase()) : keywords);\n\n  class SimpleTokenizer extends BaseTokenizer {\n    readonly language = language;\n    readonly direction = direction;\n\n    constructor() {\n      super();\n      if (customExtractors) {\n        this.registerExtractors(customExtractors);\n      }\n      this.registerExtractors(getDefaultExtractors());\n      if (keywordProfile) {\n        this.initializeKeywordsFromProfile(keywordProfile, keywordExtras);\n      }\n    }\n\n    classifyToken(token: string): TokenKind {\n      // Fast path: explicit keywords from config (respects caseInsensitive)\n      const lookup = caseInsensitive ? token.toLowerCase() : token;\n      if (keywordSet.has(lookup)) return 'keyword';\n      // Profile path: non-Latin normalization (always lowercases via profileKeywordMap)\n      if (this.isKeyword(token)) return 'keyword';\n      if (/^\\d/.test(token)) return 'literal';\n      if (/^['\"]/.test(token)) return 'literal';\n      if (includeOperators && SIMPLE_TOKENIZER_OPERATOR_SET.has(token)) return 'operator';\n      return 'identifier';\n    }\n  }\n\n  return new SimpleTokenizer();\n}\n","/**\n * Morphological Normalizer Types\n *\n * Defines interfaces for language-specific morphological analysis.\n * Normalizers reduce conjugated/inflected forms to canonical stems\n * that can be matched against keyword dictionaries.\n */\n\n/**\n * Result of morphological normalization.\n */\nexport interface NormalizationResult {\n  /** The extracted stem/root form */\n  readonly stem: string;\n\n  /** Confidence in the normalization (0.0-1.0) */\n  readonly confidence: number;\n\n  /** Optional metadata about the transformation */\n  readonly metadata?: NormalizationMetadata;\n}\n\n/**\n * Metadata about morphological transformations applied.\n */\nexport interface NormalizationMetadata {\n  /** Prefixes that were removed */\n  readonly removedPrefixes?: readonly string[];\n\n  /** Suffixes that were removed */\n  readonly removedSuffixes?: readonly string[];\n\n  /** Type of conjugation detected */\n  readonly conjugationType?: ConjugationType;\n\n  /** Original form classification */\n  readonly originalForm?: string;\n\n  /** Applied transformation rules (for debugging) */\n  readonly appliedRules?: readonly string[];\n}\n\n/**\n * Types of verb conjugation/inflection.\n */\nexport type ConjugationType =\n  // Tense\n  | 'present'\n  | 'past'\n  | 'future'\n  | 'progressive'\n  | 'perfect'\n  // Mood\n  | 'imperative'\n  | 'subjunctive'\n  | 'conditional'\n  // Voice\n  | 'passive'\n  | 'causative'\n  // Politeness (Japanese/Korean)\n  | 'polite'\n  | 'humble'\n  | 'honorific'\n  // Form\n  | 'negative'\n  | 'potential'\n  | 'volitional'\n  // Japanese conditional forms\n  | 'conditional-tara' // たら/したら - if/when (completed action)\n  | 'conditional-to' // と/すると - when (habitual/expected)\n  | 'conditional-ba' // ば/すれば - if (hypothetical)\n  // Korean-specific\n  | 'connective' // 하고, 해서 etc.\n  | 'conditional-myeon' // -(으)면 - if/when (general conditional)\n  | 'temporal-ttae' // -(으)ㄹ 때 - when (at the time of)\n  | 'causal-nikka' // -(으)니까 - because/since\n  // Korean honorific forms (-시- infix)\n  | 'honorific-conditional' // -하시면 - if (honorific)\n  | 'honorific-temporal' // -하실 때 - when (honorific)\n  | 'honorific-causal' // -하시니까 - because (honorific)\n  | 'honorific-past' // -하셨어요 - past (honorific)\n  | 'honorific-polite' // -하십니다 - polite (honorific)\n  // Korean sequential forms\n  | 'sequential-after' // -고 나서 - after doing\n  | 'sequential-before' // -기 전에 - before doing\n  | 'immediate' // -자마자 - as soon as\n  | 'obligation' // -아야/어야 해 - must do, should do\n  // Spanish-specific\n  | 'reflexive'\n  | 'reflexive-imperative'\n  | 'gerund'\n  | 'participle'\n  // Arabic-specific\n  | 'conditional-idha' // إذا - if/when (hypothetical)\n  | 'temporal-indama' // عندما - when (temporal conjunction)\n  | 'temporal-hina' // حين - at the time of\n  | 'temporal-lamma' // لمّا - when (past emphasis)\n  | 'past-verb' // فعل ماضي - past tense verb\n  // Turkish-specific\n  | 'conditional-se' // -se/-sa - if (hypothetical)\n  | 'temporal-ince' // -ince/-ınca/-unca/-ünce - when/as\n  | 'temporal-dikce' // -dikçe/-dıkça/-dukça/-dükçe - as/while\n  | 'aorist' // -ir/-ar - habitual/general\n  | 'optative' // -eyim/-ayım/-elim/-alım - let me/us\n  | 'necessitative' // -meli/-malı - must/should\n  // Japanese request/contracted forms\n  | 'request' // てください/でください - polite request\n  | 'casual-request' // てくれ/でくれ - casual request\n  | 'contracted' // ちゃう/じゃう - contracted completion (てしまう)\n  | 'contracted-past' // ちゃった/じゃった - contracted past completion\n  // Compound\n  | 'compound' // Multi-layer suffixes (ていなかった, 하고나서였어)\n  | 'te-form' // Japanese て-form\n  | 'dictionary'; // Base/infinitive form\n\n/**\n * Interface for language-specific morphological normalizers.\n *\n * Normalizers attempt to reduce inflected word forms to their\n * canonical stems. This enables matching conjugated verbs against\n * keyword dictionaries that only contain base forms.\n *\n * Example (Japanese):\n *   切り替えた (past) → { stem: '切り替え', confidence: 0.85 }\n *   切り替えます (polite) → { stem: '切り替え', confidence: 0.85 }\n *\n * Example (Spanish):\n *   mostrarse (reflexive infinitive) → { stem: 'mostrar', confidence: 0.85 }\n *   alternando (gerund) → { stem: 'alternar', confidence: 0.85 }\n */\nexport interface MorphologicalNormalizer {\n  /** Language code this normalizer handles */\n  readonly language: string;\n\n  /**\n   * Normalize a word to its canonical stem form.\n   *\n   * @param word - The word to normalize\n   * @returns Normalization result with stem and confidence\n   */\n  normalize(word: string): NormalizationResult;\n\n  /**\n   * Check if a word appears to be a verb form that can be normalized.\n   * Optional optimization to skip normalization for non-verb tokens.\n   *\n   * @param word - The word to check\n   * @returns true if the word might be a normalizable verb form\n   */\n  isNormalizable?(word: string): boolean;\n}\n\n/**\n * Configuration for suffix-based normalization rules.\n * Used by agglutinative languages (Japanese, Korean, Turkish).\n */\nexport interface SuffixRule {\n  /** The suffix pattern to match */\n  readonly pattern: string;\n\n  /** Confidence when this suffix is stripped */\n  readonly confidence: number;\n\n  /** What to replace the suffix with (empty string for simple removal) */\n  readonly replacement?: string;\n\n  /** Conjugation type this suffix indicates */\n  readonly conjugationType?: ConjugationType;\n\n  /** Minimum stem length after stripping (to avoid over-stripping) */\n  readonly minStemLength?: number;\n}\n\n/**\n * Configuration for prefix-based normalization rules.\n * Used primarily by Arabic for article/conjunction prefixes.\n */\nexport interface PrefixRule {\n  /** The prefix pattern to match */\n  readonly pattern: string;\n\n  /** Confidence penalty when this prefix is stripped */\n  readonly confidencePenalty: number;\n\n  /** What the prefix indicates (for metadata) */\n  readonly prefixType?: 'article' | 'conjunction' | 'preposition' | 'verb-marker';\n\n  /** Minimum remaining characters after stripping (to avoid over-stripping) */\n  readonly minRemaining?: number;\n}\n\n/**\n * Helper to create a \"no change\" normalization result.\n */\nexport function noChange(word: string): NormalizationResult {\n  return { stem: word, confidence: 1.0 };\n}\n\n/**\n * Helper to create a normalization result with metadata.\n */\nexport function normalized(\n  stem: string,\n  confidence: number,\n  metadata?: NormalizationMetadata\n): NormalizationResult {\n  if (metadata) {\n    return { stem, confidence, metadata };\n  }\n  return { stem, confidence };\n}\n"],"mappings":";AAuCO,IAAM,kBAAN,MAA6C;AAAA,EAKlD,YAAY,QAAyB,UAAkB;AAFvD,SAAQ,MAAc;AAGpB,SAAK,SAAS;AACd,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,KAAK,SAAiB,GAAyB;AAC7C,UAAM,QAAQ,KAAK,MAAM;AACzB,QAAI,QAAQ,KAAK,SAAS,KAAK,OAAO,QAAQ;AAC5C,aAAO;AAAA,IACT;AACA,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA,EAEA,UAAyB;AACvB,QAAI,KAAK,QAAQ,GAAG;AAClB,YAAM,IAAI,MAAM,gCAAgC;AAAA,IAClD;AACA,WAAO,KAAK,OAAO,KAAK,KAAK;AAAA,EAC/B;AAAA,EAEA,UAAmB;AACjB,WAAO,KAAK,OAAO,KAAK,OAAO;AAAA,EACjC;AAAA,EAEA,OAAmB;AACjB,WAAO,EAAE,UAAU,KAAK,IAAI;AAAA,EAC9B;AAAA,EAEA,MAAM,MAAwB;AAC5B,SAAK,MAAM,KAAK;AAAA,EAClB;AAAA,EAEA,WAAmB;AACjB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA,EAKA,YAA6B;AAC3B,WAAO,KAAK,OAAO,MAAM,KAAK,GAAG;AAAA,EACnC;AAAA;AAAA;AAAA;AAAA,EAKA,UAAU,WAA+D;AACvE,UAAM,SAA0B,CAAC;AACjC,WAAO,CAAC,KAAK,QAAQ,KAAK,UAAU,KAAK,KAAK,CAAE,GAAG;AACjD,aAAO,KAAK,KAAK,QAAQ,CAAC;AAAA,IAC5B;AACA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKA,UAAU,WAAoD;AAC5D,WAAO,CAAC,KAAK,QAAQ,KAAK,UAAU,KAAK,KAAK,CAAE,GAAG;AACjD,WAAK,QAAQ;AAAA,IACf;AAAA,EACF;AACF;AASO,SAAS,eAAe,OAAe,KAA6B;AACzE,SAAO,EAAE,OAAO,IAAI;AACtB;AA4CO,SAAS,YACd,eACA,MACA,UACA,qBACe;AAEf,MAAI,OAAO,kBAAkB,UAAU;AACrC,UAAM,EAAE,OAAAA,QAAO,MAAAC,OAAM,UAAAC,WAAU,YAAAC,aAAY,MAAM,gBAAgB,SAAS,IAAI;AAC9E,WAAO;AAAA,MACL,OAAAH;AAAA,MACA,MAAAC;AAAA,MACA,UAAAC;AAAA,MACA,GAAIC,gBAAe,UAAa,EAAE,YAAAA,YAAW;AAAA,MAC7C,GAAI,SAAS,UAAa,EAAE,KAAK;AAAA,MACjC,GAAI,mBAAmB,UAAa,EAAE,eAAe;AAAA,MACrD,GAAI,aAAa,UAAa,EAAE,SAAS;AAAA,IAC3C;AAAA,EACF;AAGA,QAAM,QAAQ;AACd,MAAI,CAAC,QAAQ,CAAC,UAAU;AACtB,UAAM,IAAI,MAAM,mDAAmD;AAAA,EACrE;AAEA,MAAI,OAAO,wBAAwB,UAAU;AAC3C,WAAO,EAAE,OAAO,MAAM,UAAU,YAAY,oBAAoB;AAAA,EAClE;AAGA,MAAI,qBAAqB;AACvB,UAAM,EAAE,YAAAA,aAAY,MAAM,gBAAgB,SAAS,IAAI;AACvD,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA,GAAIA,gBAAe,UAAa,EAAE,YAAAA,YAAW;AAAA,MAC7C,GAAI,SAAS,UAAa,EAAE,KAAK;AAAA,MACjC,GAAI,mBAAmB,UAAa,EAAE,eAAe;AAAA,MACrD,GAAI,aAAa,UAAa,EAAE,SAAS;AAAA,IAC3C;AAAA,EACF;AAEA,SAAO,EAAE,OAAO,MAAM,SAAS;AACjC;AAKO,SAAS,aAAa,MAAuB;AAClD,SAAO,KAAK,KAAK,IAAI;AACvB;AAMO,SAAS,gBAAgB,MAAuB;AACrD,SACE,SAAS,OAAO,SAAS,OAAO,SAAS,OAAO,SAAS,OAAO,SAAS,OAAO,SAAS;AAE7F;AAKO,SAAS,QAAQ,MAAuB;AAC7C,SAAO,SAAS,OAAO,SAAS,OAAO,SAAS,OAAO,SAAS,YAAO,SAAS;AAClF;AAKO,SAAS,QAAQ,MAAuB;AAC7C,SAAO,KAAK,KAAK,IAAI;AACvB;AAKO,SAAS,cAAc,MAAuB;AACnD,SAAO,WAAW,KAAK,IAAI;AAC7B;AAKO,SAAS,sBAAsB,MAAuB;AAC3D,SAAO,gBAAgB,KAAK,IAAI;AAClC;;;ACtNO,SAAS,mBAAmB,OAAe,UAAiC;AACjF,MAAI,YAAY,MAAM,OAAQ,QAAO;AAErC,QAAM,OAAO,MAAM,QAAQ;AAC3B,MAAI,CAAC,gBAAgB,IAAI,EAAG,QAAO;AAEnC,MAAI,MAAM;AACV,MAAI,WAAW;AAGf,MAAI,SAAS,OAAO,SAAS,KAAK;AAEhC,gBAAY,MAAM,KAAK;AACvB,WAAO,MAAM,MAAM,UAAU,sBAAsB,MAAM,GAAG,CAAC,GAAG;AAC9D,kBAAY,MAAM,KAAK;AAAA,IACzB;AAEA,QAAI,SAAS,UAAU,EAAG,QAAO;AAIjC,QAAI,MAAM,MAAM,UAAU,MAAM,GAAG,MAAM,OAAO,SAAS,KAAK;AAE5D,YAAM,cAAc,MAAM;AAC1B,UAAI,YAAY;AAChB,aAAO,YAAY,MAAM,UAAU,sBAAsB,MAAM,SAAS,CAAC,GAAG;AAC1E;AAAA,MACF;AAEA,UAAI,YAAY,MAAM,UAAU,MAAM,SAAS,MAAM,KAAK;AACxD,eAAO;AAAA,MACT;AAAA,IACF;AAAA,EACF,WAAW,SAAS,KAAK;AAGvB,QAAI,QAAQ;AACZ,QAAI,UAAU;AACd,QAAI,YAA2B;AAC/B,QAAI,UAAU;AAEd,gBAAY,MAAM,KAAK;AAEvB,WAAO,MAAM,MAAM,UAAU,QAAQ,GAAG;AACtC,YAAM,IAAI,MAAM,GAAG;AACnB,kBAAY;AAEZ,UAAI,SAAS;AAEX,kBAAU;AAAA,MACZ,WAAW,MAAM,MAAM;AAErB,kBAAU;AAAA,MACZ,WAAW,SAAS;AAElB,YAAI,MAAM,WAAW;AACnB,oBAAU;AACV,sBAAY;AAAA,QACd;AAAA,MACF,OAAO;AAEL,YAAI,MAAM,OAAO,MAAM,OAAO,MAAM,KAAK;AACvC,oBAAU;AACV,sBAAY;AAAA,QACd,WAAW,MAAM,KAAK;AACpB;AAAA,QACF,WAAW,MAAM,KAAK;AACpB;AAAA,QACF;AAAA,MACF;AACA;AAAA,IACF;AACA,QAAI,UAAU,EAAG,QAAO;AAAA,EAC1B,WAAW,SAAS,KAAK;AAEvB,gBAAY,MAAM,KAAK;AACvB,WAAO,MAAM,MAAM,UAAU,sBAAsB,MAAM,GAAG,CAAC,GAAG;AAC9D,kBAAY,MAAM,KAAK;AAAA,IACzB;AACA,QAAI,SAAS,UAAU,EAAG,QAAO;AAAA,EACnC,WAAW,SAAS,KAAK;AAEvB,gBAAY,MAAM,KAAK;AACvB,WAAO,MAAM,MAAM,UAAU,sBAAsB,MAAM,GAAG,CAAC,GAAG;AAC9D,kBAAY,MAAM,KAAK;AAAA,IACzB;AACA,QAAI,SAAS,UAAU,EAAG,QAAO;AAAA,EACnC,WAAW,SAAS,KAAK;AASvB,gBAAY,MAAM,KAAK;AAGvB,QAAI,OAAO,MAAM,UAAU,CAAC,cAAc,MAAM,GAAG,CAAC,EAAG,QAAO;AAG9D,WAAO,MAAM,MAAM,UAAU,sBAAsB,MAAM,GAAG,CAAC,GAAG;AAC9D,kBAAY,MAAM,KAAK;AAAA,IACzB;AAIA,WAAO,MAAM,MAAM,QAAQ;AACzB,YAAM,UAAU,MAAM,GAAG;AAEzB,UAAI,YAAY,KAAK;AAEnB,oBAAY,MAAM,KAAK;AACvB,YAAI,OAAO,MAAM,UAAU,CAAC,sBAAsB,MAAM,GAAG,CAAC,GAAG;AAC7D,iBAAO;AAAA,QACT;AACA,eAAO,MAAM,MAAM,UAAU,sBAAsB,MAAM,GAAG,CAAC,GAAG;AAC9D,sBAAY,MAAM,KAAK;AAAA,QACzB;AAAA,MACF,WAAW,YAAY,KAAK;AAE1B,oBAAY,MAAM,KAAK;AACvB,YAAI,OAAO,MAAM,UAAU,CAAC,sBAAsB,MAAM,GAAG,CAAC,GAAG;AAC7D,iBAAO;AAAA,QACT;AACA,eAAO,MAAM,MAAM,UAAU,sBAAsB,MAAM,GAAG,CAAC,GAAG;AAC9D,sBAAY,MAAM,KAAK;AAAA,QACzB;AAAA,MACF,WAAW,YAAY,KAAK;AAG1B,YAAI,QAAQ;AACZ,YAAI,UAAU;AACd,YAAI,YAA2B;AAC/B,YAAI,UAAU;AAEd,oBAAY,MAAM,KAAK;AAEvB,eAAO,MAAM,MAAM,UAAU,QAAQ,GAAG;AACtC,gBAAM,IAAI,MAAM,GAAG;AACnB,sBAAY;AAEZ,cAAI,SAAS;AACX,sBAAU;AAAA,UACZ,WAAW,MAAM,MAAM;AACrB,sBAAU;AAAA,UACZ,WAAW,SAAS;AAClB,gBAAI,MAAM,WAAW;AACnB,wBAAU;AACV,0BAAY;AAAA,YACd;AAAA,UACF,OAAO;AACL,gBAAI,MAAM,OAAO,MAAM,OAAO,MAAM,KAAK;AACvC,wBAAU;AACV,0BAAY;AAAA,YACd,WAAW,MAAM,KAAK;AACpB;AAAA,YACF,WAAW,MAAM,KAAK;AACpB;AAAA,YACF;AAAA,UACF;AACA;AAAA,QACF;AACA,YAAI,UAAU,EAAG,QAAO;AAAA,MAC1B,OAAO;AAEL;AAAA,MACF;AAAA,IACF;AAGA,WAAO,MAAM,MAAM,UAAU,aAAa,MAAM,GAAG,CAAC,GAAG;AACrD,kBAAY,MAAM,KAAK;AAAA,IACzB;AAGA,QAAI,MAAM,MAAM,UAAU,MAAM,GAAG,MAAM,KAAK;AAC5C,kBAAY,MAAM,KAAK;AAEvB,aAAO,MAAM,MAAM,UAAU,aAAa,MAAM,GAAG,CAAC,GAAG;AACrD,oBAAY,MAAM,KAAK;AAAA,MACzB;AAAA,IACF;AAGA,QAAI,OAAO,MAAM,UAAU,MAAM,GAAG,MAAM,IAAK,QAAO;AACtD,gBAAY,MAAM,KAAK;AAAA,EACzB;AAEA,SAAO,YAAY;AACrB;AAeO,SAAS,mBAAmB,OAAe,KAAsB;AACtE,MAAI,OAAO,MAAM,UAAU,MAAM,GAAG,MAAM,IAAK,QAAO;AAGtD,MAAI,MAAM,KAAK,MAAM,OAAQ,QAAO;AACpC,QAAM,WAAW,MAAM,MAAM,CAAC,EAAE,YAAY;AAC5C,MAAI,aAAa,IAAK,QAAO;AAG7B,MAAI,MAAM,KAAK,MAAM,OAAQ,QAAO;AACpC,QAAM,SAAS,MAAM,MAAM,CAAC;AAC5B,SAAO,aAAa,MAAM,KAAK,WAAW,OAAO,CAAC,sBAAsB,MAAM;AAChF;AAQO,SAAS,qBAAqB,OAAe,UAAiC;AACnF,MAAI,YAAY,MAAM,OAAQ,QAAO;AAErC,QAAM,YAAY,MAAM,QAAQ;AAChC,MAAI,CAAC,QAAQ,SAAS,EAAG,QAAO;AAGhC,MAAI,cAAc,OAAO,mBAAmB,OAAO,QAAQ,GAAG;AAC5D,WAAO;AAAA,EACT;AAGA,QAAM,gBAAwC;AAAA,IAC5C,KAAK;AAAA,IACL,KAAK;AAAA,IACL,KAAK;AAAA,IACL,UAAK;AAAA,EACP;AAEA,QAAM,aAAa,cAAc,SAAS;AAC1C,MAAI,CAAC,WAAY,QAAO;AAExB,MAAI,MAAM,WAAW;AACrB,MAAI,UAAU;AACd,MAAI,UAAU;AAEd,SAAO,MAAM,MAAM,QAAQ;AACzB,UAAM,OAAO,MAAM,GAAG;AACtB,eAAW;AAEX,QAAI,SAAS;AACX,gBAAU;AAAA,IACZ,WAAW,SAAS,MAAM;AACxB,gBAAU;AAAA,IACZ,WAAW,SAAS,YAAY;AAE9B,aAAO;AAAA,IACT;AACA;AAAA,EACF;AAGA,SAAO;AACT;AAUO,SAAS,WAAW,OAAe,KAAsB;AAC9D,MAAI,OAAO,MAAM,OAAQ,QAAO;AAEhC,QAAM,OAAO,MAAM,GAAG;AACtB,QAAM,OAAO,MAAM,MAAM,CAAC,KAAK;AAC/B,QAAM,QAAQ,MAAM,MAAM,CAAC,KAAK;AAIhC,MAAI,SAAS,OAAO,SAAS,OAAO,iBAAiB,KAAK,IAAI,GAAG;AAC/D,WAAO;AAAA,EACT;AAGA,MAAI,SAAS,OAAO,SAAS,OAAO,WAAW,KAAK,KAAK,GAAG;AAC1D,WAAO;AAAA,EACT;AAGA,MAAI,SAAS,QAAQ,SAAS,OAAQ,SAAS,OAAO,UAAU,MAAO;AACrE,WAAO;AAAA,EACT;AAGA,QAAM,QAAQ,MAAM,MAAM,KAAK,MAAM,CAAC,EAAE,YAAY;AACpD,MAAI,MAAM,WAAW,SAAS,KAAK,MAAM,WAAW,UAAU,GAAG;AAC/D,WAAO;AAAA,EACT;AAEA,SAAO;AACT;AAUO,SAAS,WAAW,OAAe,UAAiC;AACzE,MAAI,CAAC,WAAW,OAAO,QAAQ,EAAG,QAAO;AAEzC,MAAI,MAAM;AACV,MAAI,MAAM;AAIV,QAAM,WAAW;AAEjB,SAAO,MAAM,MAAM,QAAQ;AACzB,UAAM,OAAO,MAAM,GAAG;AAGtB,QAAI,SAAS,KAAK;AAGhB,UAAI,IAAI,SAAS,KAAK,iBAAiB,KAAK,GAAG,GAAG;AAEhD,eAAO;AACP;AAEA,eAAO,MAAM,MAAM,UAAU,gBAAgB,KAAK,MAAM,GAAG,CAAC,GAAG;AAC7D,iBAAO,MAAM,KAAK;AAAA,QACpB;AAAA,MACF;AAEA;AAAA,IACF;AAEA,QAAI,SAAS,KAAK,IAAI,GAAG;AACvB,aAAO;AACP;AAAA,IACF,OAAO;AACL;AAAA,IACF;AAAA,EACF;AAGA,MAAI,IAAI,SAAS,EAAG,QAAO;AAE3B,SAAO;AACT;AAUO,SAAS,cAAc,OAAe,UAAiC;AAC5E,MAAI,YAAY,MAAM,OAAQ,QAAO;AAErC,QAAM,OAAO,MAAM,QAAQ;AAC3B,MAAI,CAAC,QAAQ,IAAI,KAAK,SAAS,OAAO,SAAS,IAAK,QAAO;AAE3D,MAAI,MAAM;AACV,MAAI,SAAS;AAGb,MAAI,MAAM,GAAG,MAAM,OAAO,MAAM,GAAG,MAAM,KAAK;AAC5C,cAAU,MAAM,KAAK;AAAA,EACvB;AAGA,MAAI,OAAO,MAAM,UAAU,CAAC,QAAQ,MAAM,GAAG,CAAC,GAAG;AAC/C,WAAO;AAAA,EACT;AAGA,SAAO,MAAM,MAAM,UAAU,QAAQ,MAAM,GAAG,CAAC,GAAG;AAChD,cAAU,MAAM,KAAK;AAAA,EACvB;AAGA,MAAI,MAAM,MAAM,UAAU,MAAM,GAAG,MAAM,KAAK;AAC5C,cAAU,MAAM,KAAK;AACrB,WAAO,MAAM,MAAM,UAAU,QAAQ,MAAM,GAAG,CAAC,GAAG;AAChD,gBAAU,MAAM,KAAK;AAAA,IACvB;AAAA,EACF;AAGA,MAAI,MAAM,MAAM,QAAQ;AACtB,UAAM,SAAS,MAAM,MAAM,KAAK,MAAM,CAAC;AACvC,QAAI,WAAW,MAAM;AACnB,gBAAU;AAAA,IACZ,WAAW,MAAM,GAAG,MAAM,OAAO,MAAM,GAAG,MAAM,OAAO,MAAM,GAAG,MAAM,KAAK;AACzE,gBAAU,MAAM,GAAG;AAAA,IACrB;AAAA,EACF;AAEA,SAAO;AACT;;;ACtbO,IAAM,oBAAoB;AAAA;AAAA,EAE/B;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAEA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAEA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAKO,IAAM,oBAAN,MAAkD;AAAA,EAGvD,YAAoB,YAAsB,mBAAmB;AAAzC;AAFpB,SAAS,OAAO;AAId,SAAK,YAAY,CAAC,GAAG,SAAS,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,SAAS,EAAE,MAAM;AAAA,EACpE;AAAA,EAEA,WAAW,OAAe,UAA2B;AACnD,WAAO,KAAK,UAAU,KAAK,QAAM,MAAM,WAAW,IAAI,QAAQ,CAAC;AAAA,EACjE;AAAA,EAEA,QAAQ,OAAe,UAA2C;AAEhE,eAAW,MAAM,KAAK,WAAW;AAC/B,UAAI,MAAM,WAAW,IAAI,QAAQ,GAAG;AAClC,eAAO;AAAA,UACL,OAAO;AAAA,UACP,QAAQ,GAAG;AAAA,QACb;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AACF;;;AC9DO,IAAM,sBAAsB;AAK5B,IAAM,uBAAN,MAAqD;AAAA,EAG1D,YAAoB,cAAsB,qBAAqB;AAA3C;AAFpB,SAAS,OAAO;AAAA,EAEgD;AAAA,EAEhE,WAAW,OAAe,UAA2B;AACnD,WAAO,KAAK,YAAY,SAAS,MAAM,QAAQ,CAAC;AAAA,EAClD;AAAA,EAEA,QAAQ,OAAe,UAA2C;AAChE,UAAM,OAAO,MAAM,QAAQ;AAE3B,QAAI,KAAK,YAAY,SAAS,IAAI,GAAG;AACnC,aAAO;AAAA,QACL,OAAO;AAAA,QACP,QAAQ;AAAA,MACV;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AACF;;;AC4CO,IAAM,yBAAN,MAAuD;AAAA,EAAvD;AACL,SAAS,OAAO;AAAA;AAAA,EAEhB,WAAW,OAAe,UAA2B;AACnD,UAAM,OAAO,MAAM,QAAQ;AAC3B,WACE,SAAS,OACT,SAAS,OACT,SAAS,OACT,SAAS;AAAA,IACT,SAAS;AAAA,EAEb;AAAA,EAEA,QAAQ,OAAe,UAA2C;AAChE,UAAM,QAAQ,MAAM,QAAQ;AAG5B,QAAI,UAAU,UAAU;AACtB,UAAIC,UAAS;AACb,aAAO,WAAWA,UAAS,MAAM,QAAQ;AACvC,YAAI,MAAM,WAAWA,OAAM,MAAM,UAAU;AACzC,UAAAA;AACA,iBAAO,EAAE,OAAO,MAAM,UAAU,UAAU,WAAWA,OAAM,GAAG,QAAAA,QAAO;AAAA,QACvE;AACA,QAAAA;AAAA,MACF;AACA,aAAO;AAAA,IACT;AAGA,QAAI,UAAU,UAAU;AACtB,UAAIA,UAAS;AACb,aAAO,WAAWA,UAAS,MAAM,QAAQ;AACvC,YAAI,MAAM,WAAWA,OAAM,MAAM,UAAU;AACzC,UAAAA;AACA,iBAAO,EAAE,OAAO,MAAM,UAAU,UAAU,WAAWA,OAAM,GAAG,QAAAA,QAAO;AAAA,QACvE;AACA,QAAAA;AAAA,MACF;AACA,aAAO;AAAA,IACT;AAGA,QAAI,SAAS;AACb,QAAI,UAAU;AAEd,WAAO,WAAW,SAAS,MAAM,QAAQ;AACvC,YAAM,OAAO,MAAM,WAAW,MAAM;AAEpC,UAAI,SAAS;AACX,kBAAU;AACV;AACA;AAAA,MACF;AAEA,UAAI,SAAS,MAAM;AACjB,kBAAU;AACV;AACA;AAAA,MACF;AAEA,UAAI,SAAS,OAAO;AAClB;AACA,eAAO;AAAA,UACL,OAAO,MAAM,UAAU,UAAU,WAAW,MAAM;AAAA,UAClD;AAAA,QACF;AAAA,MACF;AAEA;AAAA,IACF;AAGA,WAAO;AAAA,EACT;AACF;AAKO,IAAM,kBAAN,MAAgD;AAAA,EAAhD;AACL,SAAS,OAAO;AAAA;AAAA,EAEhB,WAAW,OAAe,UAA2B;AACnD,WAAO,KAAK,KAAK,MAAM,QAAQ,CAAC;AAAA,EAClC;AAAA,EAEA,QAAQ,OAAe,UAA2C;AAChE,QAAI,SAAS;AACb,QAAI,aAAa;AAEjB,WAAO,WAAW,SAAS,MAAM,QAAQ;AACvC,YAAM,OAAO,MAAM,WAAW,MAAM;AAEpC,UAAI,KAAK,KAAK,IAAI,GAAG;AACnB;AAAA,MACF,WAAW,SAAS,OAAO,CAAC,YAAY;AACtC,qBAAa;AACb;AAAA,MACF,OAAO;AACL;AAAA,MACF;AAAA,IACF;AAEA,QAAI,WAAW,EAAG,QAAO;AAEzB,UAAM,WAAW,MAAM,UAAU,UAAU,WAAW,MAAM;AAC5D,UAAM,WAAW,WAAW;AAG5B,QAAI,WAAW,MAAM,QAAQ;AAC3B,YAAM,YAAY,MAAM,MAAM,QAAQ;AAGtC,YAAM,gBAAuD;AAAA,QAC3D,EAAE,SAAS,gBAAM,QAAQ,KAAK;AAAA;AAAA,QAC9B,EAAE,SAAS,gBAAM,QAAQ,IAAI;AAAA;AAAA,QAC7B,EAAE,SAAS,gBAAM,QAAQ,IAAI;AAAA;AAAA,QAC7B,EAAE,SAAS,sBAAO,QAAQ,KAAK;AAAA;AAAA,QAC/B,EAAE,SAAS,gBAAM,QAAQ,IAAI;AAAA;AAAA,MAC/B;AACA,iBAAW,QAAQ,eAAe;AAChC,YAAI,UAAU,WAAW,KAAK,OAAO,GAAG;AACtC,iBAAO;AAAA,YACL,OAAO,WAAW,KAAK;AAAA,YACvB,QAAQ,SAAS,KAAK,QAAQ;AAAA,YAC9B,UAAU,EAAE,aAAa,KAAK;AAAA,UAChC;AAAA,QACF;AAAA,MACF;AAGA,UAAI,UAAU,WAAW,IAAI,GAAG;AAC9B,eAAO;AAAA,UACL,OAAO,WAAW;AAAA,UAClB,QAAQ,SAAS;AAAA,UACjB,UAAU,EAAE,aAAa,KAAK;AAAA,QAChC;AAAA,MACF;AAGA,YAAM,iBAAwD;AAAA,QAC5D,EAAE,SAAS,UAAK,QAAQ,IAAI;AAAA;AAAA,QAC5B,EAAE,SAAS,UAAK,QAAQ,IAAI;AAAA;AAAA,MAC9B;AACA,iBAAW,QAAQ,gBAAgB;AACjC,YAAI,UAAU,WAAW,KAAK,OAAO,GAAG;AACtC,iBAAO;AAAA,YACL,OAAO,WAAW,KAAK;AAAA,YACvB,QAAQ,SAAS;AAAA,YACjB,UAAU,EAAE,aAAa,KAAK;AAAA,UAChC;AAAA,QACF;AAAA,MACF;AAGA,UAAI,qBAAqB,KAAK,SAAS,GAAG;AACxC,eAAO;AAAA,UACL,OAAO,WAAW,UAAU,CAAC;AAAA,UAC7B,QAAQ,SAAS;AAAA,UACjB,UAAU,EAAE,aAAa,KAAK;AAAA,QAChC;AAAA,MACF;AAAA,IACF;AAEA,WAAO,EAAE,OAAO,UAAU,OAAO;AAAA,EACnC;AACF;AAKO,IAAM,sBAAN,MAAoD;AAAA,EAApD;AACL,SAAS,OAAO;AAAA;AAAA,EAEhB,WAAW,OAAe,UAA2B;AACnD,WAAO,YAAY,KAAK,MAAM,QAAQ,CAAC;AAAA,EACzC;AAAA,EAEA,QAAQ,OAAe,UAA2C;AAChE,QAAI,SAAS;AAEb,WAAO,WAAW,SAAS,MAAM,QAAQ;AACvC,YAAM,OAAO,MAAM,WAAW,MAAM;AACpC,UAAI,eAAe,KAAK,IAAI,GAAG;AAC7B;AAAA,MACF,OAAO;AACL;AAAA,MACF;AAAA,IACF;AAEA,WAAO,SAAS,IACZ;AAAA,MACE,OAAO,MAAM,UAAU,UAAU,WAAW,MAAM;AAAA,MAClD;AAAA,IACF,IACA;AAAA,EACN;AACF;AASO,IAAM,6BAAN,MAA2D;AAAA,EAA3D;AACL,SAAS,OAAO;AAAA;AAAA,EAEhB,WAAW,OAAe,UAA2B;AACnD,UAAM,OAAO,MAAM,WAAW,QAAQ;AAEtC,QAAI,OAAO,IAAM,QAAO;AAExB,WAAO,SAAS,KAAK,MAAM,QAAQ,CAAC;AAAA,EACtC;AAAA,EAEA,QAAQ,OAAe,UAA2C;AAChE,QAAI,SAAS;AAEb,WAAO,WAAW,SAAS,MAAM,QAAQ;AACvC,YAAM,OAAO,MAAM,WAAW,MAAM;AAEpC,UAAI,qBAAqB,KAAK,IAAI,GAAG;AACnC;AAAA,MACF,OAAO;AACL;AAAA,MACF;AAAA,IACF;AAEA,WAAO,SAAS,IAAI,EAAE,OAAO,MAAM,UAAU,UAAU,WAAW,MAAM,GAAG,OAAO,IAAI;AAAA,EACxF;AACF;AAuFO,SAAS,wBACd,WACoC;AACpC,SAAO,gBAAgB,aAAa,OAAO,UAAU,eAAe;AACtE;AAMO,SAAS,uBAAuB,WAOlB;AACnB,QAAM,MAAwB;AAAA,IAC5B,UAAU,UAAU;AAAA,IACpB,WAAW,UAAU;AAAA,IACrB,eAAe,UAAU,cAAc,KAAK,SAAS;AAAA,IACrD,WAAW,UAAU,UAAU,KAAK,SAAS;AAAA,IAC7C,gBAAgB,UAAU,eAAe,KAAK,SAAS;AAAA,EACzD;AAEA,MAAI,UAAU,YAAY;AACxB,WAAO,EAAE,GAAG,KAAK,YAAY,UAAU,WAAW;AAAA,EACpD;AAEA,SAAO;AACT;;;AC1YO,SAAS,uBAAyC;AACvD,SAAO;AAAA,IACL,IAAI,uBAAuB;AAAA;AAAA,IAC3B,IAAI,gBAAgB;AAAA;AAAA,IACpB,IAAI,kBAAkB;AAAA;AAAA,IACtB,IAAI,qBAAqB;AAAA;AAAA,IACzB,IAAI,oBAAoB;AAAA;AAAA,IACxB,IAAI,2BAA2B;AAAA;AAAA,EACjC;AACF;AAcO,SAAS,sBAEd,WAAiB;AACjB,YAAU,mBAAmB,qBAAqB,CAAC;AACnD,SAAO;AACT;;;ACrCO,SAAS,6BACd,QAC2B;AAC3B,SAAO,CAAC,SAA0B;AAChC,UAAM,OAAO,KAAK,WAAW,CAAC;AAC9B,WAAO,OAAO,KAAK,CAAC,CAAC,OAAO,GAAG,MAAM,QAAQ,SAAS,QAAQ,GAAG;AAAA,EACnE;AACF;AASO,SAAS,sBACX,aACwB;AAC3B,SAAO,CAAC,SAA0B,YAAY,KAAK,QAAM,GAAG,IAAI,CAAC;AACnE;AAuBO,SAAS,2BAA2B,eAA6C;AACtF,QAAM,WAAW,CAAC,SAA0B,cAAc,KAAK,IAAI;AACnE,QAAM,mBAAmB,CAAC,SAA0B,SAAS,IAAI,KAAK,UAAU,KAAK,IAAI;AACzF,SAAO,EAAE,UAAU,iBAAiB;AACtC;;;AC9CA,IAAM,gCAAgC,IAAI,IAAI,iBAAiB;AAyCxD,IAAe,iBAAf,MAAe,eAA2C;AAAA,EAA1D;AAQL;AAAA,SAAU,kBAAkC,CAAC;AAG7C;AAAA,SAAU,oBAA+C,oBAAI,IAAI;AAMjE;AAAA;AAAA;AAAA;AAAA,SAAU,aAA+B,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAU1C,SAAS,OAA4B;AACnC,QAAI,KAAK,kBAAkB,GAAG;AAC5B,aAAO,KAAK,uBAAuB,KAAK;AAAA,IAC1C;AAGA,UAAM,IAAI;AAAA,MACR,GAAG,KAAK,YAAY,IAAI;AAAA,IAE1B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWA,kBAAkB,WAAiC;AACjD,QAAI,wBAAwB,SAAS,GAAG;AACtC,gBAAU,WAAW,uBAAuB,IAAW,CAAC;AAAA,IAC1D;AACA,SAAK,WAAW,KAAK,SAAS;AAAA,EAChC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,mBAAmB,YAAoC;AACrD,eAAW,aAAa,YAAY;AAClC,WAAK,kBAAkB,SAAS;AAAA,IAClC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,kBAAwB;AACtB,SAAK,aAAa,CAAC;AAAA,EACrB;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,oBAA6B;AACrC,WAAO,KAAK,WAAW,SAAS;AAAA,EAClC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASU,uBAAuB,OAA4B;AAC3D,UAAM,SAA0B,CAAC;AACjC,QAAI,MAAM;AAEV,WAAO,MAAM,MAAM,QAAQ;AAEzB,aAAO,MAAM,MAAM,UAAU,aAAa,MAAM,GAAG,CAAC,GAAG;AACrD;AAAA,MACF;AACA,UAAI,OAAO,MAAM,OAAQ;AAGzB,UAAI,YAAY;AAChB,iBAAW,aAAa,KAAK,YAAY;AACvC,YAAI,UAAU,WAAW,OAAO,GAAG,GAAG;AACpC,gBAAM,SAAS,UAAU,QAAQ,OAAO,GAAG;AAC3C,cAAI,QAAQ;AAEV,kBAAMC,cAAa,OAAO,UAAU;AACpC,kBAAM,OAAO,OAAO,UAAU;AAC9B,kBAAM,iBAAiB,OAAO,UAAU;AAGxC,kBAAM,gBAAyC,CAAC;AAChD,gBAAI,OAAO,UAAU;AACnB,yBAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,OAAO,QAAQ,GAAG;AAC1D,oBAAI,QAAQ,gBAAgB,QAAQ,UAAU,QAAQ,kBAAkB;AACtE,gCAAc,GAAG,IAAI;AAAA,gBACvB;AAAA,cACF;AAAA,YACF;AAEA,kBAAM,UAA8B,CAAC;AACrC,gBAAIA,YAAY,SAAQ,aAAaA;AACrC,gBAAI,KAAM,SAAQ,OAAO;AACzB,gBAAI,mBAAmB,OAAW,SAAQ,iBAAiB;AAC3D,gBAAI,OAAO,KAAK,aAAa,EAAE,SAAS,EAAG,SAAQ,WAAW;AAE9D,mBAAO;AAAA,cACL;AAAA,gBACE,OAAO;AAAA,gBACP,KAAK,cAAc,OAAO,KAAK;AAAA,gBAC/B,eAAe,KAAK,MAAM,OAAO,MAAM;AAAA,gBACvC,OAAO,KAAK,OAAO,EAAE,SAAS,IAAI,UAAU;AAAA,cAC9C;AAAA,YACF;AACA,mBAAO,OAAO;AACd,wBAAY;AACZ;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAGA,UAAI,CAAC,WAAW;AACd,cAAM,OAAO,MAAM,GAAG;AACtB,cAAM,OAAO,KAAK,oBAAoB,IAAI;AAC1C,eAAO,KAAK,YAAY,MAAM,MAAM,eAAe,KAAK,MAAM,CAAC,CAAC,CAAC;AACjE;AAAA,MACF;AAAA,IACF;AAEA,WAAO,IAAI,gBAAgB,QAAQ,KAAK,QAAQ;AAAA,EAClD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASU,oBAAoB,MAAyB;AACrD,QAAI,YAAY,SAAS,IAAI,EAAG,QAAO;AACvC,QAAI,aAAa,SAAS,IAAI,EAAG,QAAO;AACxC,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUU,kBAAkB,OAAe,KAAa,QAAkC;AACxF,QAAI,MAAM,GAAG,MAAM,IAAK,QAAO;AAE/B,UAAM,YAAY,OAAO,OAAO,SAAS,CAAC;AAE1C,UAAM,sBAAsB,aAAa,UAAU,SAAS,MAAM;AAClE,UAAM,mBACJ,aACA,CAAC,wBACA,UAAU,SAAS,gBAClB,UAAU,SAAS,aACnB,UAAU,SAAS;AAEvB,QAAI,kBAAkB;AACpB,aAAO,KAAK,YAAY,KAAK,YAAY,eAAe,KAAK,MAAM,CAAC,CAAC,CAAC;AACtE,aAAO;AAAA,IACT;AAGA,UAAM,cAAc,MAAM;AAC1B,QAAI,YAAY;AAChB,WAAO,YAAY,MAAM,UAAU,sBAAsB,MAAM,SAAS,CAAC,GAAG;AAC1E;AAAA,IACF;AACA,QAAI,YAAY,MAAM,UAAU,MAAM,SAAS,MAAM,KAAK;AACxD,aAAO,KAAK,YAAY,KAAK,YAAY,eAAe,KAAK,MAAM,CAAC,CAAC,CAAC;AACtE,aAAO;AAAA,IACT;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAeU,8BACR,SACA,SAAyB,CAAC,GACpB;AAEN,UAAM,aAAa,oBAAI,IAA0B;AAGjD,QAAI,QAAQ,UAAU;AACpB,iBAAW,CAACA,aAAY,WAAW,KAAK,OAAO,QAAQ,QAAQ,QAAQ,GAAG;AAExE,mBAAW,IAAI,YAAY,SAAS;AAAA,UAClC,QAAQ,YAAY;AAAA,UACpB,YAAY,YAAY,cAAcA;AAAA,QACxC,CAAC;AAGD,YAAI,YAAY,cAAc;AAC5B,qBAAW,OAAO,YAAY,cAAc;AAC1C,uBAAW,IAAI,KAAK;AAAA,cAClB,QAAQ;AAAA,cACR,YAAY,YAAY,cAAcA;AAAA,YACxC,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAGA,QAAI,QAAQ,YAAY;AACtB,iBAAW,CAACA,aAAY,MAAM,KAAK,OAAO,QAAQ,QAAQ,UAAU,GAAG;AACrE,mBAAW,IAAI,QAAQ,EAAE,QAAQ,YAAAA,YAAW,CAAC;AAAA,MAC/C;AAKA,iBAAW,aAAa,OAAO,KAAK,QAAQ,UAAU,GAAG;AACvD,YAAI,CAAC,WAAW,IAAI,SAAS,GAAG;AAC9B,qBAAW,IAAI,WAAW,EAAE,QAAQ,WAAW,YAAY,UAAU,CAAC;AAAA,QACxE;AAAA,MACF;AAAA,IACF;AAGA,QAAI,QAAQ,aAAa;AACvB,iBAAW,CAAC,MAAM,MAAM,KAAK,OAAO,QAAQ,QAAQ,WAAW,GAAG;AAChE,YAAI,OAAO,SAAS;AAClB,qBAAW,IAAI,OAAO,SAAS,EAAE,QAAQ,OAAO,SAAS,YAAY,KAAK,CAAC;AAAA,QAC7E;AACA,YAAI,OAAO,cAAc;AACvB,qBAAW,OAAO,OAAO,cAAc;AACrC,uBAAW,IAAI,KAAK,EAAE,QAAQ,KAAK,YAAY,KAAK,CAAC;AAAA,UACvD;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAGA,QAAI,QAAQ,YAAY,UAAU;AAChC,iBAAW,CAAC,QAAQA,WAAU,KAAK,OAAO,QAAQ,QAAQ,WAAW,QAAQ,GAAG;AAC9E,mBAAW,IAAI,QAAQ,EAAE,QAAQ,YAAAA,YAAW,CAAC;AAAA,MAC/C;AAAA,IACF;AAGA,eAAW,SAAS,QAAQ;AAC1B,iBAAW,IAAI,MAAM,QAAQ,KAAK;AAAA,IACpC;AAGA,SAAK,kBAAkB,MAAM,KAAK,WAAW,OAAO,CAAC,EAAE;AAAA,MACrD,CAAC,GAAG,MAAM,EAAE,OAAO,SAAS,EAAE,OAAO;AAAA,IACvC;AAIA,SAAK,oBAAoB,oBAAI,IAAI;AACjC,eAAW,WAAW,KAAK,iBAAiB;AAE1C,WAAK,kBAAkB,IAAI,QAAQ,OAAO,YAAY,GAAG,OAAO;AAGhE,YAAMA,cAAa,KAAK,iBAAiB,QAAQ,MAAM;AACvD,UAAIA,gBAAe,QAAQ,UAAU,CAAC,KAAK,kBAAkB,IAAIA,YAAW,YAAY,CAAC,GAAG;AAC1F,aAAK,kBAAkB,IAAIA,YAAW,YAAY,GAAG,OAAO;AAAA,MAC9D;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUU,iBAAiB,MAAsB;AAG/C,WAAO,KAAK,QAAQ,0BAA0B,EAAE;AAAA,EAClD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUU,kBAAkB,OAAe,KAAmC;AAC5E,eAAW,SAAS,KAAK,iBAAiB;AACxC,UAAI,MAAM,MAAM,GAAG,EAAE,WAAW,MAAM,MAAM,GAAG;AAC7C,eAAO;AAAA,UACL,MAAM;AAAA,UACN;AAAA,UACA,eAAe,KAAK,MAAM,MAAM,OAAO,MAAM;AAAA,UAC7C,MAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUU,eAAe,OAAe,KAAsB;AAC5D,UAAM,YAAY,MAAM,MAAM,GAAG;AACjC,WAAO,KAAK,gBAAgB,KAAK,WAAS,UAAU,WAAW,MAAM,MAAM,CAAC;AAAA,EAC9E;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASU,cAAc,QAA0C;AAChE,WAAO,KAAK,kBAAkB,IAAI,OAAO,YAAY,CAAC;AAAA,EACxD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASU,UAAU,QAAyB;AAC3C,WAAO,KAAK,kBAAkB,IAAI,OAAO,YAAY,CAAC;AAAA,EACxD;AAAA;AAAA;AAAA;AAAA,EAKA,cAAc,YAA2C;AACvD,SAAK,aAAa;AAAA,EACpB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUU,aAAa,MAA0C;AAC/D,QAAI,CAAC,KAAK,WAAY,QAAO;AAE7B,UAAM,SAAS,KAAK,WAAW,UAAU,IAAI;AAG7C,QAAI,OAAO,SAAS,QAAQ,OAAO,cAAc,KAAK;AACpD,aAAO;AAAA,IACT;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAkBU,qBACR,MACA,UACA,QACsB;AACtB,UAAM,SAAS,KAAK,aAAa,IAAI;AACrC,QAAI,CAAC,OAAQ,QAAO;AAGpB,UAAM,YAAY,KAAK,cAAc,OAAO,IAAI;AAChD,QAAI,CAAC,UAAW,QAAO;AAEvB,UAAM,eAAmC;AAAA,MACvC,YAAY,UAAU;AAAA,MACtB,MAAM,OAAO;AAAA,MACb,gBAAgB,OAAO;AAAA,IACzB;AACA,WAAO,YAAY,MAAM,WAAW,eAAe,UAAU,MAAM,GAAG,YAAY;AAAA,EACpF;AAAA;AAAA;AAAA;AAAA,EAKU,YAAY,OAAe,KAAmC;AACtE,UAAM,WAAW,mBAAmB,OAAO,GAAG;AAC9C,QAAI,UAAU;AACZ,aAAO,YAAY,UAAU,YAAY,eAAe,KAAK,MAAM,SAAS,MAAM,CAAC;AAAA,IACrF;AACA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,iBAAiB,OAAe,KAAmC;AAE3E,QAAI,MAAM,GAAG,MAAM,KAAK;AACtB,aAAO;AAAA,IACT;AAGA,UAAM,QAAQ,MACX,MAAM,GAAG,EACT,MAAM,gEAAgE;AACzE,QAAI,CAAC,OAAO;AACV,aAAO;AAAA,IACT;AAEA,UAAM,YAAY,MAAM,CAAC,EAAE,QAAQ,YAAY,EAAE;AACjD,UAAM,eAAe,UAAU,MAAM,CAAC,EAAE,MAAM,GAAG,EAAE,CAAC;AACpD,UAAM,QAAQ,MAAM,CAAC;AAGrB,UAAM,QAAQ;AAAA,MACZ;AAAA,MACA;AAAA,MACA,eAAe,KAAK,MAAM,UAAU,MAAM;AAAA,IAC5C;AAGA,WAAO;AAAA,MACL,GAAG;AAAA,MACH,UAAU;AAAA,QACR;AAAA,QACA,OAAO,QAAS,iBAAiB,UAAU,QAAQ,SAAS,OAAO,EAAE,IAAK;AAAA,MAC5E;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKU,UAAU,OAAe,KAAmC;AACpE,UAAM,UAAU,qBAAqB,OAAO,GAAG;AAC/C,QAAI,SAAS;AACX,aAAO,YAAY,SAAS,WAAW,eAAe,KAAK,MAAM,QAAQ,MAAM,CAAC;AAAA,IAClF;AACA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKU,UAAU,OAAe,KAAmC;AACpE,UAAM,SAAS,cAAc,OAAO,GAAG;AACvC,QAAI,QAAQ;AACV,aAAO,YAAY,QAAQ,WAAW,eAAe,KAAK,MAAM,OAAO,MAAM,CAAC;AAAA,IAChF;AACA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAsBU,iBACR,OACA,KACA,WACA,iBAAiB,OAC0B;AAC3C,QAAI,UAAU;AAGd,QAAI,gBAAgB;AAClB,aAAO,UAAU,MAAM,UAAU,aAAa,MAAM,OAAO,CAAC,GAAG;AAC7D;AAAA,MACF;AAAA,IACF;AAEA,UAAM,YAAY,MAAM,MAAM,OAAO;AAGrC,eAAW,QAAQ,WAAW;AAC5B,YAAM,YAAY,UAAU,MAAM,GAAG,KAAK,MAAM;AAChD,YAAM,UAAU,KAAK,kBACjB,UAAU,YAAY,MAAM,KAAK,QAAQ,YAAY,IACrD,cAAc,KAAK;AAEvB,UAAI,SAAS;AAEX,YAAI,KAAK,eAAe;AACtB,gBAAM,WAAW,UAAU,KAAK,MAAM,KAAK;AAC3C,cAAI,aAAa,KAAK,cAAe;AAAA,QACvC;AAGA,YAAI,KAAK,eAAe;AACtB,gBAAM,WAAW,UAAU,KAAK,MAAM,KAAK;AAC3C,cAAI,sBAAsB,QAAQ,EAAG;AAAA,QACvC;AAEA,eAAO,EAAE,QAAQ,KAAK,QAAQ,QAAQ,UAAU,KAAK,OAAO;AAAA,MAC9D;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWU,gBACR,OACA,UACA,YAAY,MAC+B;AAC3C,QAAI,MAAM;AACV,QAAI,SAAS;AAGb,QAAI,cAAc,MAAM,GAAG,MAAM,OAAO,MAAM,GAAG,MAAM,MAAM;AAC3D,gBAAU,MAAM,KAAK;AAAA,IACvB;AAGA,QAAI,OAAO,MAAM,UAAU,CAAC,QAAQ,MAAM,GAAG,CAAC,GAAG;AAC/C,aAAO;AAAA,IACT;AAGA,WAAO,MAAM,MAAM,UAAU,QAAQ,MAAM,GAAG,CAAC,GAAG;AAChD,gBAAU,MAAM,KAAK;AAAA,IACvB;AAGA,QAAI,MAAM,MAAM,UAAU,MAAM,GAAG,MAAM,KAAK;AAC5C,gBAAU,MAAM,KAAK;AACrB,aAAO,MAAM,MAAM,UAAU,QAAQ,MAAM,GAAG,CAAC,GAAG;AAChD,kBAAU,MAAM,KAAK;AAAA,MACvB;AAAA,IACF;AAEA,QAAI,CAAC,UAAU,WAAW,OAAO,WAAW,IAAK,QAAO;AAExD,WAAO,EAAE,QAAQ,QAAQ,IAAI;AAAA,EAC/B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAgBU,uBACR,OACA,KACA,iBACA,UAA6D,CAAC,GACxC;AACtB,UAAM,EAAE,YAAY,MAAM,iBAAiB,MAAM,IAAI;AAGrD,UAAM,aAAa,KAAK,gBAAgB,OAAO,KAAK,SAAS;AAC7D,QAAI,CAAC,WAAY,QAAO;AAExB,QAAI,EAAE,QAAQ,OAAO,IAAI;AAGzB,UAAM,WAAW,CAAC,GAAG,iBAAiB,GAAG,eAAc,mBAAmB;AAC1E,UAAM,YAAY,KAAK,iBAAiB,OAAO,QAAQ,UAAU,cAAc;AAE/E,QAAI,WAAW;AACb,gBAAU,UAAU;AACpB,eAAS,UAAU;AAAA,IACrB;AAEA,WAAO,YAAY,QAAQ,WAAW,eAAe,KAAK,MAAM,CAAC;AAAA,EACnE;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,OAAO,OAAe,KAAmC;AACjE,UAAM,MAAM,WAAW,OAAO,GAAG;AACjC,QAAI,KAAK;AACP,aAAO,YAAY,KAAK,OAAO,eAAe,KAAK,MAAM,IAAI,MAAM,CAAC;AAAA,IACtE;AACA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,eAAe,OAAe,KAAmC;AACzE,QAAI,MAAM,GAAG,MAAM,IAAK,QAAO;AAC/B,QAAI,MAAM,KAAK,MAAM,OAAQ,QAAO;AACpC,QAAI,CAAC,sBAAsB,MAAM,MAAM,CAAC,CAAC,EAAG,QAAO;AAEnD,QAAI,SAAS,MAAM;AACnB,WAAO,SAAS,MAAM,UAAU,sBAAsB,MAAM,MAAM,CAAC,GAAG;AACpE;AAAA,IACF;AAEA,UAAM,SAAS,MAAM,MAAM,KAAK,MAAM;AACtC,WAAO,YAAY,QAAQ,cAAc,eAAe,KAAK,MAAM,CAAC;AAAA,EACtE;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,YAAY,OAAe,KAAmC;AAEtE,UAAM,UAAU,MAAM,MAAM,KAAK,MAAM,CAAC;AACxC,QAAI,CAAC,MAAM,MAAM,MAAM,MAAM,MAAM,MAAM,IAAI,EAAE,SAAS,OAAO,GAAG;AAChE,aAAO,YAAY,SAAS,YAAY,eAAe,KAAK,MAAM,CAAC,CAAC;AAAA,IACtE;AAGA,UAAM,UAAU,MAAM,GAAG;AACzB,QAAI,CAAC,KAAK,KAAK,KAAK,KAAK,KAAK,KAAK,KAAK,GAAG,EAAE,SAAS,OAAO,GAAG;AAC9D,aAAO,YAAY,SAAS,YAAY,eAAe,KAAK,MAAM,CAAC,CAAC;AAAA,IACtE;AAGA,QAAI,CAAC,KAAK,KAAK,KAAK,KAAK,KAAK,KAAK,GAAG,EAAE,SAAS,OAAO,GAAG;AACzD,aAAO,YAAY,SAAS,eAAe,eAAe,KAAK,MAAM,CAAC,CAAC;AAAA,IACzE;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAaU,qBACR,OACA,KACA,WACsB;AACtB,eAAW,YAAY,WAAW;AAChC,UAAI,MAAM,MAAM,KAAK,MAAM,SAAS,MAAM,MAAM,UAAU;AACxD,eAAO,YAAY,UAAU,YAAY,eAAe,KAAK,MAAM,SAAS,MAAM,CAAC;AAAA,MACrF;AAAA,IACF;AACA,WAAO;AAAA,EACT;AACF;AAAA;AAAA;AAAA;AAAA;AAxuBsB,eAygBM,sBAAkD;AAAA,EAC1E,EAAE,SAAS,MAAM,QAAQ,MAAM,QAAQ,EAAE;AAAA,EACzC,EAAE,SAAS,KAAK,QAAQ,KAAK,QAAQ,GAAG,eAAe,KAAK;AAAA,EAC5D,EAAE,SAAS,KAAK,QAAQ,KAAK,QAAQ,GAAG,eAAe,MAAM,eAAe,IAAI;AAAA,EAChF,EAAE,SAAS,KAAK,QAAQ,KAAK,QAAQ,GAAG,eAAe,KAAK;AAC9D;AA9gBK,IAAe,gBAAf;AA+xBA,SAAS,sBAAsB,QAAkD;AACtF,QAAM;AAAA,IACJ;AAAA,IACA,YAAY;AAAA,IACZ;AAAA,IACA;AAAA,IACA;AAAA,IACA,mBAAmB;AAAA,IACnB,kBAAkB;AAAA,IAClB;AAAA,EACF,IAAI;AAEJ,QAAM,aAAa,IAAI,IAAI,kBAAkB,SAAS,IAAI,OAAK,EAAE,YAAY,CAAC,IAAI,QAAQ;AAAA,EAE1F,MAAM,wBAAwB,cAAc;AAAA,IAI1C,cAAc;AACZ,YAAM;AAJR,WAAS,WAAW;AACpB,WAAS,YAAY;AAInB,UAAI,kBAAkB;AACpB,aAAK,mBAAmB,gBAAgB;AAAA,MAC1C;AACA,WAAK,mBAAmB,qBAAqB,CAAC;AAC9C,UAAI,gBAAgB;AAClB,aAAK,8BAA8B,gBAAgB,aAAa;AAAA,MAClE;AAAA,IACF;AAAA,IAEA,cAAc,OAA0B;AAEtC,YAAM,SAAS,kBAAkB,MAAM,YAAY,IAAI;AACvD,UAAI,WAAW,IAAI,MAAM,EAAG,QAAO;AAEnC,UAAI,KAAK,UAAU,KAAK,EAAG,QAAO;AAClC,UAAI,MAAM,KAAK,KAAK,EAAG,QAAO;AAC9B,UAAI,QAAQ,KAAK,KAAK,EAAG,QAAO;AAChC,UAAI,oBAAoB,8BAA8B,IAAI,KAAK,EAAG,QAAO;AACzE,aAAO;AAAA,IACT;AAAA,EACF;AAEA,SAAO,IAAI,gBAAgB;AAC7B;;;ACjtBO,SAAS,SAAS,MAAmC;AAC1D,SAAO,EAAE,MAAM,MAAM,YAAY,EAAI;AACvC;AAKO,SAAS,WACd,MACA,YACA,UACqB;AACrB,MAAI,UAAU;AACZ,WAAO,EAAE,MAAM,YAAY,SAAS;AAAA,EACtC;AACA,SAAO,EAAE,MAAM,WAAW;AAC5B;","names":["value","kind","position","normalized","length","normalized"]}

package/dist/core/tokenization/morphology/index.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Morphological normalization types and interfaces
+ */
+export * from './types';
+//# sourceMappingURL=index.d.ts.map

package/dist/core/tokenization/morphology/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/core/tokenization/morphology/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,cAAc,SAAS,CAAC"}

package/dist/core/tokenization/morphology/types.d.ts

@@ -0,0 +1,110 @@
+/**
+ * Morphological Normalizer Types
+ *
+ * Defines interfaces for language-specific morphological analysis.
+ * Normalizers reduce conjugated/inflected forms to canonical stems
+ * that can be matched against keyword dictionaries.
+ */
+/**
+ * Result of morphological normalization.
+ */
+export interface NormalizationResult {
+    /** The extracted stem/root form */
+    readonly stem: string;
+    /** Confidence in the normalization (0.0-1.0) */
+    readonly confidence: number;
+    /** Optional metadata about the transformation */
+    readonly metadata?: NormalizationMetadata;
+}
+/**
+ * Metadata about morphological transformations applied.
+ */
+export interface NormalizationMetadata {
+    /** Prefixes that were removed */
+    readonly removedPrefixes?: readonly string[];
+    /** Suffixes that were removed */
+    readonly removedSuffixes?: readonly string[];
+    /** Type of conjugation detected */
+    readonly conjugationType?: ConjugationType;
+    /** Original form classification */
+    readonly originalForm?: string;
+    /** Applied transformation rules (for debugging) */
+    readonly appliedRules?: readonly string[];
+}
+/**
+ * Types of verb conjugation/inflection.
+ */
+export type ConjugationType = 'present' | 'past' | 'future' | 'progressive' | 'perfect' | 'imperative' | 'subjunctive' | 'conditional' | 'passive' | 'causative' | 'polite' | 'humble' | 'honorific' | 'negative' | 'potential' | 'volitional' | 'conditional-tara' | 'conditional-to' | 'conditional-ba' | 'connective' | 'conditional-myeon' | 'temporal-ttae' | 'causal-nikka' | 'honorific-conditional' | 'honorific-temporal' | 'honorific-causal' | 'honorific-past' | 'honorific-polite' | 'sequential-after' | 'sequential-before' | 'immediate' | 'obligation' | 'reflexive' | 'reflexive-imperative' | 'gerund' | 'participle' | 'conditional-idha' | 'temporal-indama' | 'temporal-hina' | 'temporal-lamma' | 'past-verb' | 'conditional-se' | 'temporal-ince' | 'temporal-dikce' | 'aorist' | 'optative' | 'necessitative' | 'request' | 'casual-request' | 'contracted' | 'contracted-past' | 'compound' | 'te-form' | 'dictionary';
+/**
+ * Interface for language-specific morphological normalizers.
+ *
+ * Normalizers attempt to reduce inflected word forms to their
+ * canonical stems. This enables matching conjugated verbs against
+ * keyword dictionaries that only contain base forms.
+ *
+ * Example (Japanese):
+ *   切り替えた (past) → { stem: '切り替え', confidence: 0.85 }
+ *   切り替えます (polite) → { stem: '切り替え', confidence: 0.85 }
+ *
+ * Example (Spanish):
+ *   mostrarse (reflexive infinitive) → { stem: 'mostrar', confidence: 0.85 }
+ *   alternando (gerund) → { stem: 'alternar', confidence: 0.85 }
+ */
+export interface MorphologicalNormalizer {
+    /** Language code this normalizer handles */
+    readonly language: string;
+    /**
+     * Normalize a word to its canonical stem form.
+     *
+     * @param word - The word to normalize
+     * @returns Normalization result with stem and confidence
+     */
+    normalize(word: string): NormalizationResult;
+    /**
+     * Check if a word appears to be a verb form that can be normalized.
+     * Optional optimization to skip normalization for non-verb tokens.
+     *
+     * @param word - The word to check
+     * @returns true if the word might be a normalizable verb form
+     */
+    isNormalizable?(word: string): boolean;
+}
+/**
+ * Configuration for suffix-based normalization rules.
+ * Used by agglutinative languages (Japanese, Korean, Turkish).
+ */
+export interface SuffixRule {
+    /** The suffix pattern to match */
+    readonly pattern: string;
+    /** Confidence when this suffix is stripped */
+    readonly confidence: number;
+    /** What to replace the suffix with (empty string for simple removal) */
+    readonly replacement?: string;
+    /** Conjugation type this suffix indicates */
+    readonly conjugationType?: ConjugationType;
+    /** Minimum stem length after stripping (to avoid over-stripping) */
+    readonly minStemLength?: number;
+}
+/**
+ * Configuration for prefix-based normalization rules.
+ * Used primarily by Arabic for article/conjunction prefixes.
+ */
+export interface PrefixRule {
+    /** The prefix pattern to match */
+    readonly pattern: string;
+    /** Confidence penalty when this prefix is stripped */
+    readonly confidencePenalty: number;
+    /** What the prefix indicates (for metadata) */
+    readonly prefixType?: 'article' | 'conjunction' | 'preposition' | 'verb-marker';
+    /** Minimum remaining characters after stripping (to avoid over-stripping) */
+    readonly minRemaining?: number;
+}
+/**
+ * Helper to create a "no change" normalization result.
+ */
+export declare function noChange(word: string): NormalizationResult;
+/**
+ * Helper to create a normalization result with metadata.
+ */
+export declare function normalized(stem: string, confidence: number, metadata?: NormalizationMetadata): NormalizationResult;
+//# sourceMappingURL=types.d.ts.map

package/dist/core/tokenization/morphology/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../src/core/tokenization/morphology/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,mCAAmC;IACnC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB,gDAAgD;IAChD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAE5B,iDAAiD;IACjD,QAAQ,CAAC,QAAQ,CAAC,EAAE,qBAAqB,CAAC;CAC3C;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,iCAAiC;IACjC,QAAQ,CAAC,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAE7C,iCAAiC;IACjC,QAAQ,CAAC,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAE7C,mCAAmC;IACnC,QAAQ,CAAC,eAAe,CAAC,EAAE,eAAe,CAAC;IAE3C,mCAAmC;IACnC,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAE/B,mDAAmD;IACnD,QAAQ,CAAC,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CAC3C;AAED;;GAEG;AACH,MAAM,MAAM,eAAe,GAEvB,SAAS,GACT,MAAM,GACN,QAAQ,GACR,aAAa,GACb,SAAS,GAET,YAAY,GACZ,aAAa,GACb,aAAa,GAEb,SAAS,GACT,WAAW,GAEX,QAAQ,GACR,QAAQ,GACR,WAAW,GAEX,UAAU,GACV,WAAW,GACX,YAAY,GAEZ,kBAAkB,GAClB,gBAAgB,GAChB,gBAAgB,GAEhB,YAAY,GACZ,mBAAmB,GACnB,eAAe,GACf,cAAc,GAEd,uBAAuB,GACvB,oBAAoB,GACpB,kBAAkB,GAClB,gBAAgB,GAChB,kBAAkB,GAElB,kBAAkB,GAClB,mBAAmB,GACnB,WAAW,GACX,YAAY,GAEZ,WAAW,GACX,sBAAsB,GACtB,QAAQ,GACR,YAAY,GAEZ,kBAAkB,GAClB,iBAAiB,GACjB,eAAe,GACf,gBAAgB,GAChB,WAAW,GAEX,gBAAgB,GAChB,eAAe,GACf,gBAAgB,GAChB,QAAQ,GACR,UAAU,GACV,eAAe,GAEf,SAAS,GACT,gBAAgB,GAChB,YAAY,GACZ,iBAAiB,GAEjB,UAAU,GACV,SAAS,GACT,YAAY,CAAC;AAEjB;;;;;;;;;;;;;;GAcG;AACH,MAAM,WAAW,uBAAuB;IACtC,4CAA4C;IAC5C,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAE1B;;;;;OAKG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,CAAC;IAE7C;;;;;;OAMG;IACH,cAAc,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;CACxC;AAED;;;GAGG;AACH,MAAM,WAAW,UAAU;IACzB,kCAAkC;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAEzB,8CAA8C;IAC9C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAE5B,wEAAwE;IACxE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAE9B,6CAA6C;IAC7C,QAAQ,CAAC,eAAe,CAAC,EAAE,eAAe,CAAC;IAE3C,oEAAoE;IACpE,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;CACjC;AAED;;;GAGG;AACH,MAAM,WAAW,UAAU;IACzB,kCAAkC;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAEzB,sDAAsD;IACtD,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IAEnC,+CAA+C;IAC/C,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,GAAG,aAAa,GAAG,aAAa,GAAG,aAAa,CAAC;IAEhF,6EAA6E;IAC7E,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;CAChC;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,CAE1D;AAED;;GAEG;AACH,wBAAgB,UAAU,CACxB,IAAI,EAAE,MAAM,EACZ,UAAU,EAAE,MAAM,EAClB,QAAQ,CAAC,EAAE,qBAAqB,GAC/B,mBAAmB,CAKrB"}

package/dist/core/tokenization/token-utils.d.ts

@@ -0,0 +1,111 @@
+/**
+ * Token Utilities
+ *
+ * Core token creation, stream implementation, and character classification.
+ * These are the foundational building blocks used by all tokenizers.
+ */
+import type { LanguageToken, TokenKind, TokenStream, StreamMark, SourcePosition } from '../types';
+/**
+ * Configuration for a native language time unit pattern.
+ * Used by tryNumberWithTimeUnits() to match language-specific time units.
+ */
+export interface TimeUnitMapping {
+    /** The pattern to match (e.g., 'segundos', 'ミリ秒') */
+    readonly pattern: string;
+    /** The standard suffix to use (ms, s, m, h) */
+    readonly suffix: string;
+    /** Length of the pattern (for optimization) */
+    readonly length: number;
+    /** Whether to check for word boundary after the pattern */
+    readonly checkBoundary?: boolean;
+    /** Character that cannot follow the pattern (e.g., 's' for 'm' to avoid 'ms') */
+    readonly notFollowedBy?: string;
+    /** Whether to do case-insensitive matching */
+    readonly caseInsensitive?: boolean;
+}
+/**
+ * Concrete implementation of TokenStream.
+ */
+export declare class TokenStreamImpl implements TokenStream {
+    readonly tokens: readonly LanguageToken[];
+    readonly language: string;
+    private pos;
+    constructor(tokens: LanguageToken[], language: string);
+    peek(offset?: number): LanguageToken | null;
+    advance(): LanguageToken;
+    isAtEnd(): boolean;
+    mark(): StreamMark;
+    reset(mark: StreamMark): void;
+    position(): number;
+    /**
+     * Get remaining tokens as an array.
+     */
+    remaining(): LanguageToken[];
+    /**
+     * Consume tokens while predicate is true.
+     */
+    takeWhile(predicate: (token: LanguageToken) => boolean): LanguageToken[];
+    /**
+     * Skip tokens while predicate is true.
+     */
+    skipWhile(predicate: (token: LanguageToken) => boolean): void;
+}
+/**
+ * Create a source position from start and end offsets.
+ */
+export declare function createPosition(start: number, end: number): SourcePosition;
+/**
+ * Options for creating a token with optional morphological data.
+ */
+export interface CreateTokenOptions {
+    /** Explicitly normalized form from keyword map */
+    normalized?: string;
+    /** Morphologically normalized stem */
+    stem?: string;
+    /** Confidence in the stem (0.0-1.0) */
+    stemConfidence?: number;
+    /** Additional metadata for specific token types (e.g., event modifier data) */
+    metadata?: Record<string, unknown>;
+}
+/**
+ * Token creation options (object style).
+ */
+export interface TokenCreationParams extends CreateTokenOptions {
+    value: string;
+    kind: TokenKind;
+    position: SourcePosition;
+}
+/**
+ * Create a language token (object style).
+ */
+export declare function createToken(params: TokenCreationParams): LanguageToken;
+/**
+ * Create a language token (separate parameters style).
+ */
+export declare function createToken(value: string, kind: TokenKind, position: SourcePosition, normalizedOrOptions?: string | CreateTokenOptions): LanguageToken;
+/**
+ * Check if a character is whitespace.
+ */
+export declare function isWhitespace(char: string): boolean;
+/**
+ * Check if a string starts with a CSS selector prefix.
+ * Includes JSX-style element selectors: <form />, <div>
+ */
+export declare function isSelectorStart(char: string): boolean;
+/**
+ * Check if a character is a quote (string delimiter).
+ */
+export declare function isQuote(char: string): boolean;
+/**
+ * Check if a character is a digit.
+ */
+export declare function isDigit(char: string): boolean;
+/**
+ * Check if a character is an ASCII letter.
+ */
+export declare function isAsciiLetter(char: string): boolean;
+/**
+ * Check if a character is part of an ASCII identifier.
+ */
+export declare function isAsciiIdentifierChar(char: string): boolean;
+//# sourceMappingURL=token-utils.d.ts.map

package/dist/core/tokenization/token-utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"token-utils.d.ts","sourceRoot":"","sources":["../../../src/core/tokenization/token-utils.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAMlG;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,qDAAqD;IACrD,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,+CAA+C;IAC/C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,+CAA+C;IAC/C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,2DAA2D;IAC3D,QAAQ,CAAC,aAAa,CAAC,EAAE,OAAO,CAAC;IACjC,iFAAiF;IACjF,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,8CAA8C;IAC9C,QAAQ,CAAC,eAAe,CAAC,EAAE,OAAO,CAAC;CACpC;AAMD;;GAEG;AACH,qBAAa,eAAgB,YAAW,WAAW;IACjD,QAAQ,CAAC,MAAM,EAAE,SAAS,aAAa,EAAE,CAAC;IAC1C,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,GAAG,CAAa;gBAEZ,MAAM,EAAE,aAAa,EAAE,EAAE,QAAQ,EAAE,MAAM;IAKrD,IAAI,CAAC,MAAM,GAAE,MAAU,GAAG,aAAa,GAAG,IAAI;IAQ9C,OAAO,IAAI,aAAa;IAOxB,OAAO,IAAI,OAAO;IAIlB,IAAI,IAAI,UAAU;IAIlB,KAAK,CAAC,IAAI,EAAE,UAAU,GAAG,IAAI;IAI7B,QAAQ,IAAI,MAAM;IAIlB;;OAEG;IACH,SAAS,IAAI,aAAa,EAAE;IAI5B;;OAEG;IACH,SAAS,CAAC,SAAS,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,OAAO,GAAG,aAAa,EAAE;IAQxE;;OAEG;IACH,SAAS,CAAC,SAAS,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,OAAO,GAAG,IAAI;CAK9D;AAMD;;GAEG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,cAAc,CAEzE;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,kDAAkD;IAClD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,sCAAsC;IACtC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,uCAAuC;IACvC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,mBAAoB,SAAQ,kBAAkB;IAC7D,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,SAAS,CAAC;IAChB,QAAQ,EAAE,cAAc,CAAC;CAC1B;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,mBAAmB,GAAG,aAAa,CAAC;AAExE;;GAEG;AACH,wBAAgB,WAAW,CACzB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,SAAS,EACf,QAAQ,EAAE,cAAc,EACxB,mBAAmB,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAChD,aAAa,CAAC;AAqDjB;;GAEG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAElD;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAIrD;AAED;;GAEG;AACH,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAE7C;AAED;;GAEG;AACH,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAE7C;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAEnD;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAE3D"}