@lokascript/framework 2.6.0 → 2.7.0

package/dist/multilingual/index.js

@@ -1 +1,1550 @@
+// src/interfaces/value-extractor.ts
+var StringLiteralExtractor = class {
+  constructor() {
+    this.name = "string-literal";
+  }
+  canExtract(input, position) {
+    const char = input[position];
+    return char === '"' || char === "'" || char === "`" || char === "\u201C" || // Chinese double quote open "
+    char === "\u2018";
+  }
+  extract(input, position) {
+    const quote = input[position];
+    if (quote === "\u201C") {
+      let length2 = 1;
+      while (position + length2 < input.length) {
+        if (input[position + length2] === "\u201D") {
+          length2++;
+          return { value: input.substring(position, position + length2), length: length2 };
+        }
+        length2++;
+      }
+      return null;
+    }
+    if (quote === "\u2018") {
+      let length2 = 1;
+      while (position + length2 < input.length) {
+        if (input[position + length2] === "\u2019") {
+          length2++;
+          return { value: input.substring(position, position + length2), length: length2 };
+        }
+        length2++;
+      }
+      return null;
+    }
+    let length = 1;
+    let escaped = false;
+    while (position + length < input.length) {
+      const char = input[position + length];
+      if (escaped) {
+        escaped = false;
+        length++;
+        continue;
+      }
+      if (char === "\\") {
+        escaped = true;
+        length++;
+        continue;
+      }
+      if (char === quote) {
+        length++;
+        return {
+          value: input.substring(position, position + length),
+          length
+        };
+      }
+      length++;
+    }
+    return null;
+  }
+};
+var NumberExtractor = class {
+  constructor() {
+    this.name = "number";
+  }
+  canExtract(input, position) {
+    return /\d/.test(input[position]);
+  }
+  extract(input, position) {
+    let length = 0;
+    let hasDecimal = false;
+    while (position + length < input.length) {
+      const char = input[position + length];
+      if (/\d/.test(char)) {
+        length++;
+      } else if (char === "." && !hasDecimal) {
+        hasDecimal = true;
+        length++;
+      } else {
+        break;
+      }
+    }
+    if (length === 0) return null;
+    const numValue = input.substring(position, position + length);
+    const afterNum = position + length;
+    if (afterNum < input.length) {
+      const remaining = input.slice(afterNum);
+      const cjkMultiUnits = [
+        { pattern: "\u6BEB\u79D2", suffix: "ms" },
+        // Chinese milliseconds
+        { pattern: "\u5206\u949F", suffix: "m" },
+        // Chinese minutes
+        { pattern: "\u5C0F\u65F6", suffix: "h" },
+        // Chinese hours
+        { pattern: "\u30DF\u30EA\u79D2", suffix: "ms" },
+        // Japanese milliseconds
+        { pattern: "\u6642\u9593", suffix: "h" }
+        // Japanese hours
+      ];
+      for (const unit of cjkMultiUnits) {
+        if (remaining.startsWith(unit.pattern)) {
+          return {
+            value: numValue + unit.suffix,
+            length: length + unit.pattern.length,
+            metadata: { hasTimeUnit: true }
+          };
+        }
+      }
+      if (remaining.startsWith("ms")) {
+        return {
+          value: numValue + "ms",
+          length: length + 2,
+          metadata: { hasTimeUnit: true }
+        };
+      }
+      const cjkSingleUnits = [
+        { pattern: "\u79D2", suffix: "s" },
+        // CJK seconds
+        { pattern: "\u5206", suffix: "m" }
+        // CJK minutes
+      ];
+      for (const unit of cjkSingleUnits) {
+        if (remaining.startsWith(unit.pattern)) {
+          return {
+            value: numValue + unit.suffix,
+            length: length + 1,
+            metadata: { hasTimeUnit: true }
+          };
+        }
+      }
+      if (/^[smh](?![a-zA-Z])/.test(remaining)) {
+        return {
+          value: numValue + remaining[0],
+          length: length + 1,
+          metadata: { hasTimeUnit: true }
+        };
+      }
+    }
+    return { value: numValue, length };
+  }
+};
+var IdentifierExtractor = class {
+  constructor() {
+    this.name = "identifier";
+  }
+  canExtract(input, position) {
+    return /[a-zA-Z_]/.test(input[position]);
+  }
+  extract(input, position) {
+    let length = 0;
+    while (position + length < input.length) {
+      const char = input[position + length];
+      if (/[a-zA-Z0-9_]/.test(char)) {
+        length++;
+      } else {
+        break;
+      }
+    }
+    return length > 0 ? {
+      value: input.substring(position, position + length),
+      length
+    } : null;
+  }
+};
+var UnicodeIdentifierExtractor = class {
+  constructor() {
+    this.name = "unicode-identifier";
+  }
+  canExtract(input, position) {
+    const code = input.charCodeAt(position);
+    if (code < 128) return false;
+    return /\p{L}/u.test(input[position]);
+  }
+  extract(input, position) {
+    let length = 0;
+    while (position + length < input.length) {
+      const char = input[position + length];
+      if (/[\p{L}\p{N}\p{M}]/u.test(char)) {
+        length++;
+      } else {
+        break;
+      }
+    }
+    return length > 0 ? { value: input.substring(position, position + length), length } : null;
+  }
+};
+var LatinExtendedIdentifierExtractor = class {
+  constructor() {
+    this.name = "latin-extended-identifier";
+  }
+  canExtract(input, position) {
+    return /\p{L}/u.test(input[position]);
+  }
+  extract(input, position) {
+    let end = position;
+    while (end < input.length && /[\p{L}\p{N}_-]/u.test(input[end])) {
+      end++;
+    }
+    if (end === position) return null;
+    return { value: input.slice(position, end), length: end - position };
+  }
+};
+function isContextAwareExtractor(extractor) {
+  return "setContext" in extractor && typeof extractor.setContext === "function";
+}
+function createTokenizerContext(tokenizer) {
+  const ctx = {
+    language: tokenizer.language,
+    direction: tokenizer.direction,
+    lookupKeyword: tokenizer.lookupKeyword.bind(tokenizer),
+    isKeyword: tokenizer.isKeyword.bind(tokenizer),
+    isKeywordStart: tokenizer.isKeywordStart.bind(tokenizer),
+    ...tokenizer.isKeywordStartAtBoundary ? { isKeywordStartAtBoundary: tokenizer.isKeywordStartAtBoundary.bind(tokenizer) } : {}
+  };
+  if (tokenizer.normalizer) {
+    return { ...ctx, normalizer: tokenizer.normalizer };
+  }
+  return ctx;
+}
+
+// src/core/tokenization/token-utils.ts
+var TokenStreamImpl = class {
+  constructor(tokens, language) {
+    this.pos = 0;
+    this.tokens = tokens;
+    this.language = language;
+  }
+  peek(offset = 0) {
+    const index = this.pos + offset;
+    if (index < 0 || index >= this.tokens.length) {
+      return null;
+    }
+    return this.tokens[index];
+  }
+  advance() {
+    if (this.isAtEnd()) {
+      throw new Error("Unexpected end of token stream");
+    }
+    return this.tokens[this.pos++];
+  }
+  isAtEnd() {
+    return this.pos >= this.tokens.length;
+  }
+  mark() {
+    return { position: this.pos };
+  }
+  reset(mark) {
+    this.pos = mark.position;
+  }
+  position() {
+    return this.pos;
+  }
+  /**
+   * Get remaining tokens as an array.
+   */
+  remaining() {
+    return this.tokens.slice(this.pos);
+  }
+  /**
+   * Consume tokens while predicate is true.
+   */
+  takeWhile(predicate) {
+    const result = [];
+    while (!this.isAtEnd() && predicate(this.peek())) {
+      result.push(this.advance());
+    }
+    return result;
+  }
+  /**
+   * Skip tokens while predicate is true.
+   */
+  skipWhile(predicate) {
+    while (!this.isAtEnd() && predicate(this.peek())) {
+      this.advance();
+    }
+  }
+};
+function createPosition(start, end) {
+  return { start, end };
+}
+function createToken(valueOrParams, kind, position, normalizedOrOptions) {
+  if (typeof valueOrParams === "object") {
+    const { value: value2, kind: kind2, position: position2, normalized, stem, stemConfidence, metadata } = valueOrParams;
+    return {
+      value: value2,
+      kind: kind2,
+      position: position2,
+      ...normalized !== void 0 && { normalized },
+      ...stem !== void 0 && { stem },
+      ...stemConfidence !== void 0 && { stemConfidence },
+      ...metadata !== void 0 && { metadata }
+    };
+  }
+  const value = valueOrParams;
+  if (!kind || !position) {
+    throw new Error("createToken requires kind and position parameters");
+  }
+  if (typeof normalizedOrOptions === "string") {
+    return { value, kind, position, normalized: normalizedOrOptions };
+  }
+  if (normalizedOrOptions) {
+    const { normalized, stem, stemConfidence, metadata } = normalizedOrOptions;
+    return {
+      value,
+      kind,
+      position,
+      ...normalized !== void 0 && { normalized },
+      ...stem !== void 0 && { stem },
+      ...stemConfidence !== void 0 && { stemConfidence },
+      ...metadata !== void 0 && { metadata }
+    };
+  }
+  return { value, kind, position };
+}
+function isWhitespace(char) {
+  return /\s/.test(char);
+}
+function isSelectorStart(char) {
+  return char === "#" || char === "." || char === "[" || char === "@" || char === "*" || char === "<";
+}
+function isQuote(char) {
+  return char === '"' || char === "'" || char === "`" || char === "\u300C" || char === "\u300D";
+}
+function isDigit(char) {
+  return /\d/.test(char);
+}
+function isAsciiLetter(char) {
+  return /[a-zA-Z]/.test(char);
+}
+function isAsciiIdentifierChar(char) {
+  return /[a-zA-Z0-9_-]/.test(char);
+}
+
+// src/core/tokenization/extractors.ts
+function extractCssSelector(input, startPos) {
+  if (startPos >= input.length) return null;
+  const char = input[startPos];
+  if (!isSelectorStart(char)) return null;
+  let pos = startPos;
+  let selector = "";
+  if (char === "#" || char === ".") {
+    selector += input[pos++];
+    while (pos < input.length && isAsciiIdentifierChar(input[pos])) {
+      selector += input[pos++];
+    }
+    if (selector.length <= 1) return null;
+    if (pos < input.length && input[pos] === "." && char === "#") {
+      const methodStart = pos + 1;
+      let methodEnd = methodStart;
+      while (methodEnd < input.length && isAsciiIdentifierChar(input[methodEnd])) {
+        methodEnd++;
+      }
+      if (methodEnd < input.length && input[methodEnd] === "(") {
+        return selector;
+      }
+    }
+  } else if (char === "[") {
+    let depth = 1;
+    let inQuote = false;
+    let quoteChar = null;
+    let escaped = false;
+    selector += input[pos++];
+    while (pos < input.length && depth > 0) {
+      const c = input[pos];
+      selector += c;
+      if (escaped) {
+        escaped = false;
+      } else if (c === "\\") {
+        escaped = true;
+      } else if (inQuote) {
+        if (c === quoteChar) {
+          inQuote = false;
+          quoteChar = null;
+        }
+      } else {
+        if (c === '"' || c === "'" || c === "`") {
+          inQuote = true;
+          quoteChar = c;
+        } else if (c === "[") {
+          depth++;
+        } else if (c === "]") {
+          depth--;
+        }
+      }
+      pos++;
+    }
+    if (depth !== 0) return null;
+  } else if (char === "@") {
+    selector += input[pos++];
+    while (pos < input.length && isAsciiIdentifierChar(input[pos])) {
+      selector += input[pos++];
+    }
+    if (selector.length <= 1) return null;
+  } else if (char === "*") {
+    selector += input[pos++];
+    while (pos < input.length && isAsciiIdentifierChar(input[pos])) {
+      selector += input[pos++];
+    }
+    if (selector.length <= 1) return null;
+  } else if (char === "<") {
+    selector += input[pos++];
+    if (pos >= input.length || !isAsciiLetter(input[pos])) return null;
+    while (pos < input.length && isAsciiIdentifierChar(input[pos])) {
+      selector += input[pos++];
+    }
+    while (pos < input.length) {
+      const modChar = input[pos];
+      if (modChar === ".") {
+        selector += input[pos++];
+        if (pos >= input.length || !isAsciiIdentifierChar(input[pos])) {
+          return null;
+        }
+        while (pos < input.length && isAsciiIdentifierChar(input[pos])) {
+          selector += input[pos++];
+        }
+      } else if (modChar === "#") {
+        selector += input[pos++];
+        if (pos >= input.length || !isAsciiIdentifierChar(input[pos])) {
+          return null;
+        }
+        while (pos < input.length && isAsciiIdentifierChar(input[pos])) {
+          selector += input[pos++];
+        }
+      } else if (modChar === "[") {
+        let depth = 1;
+        let inQuote = false;
+        let quoteChar = null;
+        let escaped = false;
+        selector += input[pos++];
+        while (pos < input.length && depth > 0) {
+          const c = input[pos];
+          selector += c;
+          if (escaped) {
+            escaped = false;
+          } else if (c === "\\") {
+            escaped = true;
+          } else if (inQuote) {
+            if (c === quoteChar) {
+              inQuote = false;
+              quoteChar = null;
+            }
+          } else {
+            if (c === '"' || c === "'" || c === "`") {
+              inQuote = true;
+              quoteChar = c;
+            } else if (c === "[") {
+              depth++;
+            } else if (c === "]") {
+              depth--;
+            }
+          }
+          pos++;
+        }
+        if (depth !== 0) return null;
+      } else {
+        break;
+      }
+    }
+    while (pos < input.length && isWhitespace(input[pos])) {
+      selector += input[pos++];
+    }
+    if (pos < input.length && input[pos] === "/") {
+      selector += input[pos++];
+      while (pos < input.length && isWhitespace(input[pos])) {
+        selector += input[pos++];
+      }
+    }
+    if (pos >= input.length || input[pos] !== ">") return null;
+    selector += input[pos++];
+  }
+  return selector || null;
+}
+function isPossessiveMarker(input, pos) {
+  if (pos >= input.length || input[pos] !== "'") return false;
+  if (pos + 1 >= input.length) return false;
+  const nextChar = input[pos + 1].toLowerCase();
+  if (nextChar !== "s") return false;
+  if (pos + 2 >= input.length) return true;
+  const afterS = input[pos + 2];
+  return isWhitespace(afterS) || afterS === "*" || !isAsciiIdentifierChar(afterS);
+}
+function extractStringLiteral(input, startPos) {
+  if (startPos >= input.length) return null;
+  const openQuote = input[startPos];
+  if (!isQuote(openQuote)) return null;
+  if (openQuote === "'" && isPossessiveMarker(input, startPos)) {
+    return null;
+  }
+  const closeQuoteMap = {
+    '"': '"',
+    "'": "'",
+    "`": "`",
+    "\u300C": "\u300D"
+  };
+  const closeQuote = closeQuoteMap[openQuote];
+  if (!closeQuote) return null;
+  let pos = startPos + 1;
+  let literal = openQuote;
+  let escaped = false;
+  while (pos < input.length) {
+    const char = input[pos];
+    literal += char;
+    if (escaped) {
+      escaped = false;
+    } else if (char === "\\") {
+      escaped = true;
+    } else if (char === closeQuote) {
+      return literal;
+    }
+    pos++;
+  }
+  return literal;
+}
+function isUrlStart(input, pos) {
+  if (pos >= input.length) return false;
+  const char = input[pos];
+  const next = input[pos + 1] || "";
+  const third = input[pos + 2] || "";
+  if (char === "/" && next !== "/" && /[a-zA-Z0-9._-]/.test(next)) {
+    return true;
+  }
+  if (char === "/" && next === "/" && /[a-zA-Z]/.test(third)) {
+    return true;
+  }
+  if (char === "." && (next === "/" || next === "." && third === "/")) {
+    return true;
+  }
+  const slice = input.slice(pos, pos + 8).toLowerCase();
+  if (slice.startsWith("http://") || slice.startsWith("https://")) {
+    return true;
+  }
+  return false;
+}
+function extractUrl(input, startPos) {
+  if (!isUrlStart(input, startPos)) return null;
+  let pos = startPos;
+  let url = "";
+  const urlChars = /[a-zA-Z0-9/:._\-?&=%@+~!$'()*,;[\]]/;
+  while (pos < input.length) {
+    const char = input[pos];
+    if (char === "#") {
+      if (url.length > 0 && /[a-zA-Z0-9/.]$/.test(url)) {
+        url += char;
+        pos++;
+        while (pos < input.length && /[a-zA-Z0-9_-]/.test(input[pos])) {
+          url += input[pos++];
+        }
+      }
+      break;
+    }
+    if (urlChars.test(char)) {
+      url += char;
+      pos++;
+    } else {
+      break;
+    }
+  }
+  if (url.length < 2) return null;
+  return url;
+}
+function extractNumber(input, startPos) {
+  if (startPos >= input.length) return null;
+  const char = input[startPos];
+  if (!isDigit(char) && char !== "-" && char !== "+") return null;
+  let pos = startPos;
+  let number = "";
+  if (input[pos] === "-" || input[pos] === "+") {
+    number += input[pos++];
+  }
+  if (pos >= input.length || !isDigit(input[pos])) {
+    return null;
+  }
+  while (pos < input.length && isDigit(input[pos])) {
+    number += input[pos++];
+  }
+  if (pos < input.length && input[pos] === ".") {
+    number += input[pos++];
+    while (pos < input.length && isDigit(input[pos])) {
+      number += input[pos++];
+    }
+  }
+  if (pos < input.length) {
+    const suffix = input.slice(pos, pos + 2);
+    if (suffix === "ms") {
+      number += "ms";
+    } else if (input[pos] === "s" || input[pos] === "m" || input[pos] === "h") {
+      number += input[pos];
+    }
+  }
+  return number;
+}
+
+// src/core/tokenization/extractors/operator.ts
+var DEFAULT_OPERATORS = [
+  // Three-character operators
+  "===",
+  "!==",
+  "->",
+  // Two-character operators
+  "==",
+  "!=",
+  "<=",
+  ">=",
+  "&&",
+  "||",
+  "**",
+  "+=",
+  "-=",
+  "*=",
+  "/=",
+  // Single-character operators
+  "+",
+  "-",
+  "*",
+  "/",
+  "=",
+  ">",
+  "<",
+  "!",
+  "&",
+  "|",
+  "%",
+  "^",
+  "~"
+];
+var OperatorExtractor = class {
+  constructor(operators = DEFAULT_OPERATORS) {
+    this.operators = operators;
+    this.name = "operator";
+    this.operators = [...operators].sort((a, b) => b.length - a.length);
+  }
+  canExtract(input, position) {
+    return this.operators.some((op) => input.startsWith(op, position));
+  }
+  extract(input, position) {
+    for (const op of this.operators) {
+      if (input.startsWith(op, position)) {
+        return {
+          value: op,
+          length: op.length
+        };
+      }
+    }
+    return null;
+  }
+};
+
+// src/core/tokenization/extractors/punctuation.ts
+var DEFAULT_PUNCTUATION = "()[]{},:;";
+var PunctuationExtractor = class {
+  constructor(punctuation = DEFAULT_PUNCTUATION) {
+    this.punctuation = punctuation;
+    this.name = "punctuation";
+  }
+  canExtract(input, position) {
+    return this.punctuation.includes(input[position]);
+  }
+  extract(input, position) {
+    const char = input[position];
+    if (this.punctuation.includes(char)) {
+      return {
+        value: char,
+        length: 1
+      };
+    }
+    return null;
+  }
+};
+
+// src/core/tokenization/default-extractors.ts
+function getDefaultExtractors() {
+  return [
+    new StringLiteralExtractor(),
+    // "strings", 'strings', `strings`
+    new NumberExtractor(),
+    // 123, 45.67
+    new OperatorExtractor(),
+    // +, -, *, /, =, >, <, etc.
+    new PunctuationExtractor(),
+    // ( ) [ ] { } , : ;
+    new IdentifierExtractor(),
+    // variable_names, functionNames (ASCII)
+    new UnicodeIdentifierExtractor()
+    // CJK, Arabic, Cyrillic, etc.
+  ];
+}
+
+// src/core/tokenization/base-tokenizer.ts
+var SIMPLE_TOKENIZER_OPERATOR_SET = new Set(DEFAULT_OPERATORS);
+var MARKER_CONCEPT_NORMALIZEDS = /* @__PURE__ */ new Set([
+  // Role-marker role names (profile.roleMarkers normalizeds)
+  "patient",
+  "destination",
+  "source",
+  "style",
+  "event",
+  "eventMarker",
+  "agent",
+  "goal",
+  "manner",
+  // Prepositional / positional modifier concepts matched via the role mechanism
+  // (profile.keywords "Modifiers"). `before`/`after`/`until` are intentionally
+  // NOT here — they are pattern literals (see the note above).
+  "into",
+  "from",
+  "to",
+  "with",
+  "at",
+  "of",
+  "as",
+  "by",
+  "in",
+  "on",
+  "over",
+  "under",
+  "between",
+  "through",
+  "without"
+]);
+var ENGLISH_DOM_EVENT_NAMES = [
+  "click",
+  "dblclick",
+  "input",
+  "change",
+  "submit",
+  "keydown",
+  "keyup",
+  "keypress",
+  "mousedown",
+  "mouseup",
+  "mouseover",
+  "mouseout",
+  "mouseenter",
+  "mouseleave",
+  "mousemove",
+  "pointerdown",
+  "pointerup",
+  "pointermove",
+  "focus",
+  "blur",
+  "load",
+  "resize",
+  "scroll"
+];
+var _BaseTokenizer = class _BaseTokenizer {
+  constructor() {
+    /** Keywords derived from profile, sorted longest-first for greedy matching */
+    this.profileKeywords = [];
+    /**
+     * Space-containing profile keywords (multi-word phrases), longest-first.
+     * Used by `tryMultiWordKeyword` so natural spaced forms (hi `मेल खाता`,
+     * vi `chuyển đổi`, es `tecla abajo`, …) tokenize as ONE keyword — the
+     * profile-driven replacement for the per-language hardcoded compound lists.
+     * Empty for no-space (CJK) languages, so they are unaffected.
+     */
+    this.multiWordKeywords = [];
+    /** Map for O(1) keyword lookups by lowercase native word */
+    this.profileKeywordMap = /* @__PURE__ */ new Map();
+    /**
+     * The raw EXTRAS list passed to initializeKeywordsFromProfile, kept pre-dedup.
+     * The keyword map is keyed by native word with last-wins insertion, so a
+     * duplicate native word inside the extras silently shadows the earlier entry
+     * (e.g. a `nächste→closest` entry shadowing `nächste→next` broke German
+     * positional expressions). Exposed so consistency tests can detect such
+     * intra-extras collisions, which are invisible in the deduplicated map.
+     */
+    this.rawExtraEntries = [];
+    /**
+     * Pluggable value extractors for domain-specific syntax.
+     * When registered, BaseTokenizer will use extractor-based tokenization instead of legacy methods.
+     */
+    this.extractors = [];
+  }
+  /** Raw extras as passed in, pre-dedup — for consistency tests. */
+  getExtraKeywordEntries() {
+    return this.rawExtraEntries;
+  }
+  /**
+   * Tokenize input string to token stream.
+   * Delegates to extractor-based tokenization if extractors are registered,
+   * otherwise subclass must override this method.
+   *
+   * @param input - Input string to tokenize
+   * @returns Token stream
+   */
+  tokenize(input) {
+    if (this.isUsingExtractors()) {
+      return this.tokenizeWithExtractors(input);
+    }
+    throw new Error(
+      `${this.constructor.name}: tokenize() not implemented and no extractors registered. Either register extractors or override tokenize() method.`
+    );
+  }
+  /**
+   * Register a value extractor for domain-specific syntax.
+   * Extractors are tried in registration order during tokenization.
+   * Context-aware extractors automatically receive the tokenizer context.
+   *
+   * @param extractor - Value extractor to register
+   */
+  registerExtractor(extractor) {
+    if (isContextAwareExtractor(extractor)) {
+      extractor.setContext(createTokenizerContext(this));
+    }
+    this.extractors.push(extractor);
+  }
+  /**
+   * Register multiple value extractors at once.
+   *
+   * @param extractors - Array of value extractors to register
+   */
+  registerExtractors(extractors) {
+    for (const extractor of extractors) {
+      this.registerExtractor(extractor);
+    }
+  }
+  /**
+   * Clear all registered extractors.
+   * Returns tokenizer to legacy mode.
+   */
+  clearExtractors() {
+    this.extractors = [];
+  }
+  /**
+   * Check if this tokenizer is using extractor-based tokenization.
+   * Returns true if any extractors are registered.
+   */
+  isUsingExtractors() {
+    return this.extractors.length > 0;
+  }
+  /**
+   * Tokenize input using registered value extractors.
+   * This is the new path - extractors handle all syntax detection.
+   *
+   * @param input - Input string to tokenize
+   * @returns Token stream
+   */
+  tokenizeWithExtractors(input) {
+    const tokens = [];
+    let pos = 0;
+    while (pos < input.length) {
+      while (pos < input.length && isWhitespace(input[pos])) {
+        pos++;
+      }
+      if (pos >= input.length) break;
+      const multiWord = this.tryMultiWordKeyword(input, pos);
+      if (multiWord) {
+        tokens.push(multiWord);
+        pos = multiWord.position.end;
+        continue;
+      }
+      let extracted = false;
+      for (const extractor of this.extractors) {
+        if (extractor.canExtract(input, pos)) {
+          const result = extractor.extract(input, pos);
+          if (result) {
+            const normalized = result.metadata?.normalized;
+            const stem = result.metadata?.stem;
+            const stemConfidence = result.metadata?.stemConfidence;
+            const cleanMetadata = {};
+            if (result.metadata) {
+              for (const [key, value] of Object.entries(result.metadata)) {
+                if (key !== "normalized" && key !== "stem" && key !== "stemConfidence") {
+                  cleanMetadata[key] = value;
+                }
+              }
+            }
+            const options = {};
+            if (normalized) options.normalized = normalized;
+            if (stem) options.stem = stem;
+            if (stemConfidence !== void 0) options.stemConfidence = stemConfidence;
+            if (Object.keys(cleanMetadata).length > 0) options.metadata = cleanMetadata;
+            tokens.push(
+              createToken(
+                result.value,
+                this.classifyToken(result.value),
+                createPosition(pos, pos + result.length),
+                Object.keys(options).length > 0 ? options : void 0
+              )
+            );
+            pos += result.length;
+            extracted = true;
+            break;
+          }
+        }
+      }
+      if (!extracted) {
+        const char = input[pos];
+        const kind = this.classifyUnknownChar(char);
+        tokens.push(createToken(char, kind, createPosition(pos, pos + 1)));
+        pos++;
+      }
+    }
+    return new TokenStreamImpl(tokens, this.language);
+  }
+  /**
+   * Classify an unknown character when no extractor matches.
+   * Provides sensible defaults for common syntax.
+   *
+   * @param char - Character to classify
+   * @returns Token kind
+   */
+  classifyUnknownChar(char) {
+    if ("()[]{},:;".includes(char)) return "punctuation";
+    if ("+-*/<>=!&|".includes(char)) return "operator";
+    return "identifier";
+  }
+  /**
+   * Check if current position is a property access (obj.prop) vs CSS selector (.active).
+   * Property access: no whitespace before '.', previous token is identifier/keyword/selector.
+   * Also detects standalone method calls: .identifier( pattern.
+   *
+   * Returns true if '.' was emitted as an operator token and pos should advance by 1.
+   * Returns false if this is a CSS selector and should be handled by trySelector().
+   */
+  tryPropertyAccess(input, pos, tokens) {
+    if (input[pos] !== ".") return false;
+    const lastToken = tokens[tokens.length - 1];
+    const hasWhitespaceBefore = lastToken && lastToken.position.end < pos;
+    const isPropertyAccess = lastToken && !hasWhitespaceBefore && (lastToken.kind === "identifier" || lastToken.kind === "keyword" || lastToken.kind === "selector");
+    if (isPropertyAccess) {
+      tokens.push(createToken(".", "operator", createPosition(pos, pos + 1)));
+      return true;
+    }
+    const methodStart = pos + 1;
+    let methodEnd = methodStart;
+    while (methodEnd < input.length && isAsciiIdentifierChar(input[methodEnd])) {
+      methodEnd++;
+    }
+    if (methodEnd < input.length && input[methodEnd] === "(") {
+      tokens.push(createToken(".", "operator", createPosition(pos, pos + 1)));
+      return true;
+    }
+    return false;
+  }
+  /**
+   * Initialize keyword mappings from a language profile.
+   * Builds a list of native→english mappings from:
+   * - profile.keywords (primary + alternatives)
+   * - profile.references (me, it, you, etc.)
+   * - profile.roleMarkers (into, from, with, etc.)
+   *
+   * Results are sorted longest-first for greedy matching (important for non-space languages).
+   * Extras take precedence over profile entries when there are duplicates.
+   *
+   * @param profile - Language profile containing keyword translations
+   * @param extras - Additional keyword entries to include (literals, positional, events)
+   */
+  initializeKeywordsFromProfile(profile, extras = []) {
+    const keywordMap = /* @__PURE__ */ new Map();
+    this.rawExtraEntries = extras;
+    if (profile.keywords) {
+      for (const [normalized, translation] of Object.entries(profile.keywords)) {
+        keywordMap.set(translation.primary, {
+          native: translation.primary,
+          normalized: translation.normalized || normalized
+        });
+        if (translation.alternatives) {
+          for (const alt of translation.alternatives) {
+            keywordMap.set(alt, {
+              native: alt,
+              normalized: translation.normalized || normalized
+            });
+          }
+        }
+      }
+    }
+    if (profile.references) {
+      for (const [normalized, native] of Object.entries(profile.references)) {
+        keywordMap.set(native, { native, normalized });
+      }
+      for (const canonical of Object.keys(profile.references)) {
+        if (!keywordMap.has(canonical)) {
+          keywordMap.set(canonical, { native: canonical, normalized: canonical });
+        }
+      }
+    }
+    if (profile.roleMarkers) {
+      for (const [role, marker] of Object.entries(profile.roleMarkers)) {
+        if (marker.primary) {
+          keywordMap.set(marker.primary, { native: marker.primary, normalized: role });
+        }
+        if (marker.alternatives) {
+          for (const alt of marker.alternatives) {
+            keywordMap.set(alt, { native: alt, normalized: role });
+          }
+        }
+      }
+    }
+    if (profile.possessive?.keywords) {
+      for (const [native, normalized] of Object.entries(profile.possessive.keywords)) {
+        keywordMap.set(native, { native, normalized });
+      }
+    }
+    for (const evt of ENGLISH_DOM_EVENT_NAMES) {
+      if (!keywordMap.has(evt)) {
+        keywordMap.set(evt, { native: evt, normalized: evt });
+      }
+    }
+    for (const extra of extras) {
+      keywordMap.set(extra.native, extra);
+    }
+    this.profileKeywords = Array.from(keywordMap.values()).sort(
+      (a, b) => b.native.length - a.native.length
+    );
+    this.multiWordKeywords = this.profileKeywords.filter(
+      (k) => k.native.includes(" ") && !MARKER_CONCEPT_NORMALIZEDS.has(k.normalized)
+    );
+    this.profileKeywordMap = /* @__PURE__ */ new Map();
+    for (const keyword of this.profileKeywords) {
+      this.profileKeywordMap.set(keyword.native.toLowerCase(), keyword);
+      const normalized = this.removeDiacritics(keyword.native);
+      if (normalized !== keyword.native && !this.profileKeywordMap.has(normalized.toLowerCase())) {
+        this.profileKeywordMap.set(normalized.toLowerCase(), keyword);
+      }
+    }
+  }
+  /**
+   * Remove diacritical marks from a word for normalization.
+   * Primarily for Arabic (shadda, fatha, kasra, damma, sukun, etc.)
+   * but could be extended for other languages.
+   *
+   * @param word - Word to normalize
+   * @returns Word without diacritics
+   */
+  removeDiacritics(word) {
+    return word.replace(/[\u064B-\u0652\u0670]/g, "");
+  }
+  /**
+   * Try to match a keyword from profile at the current position.
+   * Uses longest-first greedy matching (important for non-space languages).
+   *
+   * @param input - Input string
+   * @param pos - Current position
+   * @returns Token if matched, null otherwise
+   */
+  tryProfileKeyword(input, pos) {
+    for (const entry of this.profileKeywords) {
+      if (input.slice(pos).startsWith(entry.native)) {
+        return createToken(
+          entry.native,
+          "keyword",
+          createPosition(pos, pos + entry.native.length),
+          entry.normalized
+        );
+      }
+    }
+    return null;
+  }
+  /**
+   * Match the longest multi-word (space-containing) profile keyword at `pos`,
+   * requiring the match to end at a word boundary. The profile-driven
+   * counterpart of the per-language hardcoded compound lists (the hindi and
+   * vietnamese keyword extractors). Returns a keyword token (with the normalized
+   * form) or null. Case-sensitive against the stored native form, mirroring
+   * `tryProfileKeyword`/`isKeywordStart` (the i18n dicts emit a fixed surface
+   * case). No-op when `multiWordKeywords` is empty (no-space/CJK languages).
+   *
+   * @param input - Input string
+   * @param pos - Current position (must be a token-start boundary)
+   * @param isWordChar - End-boundary predicate (defaults to Unicode letter/digit/_)
+   */
+  tryMultiWordKeyword(input, pos, isWordChar = (ch) => /[\p{L}\p{N}_]/u.test(ch)) {
+    if (this.multiWordKeywords.length === 0) return null;
+    const rest = input.slice(pos);
+    for (const entry of this.multiWordKeywords) {
+      if (!rest.startsWith(entry.native)) continue;
+      const after = input[pos + entry.native.length];
+      if (after !== void 0 && isWordChar(after)) continue;
+      return createToken(
+        entry.native,
+        "keyword",
+        createPosition(pos, pos + entry.native.length),
+        entry.normalized
+      );
+    }
+    return null;
+  }
+  /**
+   * Check if the remaining input starts with any known keyword.
+   * Useful for non-space languages to detect word boundaries.
+   *
+   * @param input - Input string
+   * @param pos - Current position
+   * @returns true if a keyword starts at this position
+   */
+  isKeywordStart(input, pos) {
+    const remaining = input.slice(pos);
+    return this.profileKeywords.some((entry) => remaining.startsWith(entry.native));
+  }
+  /**
+   * Check if a known keyword starts at the given position AND ends at a word
+   * boundary (end of input or a non-word character).
+   *
+   * Space-delimited languages must use this (not `isKeywordStart`) for
+   * word-walk break checks: the keyword table includes English canonical
+   * fallbacks (me, it, you, …), so a raw `startsWith` check splits any native
+   * word with an embedded fallback mid-word (e.g. Quechua ñit'iy contains
+   * "it"). CJK/no-space tokenizers rely on mid-text keyword starts and must
+   * keep using `isKeywordStart`.
+   *
+   * @param input - Input string
+   * @param pos - Current position
+   * @param isWordChar - Language-specific word-character predicate; pass the
+   *   tokenizer's letter classifier so e.g. the Quechua glottal apostrophe
+   *   counts as part of a word. Defaults to Unicode letters/digits/underscore.
+   * @returns true if a keyword starts here and is not followed by a word char
+   */
+  isKeywordStartAtBoundary(input, pos, isWordChar = (ch) => /[\p{L}\p{N}_]/u.test(ch)) {
+    const remaining = input.slice(pos);
+    return this.profileKeywords.some((entry) => {
+      if (!remaining.startsWith(entry.native)) return false;
+      const after = input[pos + entry.native.length];
+      return after === void 0 || !isWordChar(after);
+    });
+  }
+  /**
+   * Look up a keyword by native word (case-insensitive).
+   * O(1) lookup using the keyword map.
+   *
+   * @param native - Native word to look up
+   * @returns KeywordEntry if found, undefined otherwise
+   */
+  lookupKeyword(native) {
+    return this.profileKeywordMap.get(native.toLowerCase());
+  }
+  /**
+   * Check if a word is a known keyword (case-insensitive).
+   * O(1) lookup using the keyword map.
+   *
+   * @param native - Native word to check
+   * @returns true if the word is a keyword
+   */
+  isKeyword(native) {
+    return this.profileKeywordMap.has(native.toLowerCase());
+  }
+  /**
+   * Set the morphological normalizer for this tokenizer.
+   */
+  setNormalizer(normalizer) {
+    this.normalizer = normalizer;
+  }
+  /**
+   * Try to normalize a word using the morphological normalizer.
+   * Returns null if no normalizer is set or normalization fails.
+   *
+   * Note: We don't check isNormalizable() here because the individual tokenizers
+   * historically called normalize() directly without that check. The normalize()
+   * method itself handles returning noChange() for words that can't be normalized.
+   */
+  tryNormalize(word) {
+    if (!this.normalizer) return null;
+    const result = this.normalizer.normalize(word);
+    if (result.stem !== word && result.confidence >= 0.7) {
+      return result;
+    }
+    return null;
+  }
+  /**
+   * Try morphological normalization and keyword lookup.
+   *
+   * If the word can be normalized to a stem that matches a known keyword,
+   * returns a keyword token with morphological metadata (stem, stemConfidence).
+   *
+   * This is the common pattern for handling conjugated verbs across languages:
+   * 1. Normalize the word (e.g., "toggled" → "toggle")
+   * 2. Look up the stem in the keyword map
+   * 3. Create a token with both the original form and stem metadata
+   *
+   * @param word - The word to normalize and look up
+   * @param startPos - Start position for the token
+   * @param endPos - End position for the token
+   * @returns Token if stem matches a keyword, null otherwise
+   */
+  tryMorphKeywordMatch(word, startPos, endPos) {
+    const result = this.tryNormalize(word);
+    if (!result) return null;
+    const stemEntry = this.lookupKeyword(result.stem);
+    if (!stemEntry) return null;
+    const tokenOptions = {
+      normalized: stemEntry.normalized,
+      stem: result.stem,
+      stemConfidence: result.confidence
+    };
+    return createToken(word, "keyword", createPosition(startPos, endPos), tokenOptions);
+  }
+  /**
+   * Try to extract a CSS selector at the current position.
+   */
+  trySelector(input, pos) {
+    const selector = extractCssSelector(input, pos);
+    if (selector) {
+      return createToken(selector, "selector", createPosition(pos, pos + selector.length));
+    }
+    return null;
+  }
+  /**
+   * Try to extract an event modifier at the current position.
+   * Event modifiers are .once, .debounce(N), .throttle(N), .queue(strategy)
+   */
+  tryEventModifier(input, pos) {
+    if (input[pos] !== ".") {
+      return null;
+    }
+    const match = input.slice(pos).match(/^\.(?:once|debounce|throttle|queue)(?:\(([^)]+)\))?(?:\s|$|\.)/);
+    if (!match) {
+      return null;
+    }
+    const fullMatch = match[0].replace(/(\s|\.)$/, "");
+    const modifierName = fullMatch.slice(1).split("(")[0];
+    const value = match[1];
+    const token = createToken(
+      fullMatch,
+      "event-modifier",
+      createPosition(pos, pos + fullMatch.length)
+    );
+    return {
+      ...token,
+      metadata: {
+        modifierName,
+        value: value ? modifierName === "queue" ? value : parseInt(value, 10) : void 0
+      }
+    };
+  }
+  /**
+   * Try to extract a string literal at the current position.
+   */
+  tryString(input, pos) {
+    const literal = extractStringLiteral(input, pos);
+    if (literal) {
+      return createToken(literal, "literal", createPosition(pos, pos + literal.length));
+    }
+    return null;
+  }
+  /**
+   * Try to extract a number at the current position.
+   */
+  tryNumber(input, pos) {
+    const number = extractNumber(input, pos);
+    if (number) {
+      return createToken(number, "literal", createPosition(pos, pos + number.length));
+    }
+    return null;
+  }
+  /**
+   * Try to match a time unit from a list of patterns.
+   *
+   * @param input - Input string
+   * @param pos - Position after the number
+   * @param timeUnits - Array of time unit mappings (native pattern → standard suffix)
+   * @param skipWhitespace - Whether to skip whitespace before time unit (default: false)
+   * @returns Object with matched suffix and new position, or null if no match
+   */
+  tryMatchTimeUnit(input, pos, timeUnits, skipWhitespace = false) {
+    let unitPos = pos;
+    if (skipWhitespace) {
+      while (unitPos < input.length && isWhitespace(input[unitPos])) {
+        unitPos++;
+      }
+    }
+    const remaining = input.slice(unitPos);
+    for (const unit of timeUnits) {
+      const candidate = remaining.slice(0, unit.length);
+      const matches = unit.caseInsensitive ? candidate.toLowerCase() === unit.pattern.toLowerCase() : candidate === unit.pattern;
+      if (matches) {
+        if (unit.notFollowedBy) {
+          const nextChar = remaining[unit.length] || "";
+          if (nextChar === unit.notFollowedBy) continue;
+        }
+        if (unit.checkBoundary) {
+          const nextChar = remaining[unit.length] || "";
+          if (isAsciiIdentifierChar(nextChar)) continue;
+        }
+        return { suffix: unit.suffix, endPos: unitPos + unit.length };
+      }
+    }
+    return null;
+  }
+  /**
+   * Parse a base number (sign, integer, decimal) without time units.
+   * Returns the number string and end position.
+   *
+   * @param input - Input string
+   * @param startPos - Start position
+   * @param allowSign - Whether to allow +/- sign (default: true)
+   * @returns Object with number string and end position, or null
+   */
+  parseBaseNumber(input, startPos, allowSign = true) {
+    let pos = startPos;
+    let number = "";
+    if (allowSign && (input[pos] === "-" || input[pos] === "+")) {
+      number += input[pos++];
+    }
+    if (pos >= input.length || !isDigit(input[pos])) {
+      return null;
+    }
+    while (pos < input.length && isDigit(input[pos])) {
+      number += input[pos++];
+    }
+    if (pos < input.length && input[pos] === ".") {
+      number += input[pos++];
+      while (pos < input.length && isDigit(input[pos])) {
+        number += input[pos++];
+      }
+    }
+    if (!number || number === "-" || number === "+") return null;
+    return { number, endPos: pos };
+  }
+  /**
+   * Try to extract a number with native language time units.
+   *
+   * This is a template method that handles the common pattern:
+   * 1. Parse the base number (sign, integer, decimal)
+   * 2. Try to match native language time units
+   * 3. Fall back to standard time units (ms, s, m, h)
+   *
+   * @param input - Input string
+   * @param pos - Start position
+   * @param nativeTimeUnits - Language-specific time unit mappings
+   * @param options - Configuration options
+   * @returns Token if number found, null otherwise
+   */
+  tryNumberWithTimeUnits(input, pos, nativeTimeUnits, options = {}) {
+    const { allowSign = true, skipWhitespace = false } = options;
+    const baseResult = this.parseBaseNumber(input, pos, allowSign);
+    if (!baseResult) return null;
+    let { number, endPos } = baseResult;
+    const allUnits = [...nativeTimeUnits, ..._BaseTokenizer.STANDARD_TIME_UNITS];
+    const timeMatch = this.tryMatchTimeUnit(input, endPos, allUnits, skipWhitespace);
+    if (timeMatch) {
+      number += timeMatch.suffix;
+      endPos = timeMatch.endPos;
+    }
+    return createToken(number, "literal", createPosition(pos, endPos));
+  }
+  /**
+   * Try to extract a URL at the current position.
+   * Handles /path, ./path, ../path, //domain.com, http://, https://
+   */
+  tryUrl(input, pos) {
+    const url = extractUrl(input, pos);
+    if (url) {
+      return createToken(url, "url", createPosition(pos, pos + url.length));
+    }
+    return null;
+  }
+  /**
+   * Try to extract a variable reference (:varname) at the current position.
+   * In hyperscript, :x refers to a local variable named x.
+   */
+  tryVariableRef(input, pos) {
+    if (input[pos] !== ":") return null;
+    if (pos + 1 >= input.length) return null;
+    if (!isAsciiIdentifierChar(input[pos + 1])) return null;
+    let endPos = pos + 1;
+    while (endPos < input.length && isAsciiIdentifierChar(input[endPos])) {
+      endPos++;
+    }
+    const varRef = input.slice(pos, endPos);
+    return createToken(varRef, "identifier", createPosition(pos, endPos));
+  }
+  /**
+   * Try to extract an operator or punctuation token at the current position.
+   * Handles two-character operators (==, !=, etc.) and single-character operators.
+   */
+  tryOperator(input, pos) {
+    const twoChar = input.slice(pos, pos + 2);
+    if (["==", "!=", "<=", ">=", "&&", "||", "->"].includes(twoChar)) {
+      return createToken(twoChar, "operator", createPosition(pos, pos + 2));
+    }
+    const oneChar = input[pos];
+    if (["<", ">", "!", "+", "-", "*", "/", "="].includes(oneChar)) {
+      return createToken(oneChar, "operator", createPosition(pos, pos + 1));
+    }
+    if (["(", ")", "{", "}", ",", ";", ":"].includes(oneChar)) {
+      return createToken(oneChar, "punctuation", createPosition(pos, pos + 1));
+    }
+    return null;
+  }
+  /**
+   * Try to match a multi-character particle from a list.
+   *
+   * Used by languages like Japanese, Korean, and Chinese that have
+   * multi-character particles (e.g., Japanese から, まで, より).
+   *
+   * @param input - Input string
+   * @param pos - Current position
+   * @param particles - Array of multi-character particles to match
+   * @returns Token if matched, null otherwise
+   */
+  tryMultiCharParticle(input, pos, particles) {
+    for (const particle of particles) {
+      if (input.slice(pos, pos + particle.length) === particle) {
+        return createToken(particle, "particle", createPosition(pos, pos + particle.length));
+      }
+    }
+    return null;
+  }
+};
+/**
+ * Configuration for native language time units.
+ * Maps patterns to their standard suffix (ms, s, m, h).
+ */
+_BaseTokenizer.STANDARD_TIME_UNITS = [
+  { pattern: "ms", suffix: "ms", length: 2 },
+  { pattern: "s", suffix: "s", length: 1, checkBoundary: true },
+  { pattern: "m", suffix: "m", length: 1, checkBoundary: true, notFollowedBy: "s" },
+  { pattern: "h", suffix: "h", length: 1, checkBoundary: true }
+];
+var BaseTokenizer = _BaseTokenizer;
+function createSimpleTokenizer(config) {
+  const {
+    language,
+    direction = "ltr",
+    keywords,
+    keywordExtras,
+    keywordProfile,
+    includeOperators = false,
+    caseInsensitive = true,
+    customExtractors
+  } = config;
+  const keywordSet = new Set(caseInsensitive ? keywords.map((k) => k.toLowerCase()) : keywords);
+  class SimpleTokenizer extends BaseTokenizer {
+    constructor() {
+      super();
+      this.language = language;
+      this.direction = direction;
+      if (customExtractors) {
+        this.registerExtractors(customExtractors);
+      }
+      this.registerExtractors(getDefaultExtractors());
+      if (keywordProfile) {
+        this.initializeKeywordsFromProfile(keywordProfile, keywordExtras);
+      }
+    }
+    classifyToken(token) {
+      const lookup = caseInsensitive ? token.toLowerCase() : token;
+      if (keywordSet.has(lookup)) return "keyword";
+      if (this.isKeyword(token)) return "keyword";
+      if (/^\d/.test(token)) return "literal";
+      if (/^['"]/.test(token)) return "literal";
+      if (includeOperators && SIMPLE_TOKENIZER_OPERATOR_SET.has(token)) return "operator";
+      return "identifier";
+    }
+  }
+  return new SimpleTokenizer();
+}
+
+// src/multilingual/builders.ts
+function mergeRoleMarkers(slice, vocab) {
+  const merged = {};
+  const add = (role, marker) => {
+    if (!marker?.primary) {
+      delete merged[role];
+      return;
+    }
+    merged[role] = {
+      primary: marker.primary,
+      ...marker.alternatives?.length && { alternatives: [...marker.alternatives] },
+      ...marker.position && { position: marker.position }
+    };
+  };
+  for (const [role, marker] of Object.entries(slice.roleMarkers ?? {})) add(role, marker);
+  for (const [role, marker] of Object.entries(vocab.roleMarkerOverrides ?? {})) add(role, marker);
+  return merged;
+}
+function buildPatternProfile(slice, vocab) {
+  const keywords = {};
+  for (const [action, translation] of Object.entries(vocab.keywords)) {
+    keywords[action] = {
+      primary: translation.primary,
+      ...translation.alternatives?.length && { alternatives: [...translation.alternatives] }
+    };
+  }
+  const roleMarkers = mergeRoleMarkers(slice, vocab);
+  return {
+    code: slice.code,
+    wordOrder: slice.wordOrder,
+    keywords,
+    ...Object.keys(roleMarkers).length > 0 && { roleMarkers }
+  };
+}
+function defaultCaseInsensitive(script) {
+  return script === void 0 || script === "latin" || script === "cyrillic";
+}
+function buildDomainTokenizer(slice, vocab, options = {}) {
+  const roleMarkers = mergeRoleMarkers(slice, vocab);
+  const keywords = /* @__PURE__ */ new Set();
+  for (const translation of Object.values(vocab.keywords)) {
+    keywords.add(translation.primary);
+    for (const alt of translation.alternatives ?? []) keywords.add(alt);
+  }
+  for (const marker of Object.values(roleMarkers)) {
+    keywords.add(marker.primary);
+    for (const alt of marker.alternatives ?? []) keywords.add(alt);
+  }
+  for (const particle of slice.tokenization?.particles ?? []) keywords.add(particle);
+  for (const extra of vocab.tokenizerKeywords ?? []) keywords.add(extra);
+  const profileKeywords = {};
+  for (const [action, translation] of Object.entries(vocab.keywords)) {
+    profileKeywords[action] = {
+      primary: translation.primary,
+      ...translation.alternatives?.length && { alternatives: [...translation.alternatives] },
+      normalized: translation.normalized ?? action
+    };
+  }
+  const keywordProfile = {
+    keywords: profileKeywords,
+    ...Object.keys(roleMarkers).length > 0 && { roleMarkers }
+  };
+  const customExtractors = [
+    ...options.customExtractors ?? [],
+    ...slice.script === "latin" ? [new LatinExtendedIdentifierExtractor()] : []
+  ];
+  return createSimpleTokenizer({
+    language: slice.code,
+    direction: slice.direction ?? "ltr",
+    keywords: [...keywords],
+    ...vocab.keywordExtras?.length && { keywordExtras: vocab.keywordExtras.map((e) => ({ ...e })) },
+    keywordProfile,
+    includeOperators: options.includeOperators ?? false,
+    caseInsensitive: options.caseInsensitive ?? defaultCaseInsensitive(slice.script),
+    ...customExtractors.length > 0 && { customExtractors }
+  });
+}
+function buildLanguageConfig(slice, vocab, meta = {}) {
+  const name = meta.name ?? slice.name ?? slice.code;
+  return {
+    code: slice.code,
+    name,
+    nativeName: meta.nativeName ?? slice.nativeName ?? name,
+    tokenizer: meta.tokenizer ?? buildDomainTokenizer(slice, vocab, meta.tokenizerOptions),
+    patternProfile: buildPatternProfile(slice, vocab),
+    ...meta.grammarProfile && { grammarProfile: meta.grammarProfile }
+  };
+}
+function deriveRoleMarkers(slice, roleMapping) {
+  const derived = {};
+  for (const [domainRole, semanticRole] of Object.entries(roleMapping)) {
+    const marker = slice.roleMarkers?.[semanticRole];
+    if (marker?.primary) derived[domainRole] = marker.primary;
+  }
+  return derived;
+}
+export {
+  buildDomainTokenizer,
+  buildLanguageConfig,
+  buildPatternProfile,
+  deriveRoleMarkers
+};
 //# sourceMappingURL=index.js.map