@lokascript/framework 2.5.1 → 2.7.0

package/dist/index.d.ts

@@ -28,6 +28,8 @@ export { GrammarTransformer } from './grammar/transformer';
 export type { TransformerConfig } from './grammar/transformer';
 export { reorderRoles, insertMarkers, joinTokens } from './grammar/types';
 export type { LanguageProfile, PatternTransform, ParsedElement, WordOrder, GrammaticalMarker, AdpositionType, } from './grammar/types';
+export { buildPatternProfile, buildDomainTokenizer, buildLanguageConfig, deriveRoleMarkers, } from './multilingual';
+export type { GrammarProfileSlice, DomainVocabulary, DomainKeywordTranslation, DomainKeywordEntry, RoleMarkerSlice, TokenizationSlice, VerbSlice, DomainTokenizerOptions, LanguageConfigMeta, } from './multilingual';
 export * from './core';
 export type { ActionType, SemanticRole as SemanticRoleType, SemanticValue, SemanticNode, CommandSemanticNode, EventHandlerSemanticNode, ConditionalSemanticNode, CompoundSemanticNode, LoopSemanticNode, LiteralValue, SelectorValue, ReferenceValue, PropertyPathValue, ExpressionValue, SemanticMetadata, SourcePosition, LanguageToken, TokenStream, LanguageTokenizer, LanguagePattern, Annotation, ProtocolDiagnostic, AsyncVariant, MatchArm, LSEEnvelope, } from './core/types';
 export type { CommandSchema, RoleSpec } from './schema';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAGH,cAAc,OAAO,CAAC;AAGtB,cAAc,OAAO,CAAC;AAGtB,cAAc,cAAc,CAAC;AAG7B,cAAc,UAAU,CAAC;AACzB,cAAc,cAAc,CAAC;AAG7B,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAC3D,YAAY,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC1E,YAAY,EACV,eAAe,EACf,gBAAgB,EAChB,aAAa,EACb,SAAS,EACT,iBAAiB,EACjB,cAAc,GACf,MAAM,iBAAiB,CAAC;AAGzB,cAAc,QAAQ,CAAC;AAGvB,YAAY,EACV,UAAU,EACV,YAAY,IAAI,gBAAgB,EAChC,aAAa,EACb,YAAY,EACZ,mBAAmB,EACnB,wBAAwB,EACxB,uBAAuB,EACvB,oBAAoB,EACpB,gBAAgB,EAChB,YAAY,EACZ,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,eAAe,EACf,gBAAgB,EAChB,cAAc,EACd,aAAa,EACb,WAAW,EACX,iBAAiB,EACjB,eAAe,EAEf,UAAU,EACV,kBAAkB,EAClB,YAAY,EACZ,QAAQ,EACR,WAAW,GACZ,MAAM,cAAc,CAAC;AAEtB,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAGxD,YAAY,EACV,uBAAuB,EACvB,YAAY,EACZ,WAAW,EACX,cAAc,GACf,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,aAAa,EACb,YAAY,EACZ,WAAW,EACX,uBAAuB,EACvB,gBAAgB,EAChB,oBAAoB,GACrB,MAAM,uBAAuB,CAAC;AAG/B,YAAY,EACV,kBAAkB,EAClB,UAAU,EACV,gBAAgB,EAChB,iBAAiB,EACjB,mBAAmB,EACnB,iBAAiB,GAClB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,yBAAyB,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAElG,YAAY,EACV,cAAc,EACd,SAAS,EACT,eAAe,EACf,aAAa,EACb,gBAAgB,EAChB,aAAa,EACb,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,kBAAkB,EAClB,cAAc,EACd,iBAAiB,GAClB,MAAM,OAAO,CAAC;AAEf,OAAO,EAAE,cAAc,EAAE,qBAAqB,EAAE,MAAM,OAAO,CAAC;AAG9D,OAAO,EACL,aAAa,EACb,cAAc,EACd,eAAe,EACf,kBAAkB,EAClB,gBAAgB,EAChB,iBAAiB,EACjB,sBAAsB,EACtB,qBAAqB,EACrB,kBAAkB,EAClB,cAAc,EACd,YAAY,EACZ,gBAAgB,EAEhB,aAAa,EACb,eAAe,EACf,eAAe,GAChB,MAAM,cAAc,CAAC;AAEtB,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAC;AAC3E,YAAY,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAC;AAGhF,OAAO,EAAE,2BAA2B,EAAE,MAAM,gDAAgD,CAAC;AAC7F,YAAY,EACV,iBAAiB,EACjB,gBAAgB,GACjB,MAAM,gDAAgD,CAAC;AAGxD,cAAc,MAAM,CAAC;AAGrB,cAAc,WAAW,CAAC;AAG1B,cAAc,YAAY,CAAC;AAG3B,cAAc,YAAY,CAAC;AAG3B,OAAO,EAAE,0BAA0B,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AACzF,YAAY,EACV,oBAAoB,EACpB,oBAAoB,EACpB,oBAAoB,EACpB,eAAe,EACf,cAAc,EACd,cAAc,EACd,WAAW,EACX,WAAW,EACX,UAAU,EACV,aAAa,GACd,MAAM,2BAA2B,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAGH,cAAc,OAAO,CAAC;AAGtB,cAAc,OAAO,CAAC;AAGtB,cAAc,cAAc,CAAC;AAG7B,cAAc,UAAU,CAAC;AACzB,cAAc,cAAc,CAAC;AAG7B,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAC3D,YAAY,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC1E,YAAY,EACV,eAAe,EACf,gBAAgB,EAChB,aAAa,EACb,SAAS,EACT,iBAAiB,EACjB,cAAc,GACf,MAAM,iBAAiB,CAAC;AAIzB,OAAO,EACL,mBAAmB,EACnB,oBAAoB,EACpB,mBAAmB,EACnB,iBAAiB,GAClB,MAAM,gBAAgB,CAAC;AACxB,YAAY,EACV,mBAAmB,EACnB,gBAAgB,EAChB,wBAAwB,EACxB,kBAAkB,EAClB,eAAe,EACf,iBAAiB,EACjB,SAAS,EACT,sBAAsB,EACtB,kBAAkB,GACnB,MAAM,gBAAgB,CAAC;AAGxB,cAAc,QAAQ,CAAC;AAGvB,YAAY,EACV,UAAU,EACV,YAAY,IAAI,gBAAgB,EAChC,aAAa,EACb,YAAY,EACZ,mBAAmB,EACnB,wBAAwB,EACxB,uBAAuB,EACvB,oBAAoB,EACpB,gBAAgB,EAChB,YAAY,EACZ,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,eAAe,EACf,gBAAgB,EAChB,cAAc,EACd,aAAa,EACb,WAAW,EACX,iBAAiB,EACjB,eAAe,EAEf,UAAU,EACV,kBAAkB,EAClB,YAAY,EACZ,QAAQ,EACR,WAAW,GACZ,MAAM,cAAc,CAAC;AAEtB,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAGxD,YAAY,EACV,uBAAuB,EACvB,YAAY,EACZ,WAAW,EACX,cAAc,GACf,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,aAAa,EACb,YAAY,EACZ,WAAW,EACX,uBAAuB,EACvB,gBAAgB,EAChB,oBAAoB,GACrB,MAAM,uBAAuB,CAAC;AAG/B,YAAY,EACV,kBAAkB,EAClB,UAAU,EACV,gBAAgB,EAChB,iBAAiB,EACjB,mBAAmB,EACnB,iBAAiB,GAClB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,yBAAyB,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAElG,YAAY,EACV,cAAc,EACd,SAAS,EACT,eAAe,EACf,aAAa,EACb,gBAAgB,EAChB,aAAa,EACb,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,kBAAkB,EAClB,cAAc,EACd,iBAAiB,GAClB,MAAM,OAAO,CAAC;AAEf,OAAO,EAAE,cAAc,EAAE,qBAAqB,EAAE,MAAM,OAAO,CAAC;AAG9D,OAAO,EACL,aAAa,EACb,cAAc,EACd,eAAe,EACf,kBAAkB,EAClB,gBAAgB,EAChB,iBAAiB,EACjB,sBAAsB,EACtB,qBAAqB,EACrB,kBAAkB,EAClB,cAAc,EACd,YAAY,EACZ,gBAAgB,EAEhB,aAAa,EACb,eAAe,EACf,eAAe,GAChB,MAAM,cAAc,CAAC;AAEtB,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAC;AAC3E,YAAY,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAC;AAGhF,OAAO,EAAE,2BAA2B,EAAE,MAAM,gDAAgD,CAAC;AAC7F,YAAY,EACV,iBAAiB,EACjB,gBAAgB,GACjB,MAAM,gDAAgD,CAAC;AAGxD,cAAc,MAAM,CAAC;AAGrB,cAAc,WAAW,CAAC;AAG1B,cAAc,YAAY,CAAC;AAG3B,cAAc,YAAY,CAAC;AAG3B,OAAO,EAAE,0BAA0B,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AACzF,YAAY,EACV,oBAAoB,EACpB,oBAAoB,EACpB,oBAAoB,EACpB,eAAe,EACf,cAAc,EACd,cAAc,EACd,WAAW,EACX,WAAW,EACX,UAAU,EACV,aAAa,GACd,MAAM,2BAA2B,CAAC"}

package/dist/index.js

@@ -1908,7 +1908,8 @@ function createTokenizerContext(tokenizer) {
     direction: tokenizer.direction,
     lookupKeyword: tokenizer.lookupKeyword.bind(tokenizer),
     isKeyword: tokenizer.isKeyword.bind(tokenizer),
-    isKeywordStart: tokenizer.isKeywordStart.bind(tokenizer)
+    isKeywordStart: tokenizer.isKeywordStart.bind(tokenizer),
+    ...tokenizer.isKeywordStartAtBoundary ? { isKeywordStartAtBoundary: tokenizer.isKeywordStartAtBoundary.bind(tokenizer) } : {}
   };
   if (tokenizer.normalizer) {
     return { ...ctx, normalizer: tokenizer.normalizer };
@@ -4879,36 +4880,96 @@ function withDefaultExtractors(tokenizer) {
   return tokenizer;
 }
 
-// src/core/tokenization/char-classifiers.ts
-function createUnicodeRangeClassifier(ranges) {
-  return (char) => {
-    const code = char.charCodeAt(0);
-    return ranges.some(([start, end]) => code >= start && code <= end);
-  };
-}
-function combineClassifiers(...classifiers) {
-  return (char) => classifiers.some((fn) => fn(char));
-}
-function createLatinCharClassifiers(letterPattern) {
-  const isLetter = (char) => letterPattern.test(char);
-  const isIdentifierChar = (char) => isLetter(char) || /[0-9_-]/.test(char);
-  return { isLetter, isIdentifierChar };
-}
-
 // src/core/tokenization/base-tokenizer.ts
 var SIMPLE_TOKENIZER_OPERATOR_SET = new Set(DEFAULT_OPERATORS);
+var MARKER_CONCEPT_NORMALIZEDS = /* @__PURE__ */ new Set([
+  // Role-marker role names (profile.roleMarkers normalizeds)
+  "patient",
+  "destination",
+  "source",
+  "style",
+  "event",
+  "eventMarker",
+  "agent",
+  "goal",
+  "manner",
+  // Prepositional / positional modifier concepts matched via the role mechanism
+  // (profile.keywords "Modifiers"). `before`/`after`/`until` are intentionally
+  // NOT here — they are pattern literals (see the note above).
+  "into",
+  "from",
+  "to",
+  "with",
+  "at",
+  "of",
+  "as",
+  "by",
+  "in",
+  "on",
+  "over",
+  "under",
+  "between",
+  "through",
+  "without"
+]);
+var ENGLISH_DOM_EVENT_NAMES = [
+  "click",
+  "dblclick",
+  "input",
+  "change",
+  "submit",
+  "keydown",
+  "keyup",
+  "keypress",
+  "mousedown",
+  "mouseup",
+  "mouseover",
+  "mouseout",
+  "mouseenter",
+  "mouseleave",
+  "mousemove",
+  "pointerdown",
+  "pointerup",
+  "pointermove",
+  "focus",
+  "blur",
+  "load",
+  "resize",
+  "scroll"
+];
 var _BaseTokenizer = class _BaseTokenizer {
   constructor() {
     /** Keywords derived from profile, sorted longest-first for greedy matching */
     this.profileKeywords = [];
+    /**
+     * Space-containing profile keywords (multi-word phrases), longest-first.
+     * Used by `tryMultiWordKeyword` so natural spaced forms (hi `मेल खाता`,
+     * vi `chuyển đổi`, es `tecla abajo`, …) tokenize as ONE keyword — the
+     * profile-driven replacement for the per-language hardcoded compound lists.
+     * Empty for no-space (CJK) languages, so they are unaffected.
+     */
+    this.multiWordKeywords = [];
     /** Map for O(1) keyword lookups by lowercase native word */
     this.profileKeywordMap = /* @__PURE__ */ new Map();
+    /**
+     * The raw EXTRAS list passed to initializeKeywordsFromProfile, kept pre-dedup.
+     * The keyword map is keyed by native word with last-wins insertion, so a
+     * duplicate native word inside the extras silently shadows the earlier entry
+     * (e.g. a `nächste→closest` entry shadowing `nächste→next` broke German
+     * positional expressions). Exposed so consistency tests can detect such
+     * intra-extras collisions, which are invisible in the deduplicated map.
+     */
+    this.rawExtraEntries = [];
     /**
      * Pluggable value extractors for domain-specific syntax.
      * When registered, BaseTokenizer will use extractor-based tokenization instead of legacy methods.
      */
     this.extractors = [];
   }
+  /** Raw extras as passed in, pre-dedup — for consistency tests. */
+  getExtraKeywordEntries() {
+    return this.rawExtraEntries;
+  }
   /**
    * Tokenize input string to token stream.
    * Delegates to extractor-based tokenization if extractors are registered,
@@ -4977,6 +5038,12 @@ var _BaseTokenizer = class _BaseTokenizer {
         pos++;
       }
       if (pos >= input.length) break;
+      const multiWord = this.tryMultiWordKeyword(input, pos);
+      if (multiWord) {
+        tokens.push(multiWord);
+        pos = multiWord.position.end;
+        continue;
+      }
       let extracted = false;
       for (const extractor of this.extractors) {
         if (extractor.canExtract(input, pos)) {
@@ -5076,6 +5143,7 @@ var _BaseTokenizer = class _BaseTokenizer {
    */
   initializeKeywordsFromProfile(profile, extras = []) {
     const keywordMap = /* @__PURE__ */ new Map();
+    this.rawExtraEntries = extras;
     if (profile.keywords) {
       for (const [normalized2, translation] of Object.entries(profile.keywords)) {
         keywordMap.set(translation.primary, {
@@ -5119,12 +5187,20 @@ var _BaseTokenizer = class _BaseTokenizer {
         keywordMap.set(native, { native, normalized: normalized2 });
       }
     }
+    for (const evt of ENGLISH_DOM_EVENT_NAMES) {
+      if (!keywordMap.has(evt)) {
+        keywordMap.set(evt, { native: evt, normalized: evt });
+      }
+    }
     for (const extra of extras) {
       keywordMap.set(extra.native, extra);
     }
     this.profileKeywords = Array.from(keywordMap.values()).sort(
       (a, b) => b.native.length - a.native.length
     );
+    this.multiWordKeywords = this.profileKeywords.filter(
+      (k) => k.native.includes(" ") && !MARKER_CONCEPT_NORMALIZEDS.has(k.normalized)
+    );
     this.profileKeywordMap = /* @__PURE__ */ new Map();
     for (const keyword of this.profileKeywords) {
       this.profileKeywordMap.set(keyword.native.toLowerCase(), keyword);
@@ -5166,6 +5242,35 @@ var _BaseTokenizer = class _BaseTokenizer {
     }
     return null;
   }
+  /**
+   * Match the longest multi-word (space-containing) profile keyword at `pos`,
+   * requiring the match to end at a word boundary. The profile-driven
+   * counterpart of the per-language hardcoded compound lists (the hindi and
+   * vietnamese keyword extractors). Returns a keyword token (with the normalized
+   * form) or null. Case-sensitive against the stored native form, mirroring
+   * `tryProfileKeyword`/`isKeywordStart` (the i18n dicts emit a fixed surface
+   * case). No-op when `multiWordKeywords` is empty (no-space/CJK languages).
+   *
+   * @param input - Input string
+   * @param pos - Current position (must be a token-start boundary)
+   * @param isWordChar - End-boundary predicate (defaults to Unicode letter/digit/_)
+   */
+  tryMultiWordKeyword(input, pos, isWordChar = (ch) => /[\p{L}\p{N}_]/u.test(ch)) {
+    if (this.multiWordKeywords.length === 0) return null;
+    const rest = input.slice(pos);
+    for (const entry of this.multiWordKeywords) {
+      if (!rest.startsWith(entry.native)) continue;
+      const after = input[pos + entry.native.length];
+      if (after !== void 0 && isWordChar(after)) continue;
+      return createToken(
+        entry.native,
+        "keyword",
+        createPosition(pos, pos + entry.native.length),
+        entry.normalized
+      );
+    }
+    return null;
+  }
   /**
    * Check if the remaining input starts with any known keyword.
    * Useful for non-space languages to detect word boundaries.
@@ -5178,6 +5283,32 @@ var _BaseTokenizer = class _BaseTokenizer {
     const remaining = input.slice(pos);
     return this.profileKeywords.some((entry) => remaining.startsWith(entry.native));
   }
+  /**
+   * Check if a known keyword starts at the given position AND ends at a word
+   * boundary (end of input or a non-word character).
+   *
+   * Space-delimited languages must use this (not `isKeywordStart`) for
+   * word-walk break checks: the keyword table includes English canonical
+   * fallbacks (me, it, you, …), so a raw `startsWith` check splits any native
+   * word with an embedded fallback mid-word (e.g. Quechua ñit'iy contains
+   * "it"). CJK/no-space tokenizers rely on mid-text keyword starts and must
+   * keep using `isKeywordStart`.
+   *
+   * @param input - Input string
+   * @param pos - Current position
+   * @param isWordChar - Language-specific word-character predicate; pass the
+   *   tokenizer's letter classifier so e.g. the Quechua glottal apostrophe
+   *   counts as part of a word. Defaults to Unicode letters/digits/underscore.
+   * @returns true if a keyword starts here and is not followed by a word char
+   */
+  isKeywordStartAtBoundary(input, pos, isWordChar = (ch) => /[\p{L}\p{N}_]/u.test(ch)) {
+    const remaining = input.slice(pos);
+    return this.profileKeywords.some((entry) => {
+      if (!remaining.startsWith(entry.native)) return false;
+      const after = input[pos + entry.native.length];
+      return after === void 0 || !isWordChar(after);
+    });
+  }
   /**
    * Look up a keyword by native word (case-insensitive).
    * O(1) lookup using the keyword map.
@@ -5510,6 +5641,119 @@ function createSimpleTokenizer(config) {
   return new SimpleTokenizer();
 }
 
+// src/multilingual/builders.ts
+function mergeRoleMarkers(slice, vocab) {
+  const merged = {};
+  const add = (role, marker) => {
+    if (!marker?.primary) {
+      delete merged[role];
+      return;
+    }
+    merged[role] = {
+      primary: marker.primary,
+      ...marker.alternatives?.length && { alternatives: [...marker.alternatives] },
+      ...marker.position && { position: marker.position }
+    };
+  };
+  for (const [role, marker] of Object.entries(slice.roleMarkers ?? {})) add(role, marker);
+  for (const [role, marker] of Object.entries(vocab.roleMarkerOverrides ?? {})) add(role, marker);
+  return merged;
+}
+function buildPatternProfile(slice, vocab) {
+  const keywords = {};
+  for (const [action, translation] of Object.entries(vocab.keywords)) {
+    keywords[action] = {
+      primary: translation.primary,
+      ...translation.alternatives?.length && { alternatives: [...translation.alternatives] }
+    };
+  }
+  const roleMarkers = mergeRoleMarkers(slice, vocab);
+  return {
+    code: slice.code,
+    wordOrder: slice.wordOrder,
+    keywords,
+    ...Object.keys(roleMarkers).length > 0 && { roleMarkers }
+  };
+}
+function defaultCaseInsensitive(script) {
+  return script === void 0 || script === "latin" || script === "cyrillic";
+}
+function buildDomainTokenizer(slice, vocab, options = {}) {
+  const roleMarkers = mergeRoleMarkers(slice, vocab);
+  const keywords = /* @__PURE__ */ new Set();
+  for (const translation of Object.values(vocab.keywords)) {
+    keywords.add(translation.primary);
+    for (const alt of translation.alternatives ?? []) keywords.add(alt);
+  }
+  for (const marker of Object.values(roleMarkers)) {
+    keywords.add(marker.primary);
+    for (const alt of marker.alternatives ?? []) keywords.add(alt);
+  }
+  for (const particle of slice.tokenization?.particles ?? []) keywords.add(particle);
+  for (const extra of vocab.tokenizerKeywords ?? []) keywords.add(extra);
+  const profileKeywords = {};
+  for (const [action, translation] of Object.entries(vocab.keywords)) {
+    profileKeywords[action] = {
+      primary: translation.primary,
+      ...translation.alternatives?.length && { alternatives: [...translation.alternatives] },
+      normalized: translation.normalized ?? action
+    };
+  }
+  const keywordProfile = {
+    keywords: profileKeywords,
+    ...Object.keys(roleMarkers).length > 0 && { roleMarkers }
+  };
+  const customExtractors = [
+    ...options.customExtractors ?? [],
+    ...slice.script === "latin" ? [new LatinExtendedIdentifierExtractor()] : []
+  ];
+  return createSimpleTokenizer({
+    language: slice.code,
+    direction: slice.direction ?? "ltr",
+    keywords: [...keywords],
+    ...vocab.keywordExtras?.length && { keywordExtras: vocab.keywordExtras.map((e) => ({ ...e })) },
+    keywordProfile,
+    includeOperators: options.includeOperators ?? false,
+    caseInsensitive: options.caseInsensitive ?? defaultCaseInsensitive(slice.script),
+    ...customExtractors.length > 0 && { customExtractors }
+  });
+}
+function buildLanguageConfig(slice, vocab, meta = {}) {
+  const name = meta.name ?? slice.name ?? slice.code;
+  return {
+    code: slice.code,
+    name,
+    nativeName: meta.nativeName ?? slice.nativeName ?? name,
+    tokenizer: meta.tokenizer ?? buildDomainTokenizer(slice, vocab, meta.tokenizerOptions),
+    patternProfile: buildPatternProfile(slice, vocab),
+    ...meta.grammarProfile && { grammarProfile: meta.grammarProfile }
+  };
+}
+function deriveRoleMarkers(slice, roleMapping) {
+  const derived = {};
+  for (const [domainRole, semanticRole] of Object.entries(roleMapping)) {
+    const marker = slice.roleMarkers?.[semanticRole];
+    if (marker?.primary) derived[domainRole] = marker.primary;
+  }
+  return derived;
+}
+
+// src/core/tokenization/char-classifiers.ts
+function createUnicodeRangeClassifier(ranges) {
+  return (char) => {
+    const code = char.charCodeAt(0);
+    return ranges.some(([start, end]) => code >= start && code <= end);
+  };
+}
+function combineClassifiers(...classifiers) {
+  return (char) => classifiers.some((fn) => fn(char));
+}
+function createLatinCharClassifiers(letterPattern) {
+  const isLetter = (char) => letterPattern.test(char);
+  const isIdentifierChar = (char) => isLetter(char) || /[0-9_-]/.test(char);
+  return { isLetter, isIdentifierChar };
+}
+
 // src/core/tokenization/morphology/types.ts
 function noChange(word) {
   return { stem: word, confidence: 1 };
@@ -6075,7 +6319,10 @@ export {
   WhitespaceExtractor,
   accumulateBlocks,
   buildDisambiguation,
+  buildDomainTokenizer,
   buildFeedback,
+  buildLanguageConfig,
+  buildPatternProfile,
   buildPhrase,
   buildTablesFromProfiles,
   combineClassifiers,
@@ -6106,6 +6353,7 @@ export {
   createUnicodeRangeClassifier,
   defineCommand,
   defineRole,
+  deriveRoleMarkers,
   detectWordOrders,
   extractCssSelector,
   extractNumber,