@lokascript/framework 2.7.2 → 2.9.0

package/dist/index.cjs

@@ -24,6 +24,7 @@ __export(index_exports, {
   BaseMorphologicalNormalizer: () => BaseMorphologicalNormalizer,
   BaseTokenizer: () => BaseTokenizer,
   CrossDomainDispatcher: () => CrossDomainDispatcher,
+  CssSelectorExtractor: () => CssSelectorExtractor,
   DEFAULT_OPERATORS: () => DEFAULT_OPERATORS,
   DEFAULT_PUNCTUATION: () => DEFAULT_PUNCTUATION,
   DEFAULT_REFERENCES: () => import_intent2.DEFAULT_REFERENCES,
@@ -60,6 +61,7 @@ __export(index_exports, {
   createCompoundNode: () => import_intent.createCompoundNode,
   createConditionalNode: () => import_intent.createConditionalNode,
   createDiagnosticCollector: () => import_intent7.createDiagnosticCollector,
+  createDomainRenderer: () => createDomainRenderer,
   createEventHandlerNode: () => import_intent.createEventHandlerNode,
   createExpression: () => import_intent.createExpression,
   createFlag: () => import_intent.createFlag,
@@ -151,6 +153,8 @@ __export(index_exports, {
   semanticNodeToJSON: () => semanticNodeToJSON,
   semanticNodeToRuntimeAST: () => semanticNodeToRuntimeAST,
   semanticValueToAST: () => semanticValueToAST,
+  sortRolesByWordOrder: () => sortRolesByWordOrder,
+  stripOptionalDiacritics: () => stripOptionalDiacritics,
   synthesizeFromSchemas: () => synthesizeFromSchemas,
   toEnvelopeJSON: () => import_intent5.toEnvelopeJSON,
   toJSONL: () => toJSONL,
@@ -1477,10 +1481,12 @@ function buildTokens(schema, profile, keyword, alternatives) {
       addRoleWithMarker(tokens, role, profile);
     }
   } else if (profile.wordOrder === "SOV") {
+    const preVerb = [];
+    const postVerb = [];
     for (const role of sortedRoles) {
-      addRoleWithMarker(tokens, role, profile);
+      addRoleWithMarker(role.sovSlot === "postVerb" ? postVerb : preVerb, role, profile);
     }
-    tokens.push(keywordToken);
+    tokens.push(...preVerb, keywordToken, ...postVerb);
   } else if (profile.wordOrder === "VSO") {
     tokens.push(keywordToken);
     for (const role of sortedRoles) {
@@ -1563,28 +1569,28 @@ function buildExtractionRules(schema, profile) {
   return rules;
 }
 function buildFormatString(schema, profile, keyword) {
-  const parts = [];
-  if (profile.wordOrder === "SVO" || profile.wordOrder === "VSO") {
-    parts.push(keyword);
-  }
-  for (const role of schema.roles) {
+  const isSOV = profile.wordOrder === "SOV";
+  const preVerb = [];
+  const postVerb = [];
+  const sortedRoles = sortRolesByWordOrder(schema.roles, profile.wordOrder);
+  for (const role of sortedRoles) {
     const marker = getMarkerForRole(role, profile);
     const roleName = `{${role.role}}`;
+    const bucket = isSOV && role.sovSlot === "postVerb" ? postVerb : preVerb;
     if (marker) {
       const markerInfo = profile.roleMarkers?.[role.role];
       if (markerInfo?.position === "after") {
-        parts.push(`${roleName} ${marker}`);
+        bucket.push(`${roleName} ${marker}`);
       } else {
-        parts.push(`${marker} ${roleName}`);
+        bucket.push(`${marker} ${roleName}`);
       }
     } else {
-      parts.push(roleName);
+      bucket.push(roleName);
     }
   }
-  if (profile.wordOrder === "SOV") {
-    parts.push(keyword);
-  }
-  return parts.join(" ");
+  return (isSOV ? [...preVerb, keyword, ...postVerb] : [keyword, ...preVerb, ...postVerb]).join(
+    " "
+  );
 }
 function generatePatternVariants(schema, profile, config = defaultConfig) {
   const patterns = [];
@@ -1592,6 +1598,119 @@ function generatePatternVariants(schema, profile, config = defaultConfig) {
   return patterns;
 }
 
+// src/generation/renderer.ts
+function lookupKeyword(keywords, action, language) {
+  return keywords[action]?.[language] ?? action;
+}
+function lookupMarker(markers, marker, language) {
+  return markers[marker]?.[language] ?? marker;
+}
+function buildPhrase(...parts) {
+  return parts.filter(Boolean).join(" ");
+}
+function buildTablesFromProfiles(schemas, profiles) {
+  const keywords = {};
+  const markers = {};
+  for (const profile of profiles) {
+    for (const [action, kw] of Object.entries(profile.keywords)) {
+      if (!keywords[action]) keywords[action] = {};
+      keywords[action][profile.code] = kw.primary;
+    }
+  }
+  for (const schema of schemas) {
+    for (const role of schema.roles) {
+      if (role.markerOverride) {
+        const markerKey = role.role;
+        if (!markers[markerKey]) markers[markerKey] = {};
+        for (const [lang, markerText] of Object.entries(role.markerOverride)) {
+          markers[markerKey][lang] = markerText;
+        }
+      }
+    }
+  }
+  for (const profile of profiles) {
+    if (profile.roleMarkers) {
+      for (const [markerKey, markerDef] of Object.entries(profile.roleMarkers)) {
+        if (!markers[markerKey]) markers[markerKey] = {};
+        markers[markerKey][profile.code] = markerDef.primary;
+      }
+    }
+  }
+  return { keywords, markers };
+}
+function detectWordOrders(profiles) {
+  const sovLanguages = /* @__PURE__ */ new Set();
+  const vsoLanguages = /* @__PURE__ */ new Set();
+  for (const profile of profiles) {
+    if (profile.wordOrder === "SOV") sovLanguages.add(profile.code);
+    if (profile.wordOrder === "VSO") vsoLanguages.add(profile.code);
+  }
+  return { sovLanguages, vsoLanguages };
+}
+function createSchemaRenderer(schemas, profiles) {
+  const tables = buildRenderTables(schemas, profiles);
+  return {
+    render(node, language) {
+      return renderFromSchema(node, language, tables) ?? node.action;
+    }
+  };
+}
+function createDomainRenderer(config) {
+  const { schemas, profiles, overrides } = config;
+  let tables;
+  return (node, language) => {
+    const override = overrides?.[node.action];
+    if (override) return override(node, language);
+    tables ?? (tables = buildRenderTables(schemas, profiles));
+    return renderFromSchema(node, language, tables);
+  };
+}
+function buildRenderTables(schemas, profiles) {
+  const { keywords, markers } = buildTablesFromProfiles(schemas, profiles);
+  const { sovLanguages } = detectWordOrders(profiles);
+  const markerPositions = {};
+  for (const profile of profiles) {
+    for (const [role, markerDef] of Object.entries(profile.roleMarkers ?? {})) {
+      if (!markerDef.position) continue;
+      markerPositions[role] ?? (markerPositions[role] = {});
+      markerPositions[role][profile.code] = markerDef.position;
+    }
+  }
+  const schemaMap = /* @__PURE__ */ new Map();
+  for (const s of schemas) schemaMap.set(s.action, s);
+  return { keywords, markers, markerPositions, sovLanguages, schemaMap };
+}
+function renderFromSchema(node, language, tables) {
+  const schema = tables.schemaMap.get(node.action);
+  if (!schema) return null;
+  const keyword = lookupKeyword(tables.keywords, node.action, language);
+  const isSOV = tables.sovLanguages.has(language);
+  const ordered = sortRolesByWordOrder([...schema.roles], isSOV ? "SOV" : "SVO");
+  const preVerb = [];
+  const postVerb = [];
+  for (const role of ordered) {
+    let value = (0, import_intent.extractRoleValue)(node, role.role);
+    if (!value && role.default !== void 0) {
+      value = String((0, import_intent.extractValue)(role.default));
+    }
+    if (!value) continue;
+    if (role.quoteMultiword && /\s/.test(value) && !/^(["']).*\1$/.test(value)) {
+      value = `"${value}"`;
+    }
+    const markerText = role.renderOverride?.[language] ?? role.markerOverride?.[language] ?? role.renderOverride?.["*"] ?? tables.markers[role.role]?.[language];
+    const markerPosition = role.markerPositionOverride?.[language] ?? role.markerPosition ?? tables.markerPositions[role.role]?.[language] ?? (isSOV ? "after" : "before");
+    const bucket = isSOV && role.sovSlot === "postVerb" ? postVerb : preVerb;
+    if (markerText && markerPosition === "before") {
+      bucket.push(markerText, value);
+    } else if (markerText) {
+      bucket.push(value, markerText);
+    } else {
+      bucket.push(value);
+    }
+  }
+  return isSOV ? buildPhrase(...preVerb, keyword, ...postVerb) : buildPhrase(keyword, ...preVerb, ...postVerb);
+}
+
 // src/grammar/types.ts
 function reorderRoles(roles, targetOrder) {
   const result = [];
@@ -2028,6 +2147,31 @@ var LatinExtendedIdentifierExtractor = class {
     return { value: input.slice(position, end), length: end - position };
   }
 };
+var SELECTOR_BODY_CHAR = /[\p{L}\p{N}_-]/u;
+var PARTICLE_SCRIPT_CHAR = /[\p{sc=Han}\p{sc=Hiragana}\p{sc=Katakana}\p{sc=Hangul}]/u;
+function isSelectorBodyChar(char) {
+  return SELECTOR_BODY_CHAR.test(char) && !PARTICLE_SCRIPT_CHAR.test(char);
+}
+var CssSelectorExtractor = class {
+  constructor() {
+    this.name = "css-selector";
+  }
+  canExtract(input, position) {
+    const char = input[position];
+    if (char !== "#" && char !== ".") return false;
+    const next = input[position + 1];
+    if (next === void 0) return false;
+    return next === "_" || next === "-" || /\p{L}/u.test(next) && !PARTICLE_SCRIPT_CHAR.test(next);
+  }
+  extract(input, position) {
+    let end = position + 1;
+    while (end < input.length && isSelectorBodyChar(input[end])) {
+      end++;
+    }
+    if (end === position + 1) return null;
+    return { value: input.slice(position, end), length: end - position };
+  }
+};
 var WhitespaceExtractor = class {
   constructor() {
     this.name = "whitespace";
@@ -2767,10 +2911,17 @@ function semanticValueToAST(value) {
 
 // src/api/create-dsl.ts
 var DSLRegistry = class {
-  constructor(config) {
+  /**
+   * @param partiallyTranslatedActions Actions permitted to lack a keyword in
+   *   some configured languages — extension commands, which may be added for a
+   *   subset of the DSL's languages. A built-in command missing a keyword stays
+   *   an error, since that is a domain-authoring mistake.
+   */
+  constructor(config, partiallyTranslatedActions = /* @__PURE__ */ new Set()) {
     this.patterns = /* @__PURE__ */ new Map();
     this.tokenizers = /* @__PURE__ */ new Map();
     this.schemas = config.schemas;
+    this.partiallyTranslatedActions = partiallyTranslatedActions;
     for (const lang of config.languages) {
       this.registerLanguage(lang);
     }
@@ -2779,6 +2930,9 @@ var DSLRegistry = class {
     this.tokenizers.set(lang.code, lang.tokenizer);
     const patterns = [];
     for (const schema of this.schemas) {
+      if (this.partiallyTranslatedActions.has(schema.action) && !lang.patternProfile.keywords[schema.action]) {
+        continue;
+      }
       const pattern = generatePattern(schema, lang.patternProfile);
       patterns.push(pattern);
     }
@@ -2795,13 +2949,25 @@ var DSLRegistry = class {
   }
 };
 var MultilingualDSLImpl = class {
-  constructor(config, registry, transformer) {
+  constructor(config, registry, transformer, profileProvider) {
     this.registry = registry;
     this.matcher = new PatternMatcher();
     this.transformer = transformer;
+    this.profileProvider = profileProvider;
     if (config.codeGenerator) {
       this.codeGenerator = config.codeGenerator;
     }
+    if (config.renderer) {
+      this.domainRenderer = config.renderer;
+    }
+    this.extensionRenderers = /* @__PURE__ */ new Map();
+    for (const extension of config.extensions ?? []) {
+      if (extension.render) this.extensionRenderers.set(extension.schema.action, extension.render);
+    }
+    this.schemaFallbackRenderer = createDomainRenderer({
+      schemas: config.schemas,
+      profiles: config.languages.map((l) => l.patternProfile)
+    });
     const schemaMap = new Map(config.schemas.map((s) => [s.action, s]));
     this.schemaLookup = {
       getSchema: (action) => schemaMap.get(action)
@@ -2863,6 +3029,13 @@ var MultilingualDSLImpl = class {
   }
   translate(input, fromLanguage, toLanguage) {
     if ((0, import_intent3.isExplicitSyntax)(input)) return input;
+    for (const language of [fromLanguage, toLanguage]) {
+      if (!this.profileProvider.getProfile(language)) {
+        throw new Error(
+          `translate() requires a grammar profile for language "${language}", but none is configured. Set 'grammarProfile' on the LanguageConfig for "${language}" in createMultilingualDSL() (parse/validate/compile do not need it), or inject a custom 'profileProvider'.`
+        );
+      }
+    }
     return this.transformer.transform(input, fromLanguage, toLanguage);
   }
   compile(input, language) {
@@ -2891,6 +3064,18 @@ var MultilingualDSLImpl = class {
       };
     }
   }
+  render(node, language) {
+    const extensionRenderer = this.extensionRenderers.get(node.action);
+    if (extensionRenderer) {
+      const rendered = extensionRenderer(node, language);
+      if (rendered != null) return rendered;
+    }
+    if (this.domainRenderer) {
+      const rendered = this.domainRenderer(node, language);
+      if (rendered != null) return rendered;
+    }
+    return this.schemaFallbackRenderer(node, language);
+  }
   getSupportedLanguages() {
     return this.registry.getSupportedLanguages();
   }
@@ -2915,15 +3100,80 @@ function createDefaultProfileProvider(config) {
   }
   return new InMemoryProfileProvider(profiles);
 }
+function applyExtensions(config) {
+  const extensions = config.extensions;
+  if (!extensions || extensions.length === 0) return config;
+  const configuredLanguages = new Set(config.languages.map((l) => l.code));
+  const actions = new Set(config.schemas.map((s) => s.action));
+  for (const extension of extensions) {
+    const { action } = extension.schema;
+    if (actions.has(action)) {
+      throw new Error(
+        `Extension command "${action}" collides with a command this DSL already defines. Pick a different action name.`
+      );
+    }
+    actions.add(action);
+    for (const code of Object.keys(extension.vocabulary)) {
+      if (!configuredLanguages.has(code)) {
+        throw new Error(
+          `Extension command "${action}" supplies vocabulary for language "${code}", which is not configured on this DSL. Configured languages: ${[...configuredLanguages].join(", ")}.`
+        );
+      }
+    }
+  }
+  const schemas = [...config.schemas, ...extensions.map((e) => e.schema)];
+  const languages = config.languages.map((lang) => {
+    const keywords = { ...lang.patternProfile.keywords };
+    let roleMarkers = lang.patternProfile.roleMarkers;
+    for (const extension of extensions) {
+      const vocabulary = extension.vocabulary[lang.code];
+      if (!vocabulary) continue;
+      keywords[extension.schema.action] = { ...vocabulary.keyword };
+      if (vocabulary.roleMarkers) {
+        roleMarkers = { ...roleMarkers, ...vocabulary.roleMarkers };
+      }
+    }
+    return {
+      ...lang,
+      patternProfile: {
+        ...lang.patternProfile,
+        keywords,
+        ...roleMarkers !== void 0 && { roleMarkers }
+      }
+    };
+  });
+  const extensionGenerators = new Map(
+    extensions.filter((e) => e.generate).map((e) => [e.schema.action, e.generate])
+  );
+  const baseGenerator = config.codeGenerator;
+  const codeGenerator = extensionGenerators.size > 0 ? {
+    generate(node) {
+      const generate = extensionGenerators.get(node.action);
+      if (generate) return generate(node);
+      if (!baseGenerator) {
+        throw new Error(`No code generator for action "${node.action}"`);
+      }
+      return baseGenerator.generate(node);
+    }
+  } : baseGenerator;
+  return {
+    ...config,
+    schemas,
+    languages,
+    ...codeGenerator !== void 0 && { codeGenerator }
+  };
+}
 function createMultilingualDSL(config) {
-  const dictionary = config.dictionary ?? createDefaultDictionary(config);
-  const profileProvider = config.profileProvider ?? createDefaultProfileProvider(config);
+  const effectiveConfig = applyExtensions(config);
+  const extensionActions = new Set((config.extensions ?? []).map((e) => e.schema.action));
+  const dictionary = effectiveConfig.dictionary ?? createDefaultDictionary(effectiveConfig);
+  const profileProvider = effectiveConfig.profileProvider ?? createDefaultProfileProvider(effectiveConfig);
   const transformer = new GrammarTransformer({
     dictionary,
     profileProvider
   });
-  const registry = new DSLRegistry(config);
-  return new MultilingualDSLImpl(config, registry, transformer);
+  const registry = new DSLRegistry(effectiveConfig, extensionActions);
+  return new MultilingualDSLImpl(effectiveConfig, registry, transformer, profileProvider);
 }
 
 // src/prompts/prompt-generator.ts
@@ -3010,7 +3260,9 @@ function buildProtocolSection() {
 3. No spaces around the colon in role:value pairs
 4. Strings with spaces must be quoted: \`patient:"hello world"\`
 5. Selectors start with \`#\`, \`.\`, \`[\`, \`@\`, or \`*\`
-6. Output must be valid bracket syntax: \`[action role:value ...]\``;
+6. A selector containing a space, a combinator (\`>\` \`+\` \`~\`), or a comma must use a selector literal: \`patient:<ul > li/>\`, \`patient:<.a, .b/>\`
+7. Inside a structural role (\`body\`, \`then\`, \`else\`, \`condition\`, \`loop-body\`, \`variable\`, \`catch\`, \`finally\`) a \`[...]\` value is always a nested command. Write an attribute selector there as \`condition:<[data-active]/>\`
+8. Output must be valid bracket syntax: \`[action role:value ...]\``;
   return {
     id: "protocol",
     title: "LSE Protocol",
@@ -3023,6 +3275,7 @@ function buildValueTypeSection() {
 
 | Type | Syntax | Example |
 |------|--------|---------|
+| Selector literal | Delimited by \`<\` and \`/>\` | \`<ul > li/>\`, \`<.a, .b/>\`, \`<[data-id]/>\` |
 | Selector | Starts with \`#\` \`.\` \`[\` \`@\` \`*\` | \`#button\`, \`.active\`, \`[data-id]\` |
 | String | Quoted with \`"\` or \`'\` | \`"hello world"\`, \`'json'\` |
 | Boolean | Exact: \`true\` / \`false\` | \`visible:true\` |
@@ -3891,6 +4144,11 @@ var DomainRegistry = class {
         rendered = typeof renderer === "function" ? renderer(node, to) : renderer.render(node, to);
       } catch {
       }
+    } else if (dsl.render) {
+      try {
+        rendered = dsl.render(node, to);
+      } catch {
+      }
     }
     const roles = {};
     for (const [key, value] of node.roles) {
@@ -4448,96 +4706,6 @@ async function registryToAOTBackends(registry) {
 // src/schema/command-schema.ts
 var import_intent6 = require("@lokascript/intent");
 
-// src/generation/renderer.ts
-function lookupKeyword(keywords, action, language) {
-  return keywords[action]?.[language] ?? action;
-}
-function lookupMarker(markers, marker, language) {
-  return markers[marker]?.[language] ?? marker;
-}
-function buildPhrase(...parts) {
-  return parts.filter(Boolean).join(" ");
-}
-function buildTablesFromProfiles(schemas, profiles) {
-  const keywords = {};
-  const markers = {};
-  for (const profile of profiles) {
-    for (const [action, kw] of Object.entries(profile.keywords)) {
-      if (!keywords[action]) keywords[action] = {};
-      keywords[action][profile.code] = kw.primary;
-    }
-  }
-  for (const schema of schemas) {
-    for (const role of schema.roles) {
-      if (role.markerOverride) {
-        const markerKey = role.role;
-        if (!markers[markerKey]) markers[markerKey] = {};
-        for (const [lang, markerText] of Object.entries(role.markerOverride)) {
-          markers[markerKey][lang] = markerText;
-        }
-      }
-    }
-  }
-  for (const profile of profiles) {
-    if (profile.roleMarkers) {
-      for (const [markerKey, markerDef] of Object.entries(profile.roleMarkers)) {
-        if (!markers[markerKey]) markers[markerKey] = {};
-        markers[markerKey][profile.code] = markerDef.primary;
-      }
-    }
-  }
-  return { keywords, markers };
-}
-function detectWordOrders(profiles) {
-  const sovLanguages = /* @__PURE__ */ new Set();
-  const vsoLanguages = /* @__PURE__ */ new Set();
-  for (const profile of profiles) {
-    if (profile.wordOrder === "SOV") sovLanguages.add(profile.code);
-    if (profile.wordOrder === "VSO") vsoLanguages.add(profile.code);
-  }
-  return { sovLanguages, vsoLanguages };
-}
-function createSchemaRenderer(schemas, profiles) {
-  const { keywords, markers } = buildTablesFromProfiles(schemas, profiles);
-  const { sovLanguages } = detectWordOrders(profiles);
-  const schemaMap = /* @__PURE__ */ new Map();
-  for (const s of schemas) schemaMap.set(s.action, s);
-  return {
-    render(node, language) {
-      const schema = schemaMap.get(node.action);
-      if (!schema) return node.action;
-      const keyword = lookupKeyword(keywords, node.action, language);
-      const isSOV = sovLanguages.has(language);
-      const roleParts = [];
-      for (const role of schema.roles) {
-        const value = (0, import_intent.extractRoleValue)(node, role.role);
-        if (!value && !role.required) continue;
-        const markerText = role.markerOverride?.[language] ?? markers[role.role]?.[language] ?? void 0;
-        roleParts.push({
-          ...markerText != null && { marker: markerText },
-          value: value || "",
-          role
-        });
-      }
-      const parts = [];
-      if (isSOV) {
-        for (const rp of roleParts) {
-          if (rp.value) parts.push(rp.value);
-          if (rp.marker) parts.push(rp.marker);
-        }
-        parts.push(keyword);
-      } else {
-        parts.push(keyword);
-        for (const rp of roleParts) {
-          if (rp.marker) parts.push(rp.marker);
-          if (rp.value) parts.push(rp.value);
-        }
-      }
-      return buildPhrase(...parts);
-    }
-  };
-}
-
 // src/generation/diagnostics.ts
 var import_intent7 = require("@lokascript/intent");
 
@@ -4647,6 +4815,9 @@ function isQuote(char) {
 function isDigit(char) {
   return /\d/.test(char);
 }
+function stripOptionalDiacritics(word) {
+  return word.replace(/[ً-ْٰ]/g, "");
+}
 function isAsciiLetter(char) {
   return /[a-zA-Z]/.test(char);
 }
@@ -5218,7 +5389,39 @@ var _BaseTokenizer = class _BaseTokenizer {
         pos++;
       }
     }
-    return new TokenStreamImpl(tokens, this.language);
+    return new TokenStreamImpl(this.mergeColonQualifiedNames(tokens), this.language);
+  }
+  /**
+   * Fuse `name` + `:qualifier` into ONE identifier (`draggable:start`).
+   *
+   * `:name` is hyperscript's local-variable sigil, but a colon IMMEDIATELY
+   * preceded by an identifier is a qualifier (custom event namespace), not a
+   * sigil. The English tokenizer already merges these inside
+   * EnglishKeywordExtractor; this post-pass gives the other 23 languages the
+   * same stream. Strict position adjacency is the discriminator: whitespace
+   * between the tokens (`trigger :start`) breaks `end === start`, so a spaced
+   * local-variable reference survives untouched.
+   *
+   * Self-gating for non-hyperscript tokenizers (domain DSLs): their extractor
+   * sets tokenize `:` as bare punctuation (length 1), which never matches
+   * COLON_QUALIFIER, so this pass is a no-op for them.
+   */
+  mergeColonQualifiedNames(tokens) {
+    const out = [];
+    for (const tok of tokens) {
+      const prev = out[out.length - 1];
+      if (prev && _BaseTokenizer.ASCII_WORD.test(prev.value) && _BaseTokenizer.COLON_QUALIFIER.test(tok.value) && prev.position.end === tok.position.start) {
+        const merged = prev.value + tok.value;
+        out[out.length - 1] = createToken(
+          merged,
+          this.classifyToken(merged),
+          createPosition(prev.position.start, tok.position.end)
+        );
+        continue;
+      }
+      out.push(tok);
+    }
+    return out;
   }
   /**
    * Classify an unknown character when no extractor matches.
@@ -5351,7 +5554,7 @@ var _BaseTokenizer = class _BaseTokenizer {
    * @returns Word without diacritics
    */
   removeDiacritics(word) {
-    return word.replace(/[\u064B-\u0652\u0670]/g, "");
+    return stripOptionalDiacritics(word);
   }
   /**
    * Try to match a keyword from profile at the current position.
@@ -5442,24 +5645,40 @@ var _BaseTokenizer = class _BaseTokenizer {
     });
   }
   /**
-   * Look up a keyword by native word (case-insensitive).
+   * Look up a keyword by native word (case-insensitive, diacritic-insensitive).
    * O(1) lookup using the keyword map.
    *
+   * The map is INDEXED both with and without diacritics (see
+   * `initializeKeywordsFromProfile`), so a stripped QUERY is the other half of
+   * that: it lets a surface form carrying harakat the profile does not happen to
+   * spell still find its entry. Only consulted after the exact lookup misses, so
+   * every previously-matching word resolves byte-identically.
+   *
+   * Half-implementing this — indexing stripped but querying exact — is what made
+   * diacritized `بَدِّل` (toggle) tokenize as `kind=particle normalized=with`:
+   * `isKeyword` returned false, so the guard in `ArabicProcliticExtractor` that
+   * exists to prevent exactly that handed the word on, and the single-char `ب`
+   * bi- proclitic claimed it. A wrong CONCEPT, not a failed parse.
+   *
    * @param native - Native word to look up
    * @returns KeywordEntry if found, undefined otherwise
    */
   lookupKeyword(native) {
-    return this.profileKeywordMap.get(native.toLowerCase());
+    const exact = this.profileKeywordMap.get(native.toLowerCase());
+    if (exact) return exact;
+    const stripped = this.removeDiacritics(native);
+    if (stripped === native) return void 0;
+    return this.profileKeywordMap.get(stripped.toLowerCase());
   }
   /**
-   * Check if a word is a known keyword (case-insensitive).
-   * O(1) lookup using the keyword map.
+   * Check if a word is a known keyword (case-insensitive, diacritic-insensitive).
+   * O(1) lookup using the keyword map. See {@link lookupKeyword}.
    *
    * @param native - Native word to check
    * @returns true if the word is a keyword
    */
   isKeyword(native) {
-    return this.profileKeywordMap.has(native.toLowerCase());
+    return this.lookupKeyword(native) !== void 0;
   }
   /**
    * Set the morphological normalizer for this tokenizer.
@@ -5724,6 +5943,14 @@ var _BaseTokenizer = class _BaseTokenizer {
     return null;
   }
 };
+/**
+ * ASCII word of the shape the English word-walker produces. Excludes `:`, so a
+ * token that already carries a qualifier never merges again — `a:b:c` yields
+ * `a:b` + `:c`, byte-matching the English extractor's single-segment merge.
+ */
+_BaseTokenizer.ASCII_WORD = /^[A-Za-z_][A-Za-z0-9_]*$/;
+/** `:name` — only a variable-ref-style extractor ever emits this token shape. */
+_BaseTokenizer.COLON_QUALIFIER = /^:[A-Za-z_][A-Za-z0-9_]*$/;
 /**
  * Configuration for native language time units.
  * Maps patterns to their standard suffix (ms, s, m, h).
@@ -6429,6 +6656,7 @@ function accumulateIndented(statements, config) {
   BaseMorphologicalNormalizer,
   BaseTokenizer,
   CrossDomainDispatcher,
+  CssSelectorExtractor,
   DEFAULT_OPERATORS,
   DEFAULT_PUNCTUATION,
   DEFAULT_REFERENCES,
@@ -6465,6 +6693,7 @@ function accumulateIndented(statements, config) {
   createCompoundNode,
   createConditionalNode,
   createDiagnosticCollector,
+  createDomainRenderer,
   createEventHandlerNode,
   createExpression,
   createFlag,
@@ -6556,6 +6785,8 @@ function accumulateIndented(statements, config) {
   semanticNodeToJSON,
   semanticNodeToRuntimeAST,
   semanticValueToAST,
+  sortRolesByWordOrder,
+  stripOptionalDiacritics,
   synthesizeFromSchemas,
   toEnvelopeJSON,
   toJSONL,