@lokascript/framework 2.7.2 → 2.9.0

package/src/api/extensions.test.ts

@@ -0,0 +1,322 @@
+/**
+ * DomainExtension: adding a command to a DSL from outside its package.
+ *
+ * Every domain used to render through a hardcoded `switch (node.action)` and
+ * generate through another, so a downstream consumer could not add a command
+ * without editing the package. This is the supported path: a schema plus one
+ * vocabulary entry per language.
+ *
+ * The command exercised here ("research") is the one lokascript-learn built by
+ * hand against 2.8.0 to prove the underlying pieces worked.
+ */
+
+import { describe, it, expect } from 'vitest';
+import {
+  createMultilingualDSL,
+  createSimpleTokenizer,
+  defineCommand,
+  defineRole,
+  type DomainExtension,
+  type ExtractionResult,
+  type LanguageTokenizer,
+  type SemanticNode,
+  type ValueExtractor,
+} from '../index';
+
+/**
+ * Keeps `#id` / `.class` a single token, as every real domain tokenizer does.
+ * Without it the `#` splits off and an SOV source role followed by a particle
+ * never matches.
+ */
+class CSSSelectorExtractor implements ValueExtractor {
+  readonly name = 'css-selector';
+
+  canExtract(input: string, position: number): boolean {
+    const char = input[position];
+    if (char !== '#' && char !== '.') return false;
+    return /[a-zA-Z_-]/.test(input[position + 1] ?? '');
+  }
+
+  extract(input: string, position: number): ExtractionResult | null {
+    let end = position + 1;
+    while (end < input.length && /[a-zA-Z0-9_-]/.test(input[end])) end++;
+    if (end === position + 1) return null;
+    return { value: input.slice(position, end), length: end - position };
+  }
+}
+
+// =============================================================================
+// A three-word-order toy DSL
+// =============================================================================
+
+const askSchema = defineCommand({
+  action: 'ask',
+  description: 'Ask a question',
+  category: 'llm',
+  primaryRole: 'patient',
+  roles: [
+    defineRole({
+      role: 'patient',
+      description: 'The question',
+      required: true,
+      expectedTypes: ['expression'],
+    }),
+    defineRole({
+      role: 'source',
+      description: 'Where to look',
+      required: true,
+      expectedTypes: ['expression'],
+      markerOverride: { en: 'from', ja: 'から', ar: 'من' },
+    }),
+  ],
+});
+
+const researchSchema = defineCommand({
+  action: 'research',
+  description: 'Research a topic',
+  category: 'llm',
+  primaryRole: 'patient',
+  roles: [
+    defineRole({
+      role: 'patient',
+      description: 'The topic',
+      required: true,
+      expectedTypes: ['expression'],
+    }),
+    defineRole({
+      role: 'source',
+      description: 'Where to look',
+      required: true,
+      expectedTypes: ['expression'],
+      markerOverride: { en: 'from', ja: 'から', ar: 'من' },
+    }),
+  ],
+});
+
+const PROFILES = {
+  en: {
+    code: 'en',
+    wordOrder: 'SVO' as const,
+    keywords: { ask: { primary: 'ask' } },
+    roleMarkers: {},
+  },
+  ja: {
+    code: 'ja',
+    wordOrder: 'SOV' as const,
+    keywords: { ask: { primary: '聞く' } },
+    roleMarkers: {},
+  },
+  ar: {
+    code: 'ar',
+    wordOrder: 'VSO' as const,
+    keywords: { ask: { primary: 'اسأل' } },
+    roleMarkers: {},
+  },
+};
+
+const research: DomainExtension = {
+  schema: researchSchema,
+  vocabulary: {
+    en: { keyword: { primary: 'research' } },
+    ja: { keyword: { primary: '調査' } },
+    ar: { keyword: { primary: 'ابحث' } },
+  },
+};
+
+function tokenizerFor(code: 'en' | 'ja' | 'ar'): LanguageTokenizer {
+  return createSimpleTokenizer({
+    language: code,
+    // Both the built-in and the extension vocabulary: a tokenizer is configured
+    // once, so it must already know the words an extension may introduce.
+    keywords: [
+      ...Object.values(PROFILES[code].keywords).map(k => k.primary),
+      'research',
+      '調査',
+      'ابحث',
+      'from',
+      'から',
+      'من',
+    ],
+    caseInsensitive: code === 'en',
+    customExtractors: [new CSSSelectorExtractor()],
+  });
+}
+
+function createToyDSL(options: { extensions?: readonly DomainExtension[] } = {}) {
+  return createMultilingualDSL({
+    name: 'Toy',
+    schemas: [askSchema],
+    languages: (['en', 'ja', 'ar'] as const).map(code => ({
+      code,
+      name: code,
+      nativeName: code,
+      tokenizer: tokenizerFor(code),
+      patternProfile: PROFILES[code],
+    })),
+    codeGenerator: {
+      generate: (node: SemanticNode) => `BASE:${node.action}`,
+    },
+    ...(options.extensions && { extensions: options.extensions }),
+  });
+}
+
+function makeNode(action: string, roles: Record<string, string>): SemanticNode {
+  const rolesMap = new Map<string, { type: 'expression'; raw: string }>();
+  for (const [k, v] of Object.entries(roles)) rolesMap.set(k, { type: 'expression', raw: v });
+  return { kind: 'command', action, roles: rolesMap };
+}
+
+// =============================================================================
+// Tests
+// =============================================================================
+
+describe('DomainExtension', () => {
+  describe('parsing', () => {
+    it('parses the extension command in an SVO language', () => {
+      const dsl = createToyDSL({ extensions: [research] });
+      const node = dsl.parse('research "climate" from #wiki', 'en');
+      expect(node.action).toBe('research');
+      expect(node.roles.get('patient')).toBeDefined();
+      expect(node.roles.get('source')).toBeDefined();
+    });
+
+    it('parses the extension command in an SOV language', () => {
+      const dsl = createToyDSL({ extensions: [research] });
+      const node = dsl.parse('"climate" #wiki から 調査', 'ja');
+      expect(node.action).toBe('research');
+    });
+
+    it('parses the extension command in a VSO language', () => {
+      const dsl = createToyDSL({ extensions: [research] });
+      const node = dsl.parse('ابحث "climate" من #wiki', 'ar');
+      expect(node.action).toBe('research');
+    });
+
+    it('does not parse the extension command without the extension', () => {
+      const dsl = createToyDSL();
+      expect(() => dsl.parse('research "climate" from #wiki', 'en')).toThrow(/No pattern matched/);
+    });
+
+    it('reports the extension action as supported by explicit syntax', () => {
+      const dsl = createToyDSL({ extensions: [research] });
+      const node = dsl.parse('[research patient:climate source:#wiki]', 'en');
+      expect(node.action).toBe('research');
+    });
+  });
+
+  describe('rendering', () => {
+    const dsl = createToyDSL({ extensions: [research] });
+    const node = makeNode('research', { patient: '"climate"', source: '#wiki' });
+
+    it('renders SVO from the schema alone', () => {
+      expect(dsl.render?.(node, 'en')).toBe('research "climate" from #wiki');
+    });
+
+    it('renders verb-final for SOV', () => {
+      expect(dsl.render?.(node, 'ja')).toBe('"climate" #wiki から 調査');
+    });
+
+    it('renders verb-initial for VSO', () => {
+      expect(dsl.render?.(node, 'ar')).toBe('ابحث "climate" من #wiki');
+    });
+
+    it('prefers an extension-supplied renderer', () => {
+      const custom = createToyDSL({
+        extensions: [{ ...research, render: () => 'CUSTOM' }],
+      });
+      expect(custom.render?.(node, 'en')).toBe('CUSTOM');
+    });
+
+    it('returns null for an action with neither renderer nor schema', () => {
+      expect(dsl.render?.(makeNode('nonsense', {}), 'en')).toBeNull();
+    });
+
+    it('prefers the domain renderer over the schema fallback for built-ins', () => {
+      const withDomainRenderer = createMultilingualDSL({
+        name: 'Toy',
+        schemas: [askSchema],
+        languages: [
+          {
+            code: 'en',
+            name: 'en',
+            nativeName: 'en',
+            tokenizer: tokenizerFor('en'),
+            patternProfile: PROFILES.en,
+          },
+        ],
+        renderer: node => (node.action === 'ask' ? 'DOMAIN RENDERED' : null),
+        extensions: [{ ...research, vocabulary: { en: research.vocabulary.en } }],
+      });
+
+      expect(withDomainRenderer.render?.(makeNode('ask', { patient: 'q' }), 'en')).toBe(
+        'DOMAIN RENDERED'
+      );
+      // …while the extension still falls through to the schema renderer
+      expect(withDomainRenderer.render?.(node, 'en')).toBe('research "climate" from #wiki');
+    });
+  });
+
+  describe('compilation', () => {
+    it('uses the extension generator for the extension action', () => {
+      const dsl = createToyDSL({
+        extensions: [{ ...research, generate: node => `EXT:${node.action}` }],
+      });
+      const result = dsl.compile('research "climate" from #wiki', 'en');
+      expect(result.ok).toBe(true);
+      expect(result.code).toBe('EXT:research');
+    });
+
+    it('leaves the base generator handling built-in actions', () => {
+      const dsl = createToyDSL({
+        extensions: [{ ...research, generate: node => `EXT:${node.action}` }],
+      });
+      const result = dsl.compile('ask "q" from #wiki', 'en');
+      expect(result.ok).toBe(true);
+      expect(result.code).toBe('BASE:ask');
+    });
+
+    it('falls back to the base generator when the extension supplies none', () => {
+      const dsl = createToyDSL({ extensions: [research] });
+      const result = dsl.compile('research "climate" from #wiki', 'en');
+      expect(result.ok).toBe(true);
+      expect(result.code).toBe('BASE:research');
+    });
+  });
+
+  describe('built-in commands are unaffected', () => {
+    it('parses and compiles identically with and without extensions', () => {
+      const plain = createToyDSL();
+      const extended = createToyDSL({ extensions: [research] });
+
+      for (const [input, language] of [
+        ['ask "q" from #wiki', 'en'],
+        ['"q" #wiki から 聞く', 'ja'],
+        ['اسأل "q" من #wiki', 'ar'],
+      ] as const) {
+        expect(extended.parse(input, language).action).toBe(plain.parse(input, language).action);
+        expect(extended.compile(input, language).code).toBe(plain.compile(input, language).code);
+      }
+    });
+  });
+
+  describe('configuration errors', () => {
+    it('rejects an extension whose action collides with a built-in', () => {
+      const collision: DomainExtension = {
+        schema: defineCommand({
+          action: 'ask',
+          roles: [defineRole({ role: 'patient', required: true, expectedTypes: ['expression'] })],
+        }),
+        vocabulary: { en: { keyword: { primary: 'inquire' } } },
+      };
+      expect(() => createToyDSL({ extensions: [collision] })).toThrow(/collides/);
+    });
+
+    it('rejects vocabulary for a language the DSL does not configure', () => {
+      const unknownLanguage: DomainExtension = {
+        ...research,
+        vocabulary: { ...research.vocabulary, xx: { keyword: { primary: 'x' } } },
+      };
+      expect(() => createToyDSL({ extensions: [unknownLanguage] })).toThrow(/not configured/);
+    });
+  });
+});

package/src/core/tokenization/base-tokenizer.ts

@@ -20,6 +20,7 @@ import {
   isWhitespace,
   isDigit,
   isAsciiIdentifierChar,
+  stripOptionalDiacritics,
   TokenStreamImpl,
   type TimeUnitMapping,
   type CreateTokenOptions,
@@ -347,7 +348,61 @@ export abstract class BaseTokenizer implements LanguageTokenizer {
       }
     }
 
-    return new TokenStreamImpl(tokens, this.language);
+    return new TokenStreamImpl(this.mergeColonQualifiedNames(tokens), this.language);
+  }
+
+  /**
+   * ASCII word of the shape the English word-walker produces. Excludes `:`, so a
+   * token that already carries a qualifier never merges again — `a:b:c` yields
+   * `a:b` + `:c`, byte-matching the English extractor's single-segment merge.
+   */
+  private static readonly ASCII_WORD = /^[A-Za-z_][A-Za-z0-9_]*$/;
+
+  /** `:name` — only a variable-ref-style extractor ever emits this token shape. */
+  private static readonly COLON_QUALIFIER = /^:[A-Za-z_][A-Za-z0-9_]*$/;
+
+  /**
+   * Fuse `name` + `:qualifier` into ONE identifier (`draggable:start`).
+   *
+   * `:name` is hyperscript's local-variable sigil, but a colon IMMEDIATELY
+   * preceded by an identifier is a qualifier (custom event namespace), not a
+   * sigil. The English tokenizer already merges these inside
+   * EnglishKeywordExtractor; this post-pass gives the other 23 languages the
+   * same stream. Strict position adjacency is the discriminator: whitespace
+   * between the tokens (`trigger :start`) breaks `end === start`, so a spaced
+   * local-variable reference survives untouched.
+   *
+   * Self-gating for non-hyperscript tokenizers (domain DSLs): their extractor
+   * sets tokenize `:` as bare punctuation (length 1), which never matches
+   * COLON_QUALIFIER, so this pass is a no-op for them.
+   */
+  protected mergeColonQualifiedNames(tokens: LanguageToken[]): LanguageToken[] {
+    const out: LanguageToken[] = [];
+    for (const tok of tokens) {
+      const prev = out[out.length - 1];
+      // No kind gate: the English extractor merges before classification, so a
+      // word some language classifies as particle/keyword (es `a`, tr `i`)
+      // must fuse the same way. ASCII_WORD already excludes every non-word
+      // kind structurally (selectors, urls, numbers, strings, operators).
+      if (
+        prev &&
+        BaseTokenizer.ASCII_WORD.test(prev.value) &&
+        BaseTokenizer.COLON_QUALIFIER.test(tok.value) &&
+        prev.position.end === tok.position.start
+      ) {
+        const merged = prev.value + tok.value;
+        // Re-classify and drop normalized/stem/metadata — the merged word is no
+        // longer the keyword the pieces may have been (matches the en shape).
+        out[out.length - 1] = createToken(
+          merged,
+          this.classifyToken(merged),
+          createPosition(prev.position.start, tok.position.end)
+        );
+        continue;
+      }
+      out.push(tok);
+    }
+    return out;
   }
 
   /**
@@ -544,9 +599,7 @@ export abstract class BaseTokenizer implements LanguageTokenizer {
    * @returns Word without diacritics
    */
   protected removeDiacritics(word: string): string {
-    // Arabic diacritics: U+064B-U+0652 (fatha, kasra, damma, sukun, shadda, etc.)
-    // U+0670 (superscript alif)
-    return word.replace(/[\u064B-\u0652\u0670]/g, '');
+    return stripOptionalDiacritics(word);
   }
 
   /**
@@ -650,25 +703,41 @@ export abstract class BaseTokenizer implements LanguageTokenizer {
   }
 
   /**
-   * Look up a keyword by native word (case-insensitive).
+   * Look up a keyword by native word (case-insensitive, diacritic-insensitive).
    * O(1) lookup using the keyword map.
    *
+   * The map is INDEXED both with and without diacritics (see
+   * `initializeKeywordsFromProfile`), so a stripped QUERY is the other half of
+   * that: it lets a surface form carrying harakat the profile does not happen to
+   * spell still find its entry. Only consulted after the exact lookup misses, so
+   * every previously-matching word resolves byte-identically.
+   *
+   * Half-implementing this — indexing stripped but querying exact — is what made
+   * diacritized `بَدِّل` (toggle) tokenize as `kind=particle normalized=with`:
+   * `isKeyword` returned false, so the guard in `ArabicProcliticExtractor` that
+   * exists to prevent exactly that handed the word on, and the single-char `ب`
+   * bi- proclitic claimed it. A wrong CONCEPT, not a failed parse.
+   *
    * @param native - Native word to look up
    * @returns KeywordEntry if found, undefined otherwise
    */
   protected lookupKeyword(native: string): KeywordEntry | undefined {
-    return this.profileKeywordMap.get(native.toLowerCase());
+    const exact = this.profileKeywordMap.get(native.toLowerCase());
+    if (exact) return exact;
+    const stripped = this.removeDiacritics(native);
+    if (stripped === native) return undefined;
+    return this.profileKeywordMap.get(stripped.toLowerCase());
   }
 
   /**
-   * Check if a word is a known keyword (case-insensitive).
-   * O(1) lookup using the keyword map.
+   * Check if a word is a known keyword (case-insensitive, diacritic-insensitive).
+   * O(1) lookup using the keyword map. See {@link lookupKeyword}.
    *
    * @param native - Native word to check
    * @returns true if the word is a keyword
    */
   protected isKeyword(native: string): boolean {
-    return this.profileKeywordMap.has(native.toLowerCase());
+    return this.lookupKeyword(native) !== undefined;
   }
 
   /**

package/src/core/tokenization/colon-qualifier.test.ts

@@ -0,0 +1,129 @@
+/**
+ * Contract tests for BaseTokenizer.mergeColonQualifiedNames().
+ *
+ * `:name` is hyperscript's local-variable sigil, but a colon immediately
+ * preceded by an identifier is a qualifier (custom event namespace:
+ * `draggable:start`). The English tokenizer merges these inside its keyword
+ * extractor; the post-pass gives every other language the same stream. The
+ * merge must fire only on strict adjacency and only on `:name`-shaped tokens —
+ * domain DSL tokenizers (SQL, BDD, …) emit `:` as bare punctuation, which the
+ * pass must never touch.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { BaseTokenizer, createSimpleTokenizer } from './base-tokenizer';
+import type { TokenKind } from '../types';
+import type { ValueExtractor, ExtractionResult } from '../../interfaces/value-extractor';
+
+/** Minimal `:name`/`$name`/`^name` extractor (shape of semantic's VariableRefExtractor). */
+class SigilRefExtractor implements ValueExtractor {
+  readonly name = 'sigil-ref';
+
+  canExtract(input: string, position: number): boolean {
+    const ch = input[position];
+    return (
+      (ch === ':' || ch === '$' || ch === '^') &&
+      position + 1 < input.length &&
+      /[a-zA-Z_]/.test(input[position + 1])
+    );
+  }
+
+  extract(input: string, position: number): ExtractionResult | null {
+    if (!this.canExtract(input, position)) return null;
+    let length = 1;
+    while (position + length < input.length && /[a-zA-Z0-9_]/.test(input[position + length])) {
+      length++;
+    }
+    return { value: input.substring(position, position + length), length };
+  }
+}
+
+/** Minimal ASCII word extractor (shape of the per-language word walkers). */
+class WordExtractor implements ValueExtractor {
+  readonly name = 'word';
+
+  canExtract(input: string, position: number): boolean {
+    return /[a-zA-Z_]/.test(input[position]);
+  }
+
+  extract(input: string, position: number): ExtractionResult | null {
+    let length = 0;
+    while (position + length < input.length && /[a-zA-Z0-9_]/.test(input[position + length])) {
+      length++;
+    }
+    return length > 0 ? { value: input.substring(position, position + length), length } : null;
+  }
+}
+
+const KEYWORDS = new Set(['trigger', 'click']);
+
+class ProbeTokenizer extends BaseTokenizer {
+  readonly language = 'xx';
+  readonly direction = 'ltr' as const;
+
+  constructor() {
+    super();
+    this.registerExtractors([new SigilRefExtractor(), new WordExtractor()]);
+  }
+
+  classifyToken(token: string): TokenKind {
+    return KEYWORDS.has(token) ? 'keyword' : 'identifier';
+  }
+}
+
+function values(
+  tokenizer: { tokenize(input: string): { tokens: readonly { value: string }[] } },
+  input: string
+): string[] {
+  return tokenizer.tokenize(input).tokens.map(t => t.value);
+}
+
+describe('mergeColonQualifiedNames', () => {
+  const t = new ProbeTokenizer();
+
+  it('fuses identifier + :qualifier into one token', () => {
+    expect(values(t, 'trigger draggable:start')).toEqual(['trigger', 'draggable:start']);
+  });
+
+  it('re-classifies the merged token and spans both positions', () => {
+    const tokens = t.tokenize('trigger draggable:start').tokens;
+    const merged = tokens[1];
+    expect(merged.kind).toBe('identifier');
+    expect(merged.position.start).toBe('trigger '.length);
+    expect(merged.position.end).toBe('trigger draggable:start'.length);
+  });
+
+  it('fuses when the pre-colon word is a keyword (en parity: merge before lookup)', () => {
+    const tokens = t.tokenize('click:foo').tokens;
+    expect(tokens.map(x => x.value)).toEqual(['click:foo']);
+    expect(tokens[0].kind).toBe('identifier');
+  });
+
+  it('does not fuse across whitespace — spaced :name stays a local-variable ref', () => {
+    expect(values(t, 'trigger :start')).toEqual(['trigger', ':start']);
+  });
+
+  it('leaves a leading bare sigil untouched', () => {
+    expect(values(t, ':start')).toEqual([':start']);
+  });
+
+  it('merges a single segment only — a:b:c matches the English extractor', () => {
+    expect(values(t, 'a:b:c')).toEqual(['a:b', ':c']);
+  });
+
+  it('never touches $ and ^ sigils', () => {
+    expect(values(t, 'put $foo into ^bar')).toEqual(['put', '$foo', 'into', '^bar']);
+  });
+
+  it('is a no-op for domain-DSL streams where : is bare punctuation', () => {
+    const sql = createSimpleTokenizer({
+      language: 'en',
+      keywords: ['select', 'from', 'where'],
+      includeOperators: true,
+    });
+    // Named params never fuse: the colon tokenizes as length-1 punctuation,
+    // which can never match the :name qualifier shape.
+    expect(values(sql, 'WHERE x = :param')).toEqual(['WHERE', 'x', '=', ':', 'param']);
+    expect(values(sql, 'x=:param')).toEqual(['x', '=', ':', 'param']);
+  });
+});

package/src/core/tokenization/css-selector-extractor.test.ts

@@ -0,0 +1,67 @@
+import { describe, it, expect } from 'vitest';
+import { CssSelectorExtractor } from '../../interfaces/value-extractor';
+
+/**
+ * `getDefaultExtractors()` has no CSS-selector extractor, so a DSL that does not
+ * register one splits the sigil off as its own token and the role capture keeps
+ * only that sigil — `add .active to #button` parsed with patient `"."` and
+ * destination `"#"` in domain-learn, -todo, -sql and -jsx, silently, in every
+ * language. Five other domains each carried a private copy of this class; this
+ * is the shared one.
+ */
+describe('CssSelectorExtractor', () => {
+  const ex = new CssSelectorExtractor();
+
+  const extract = (input: string, pos = 0) =>
+    ex.canExtract(input, pos) ? ex.extract(input, pos) : null;
+
+  it.each([
+    ['.active', '.active'],
+    ['#button', '#button'],
+    ['.btn-primary', '.btn-primary'],
+    ['._private', '._private'],
+    ['#a1', '#a1'],
+    ['.-leading-hyphen', '.-leading-hyphen'],
+  ])('extracts %s whole', (input, expected) => {
+    expect(extract(input)).toEqual({ value: expected, length: expected.length });
+  });
+
+  it('stops at the first character that cannot be in a selector', () => {
+    expect(extract('.active to #button')).toEqual({ value: '.active', length: 7 });
+  });
+
+  it('keeps diacritics, so Latin-script class names survive', () => {
+    expect(extract('.año')).toEqual({ value: '.año', length: 4 });
+    expect(extract('#botón')).toEqual({ value: '#botón', length: 6 });
+  });
+
+  // The SOV languages write their particles flush against the value, so a
+  // Unicode-wide body turns `#buttonに` into a single token and carries the role
+  // marker inside the value — which is worse than truncating, because the marker
+  // is then missing from where the pattern expects it.
+  it.each([
+    ['#buttonに', '#button'],
+    ['.activeを', '.active'],
+    ['#button에', '#button'],
+    ['.active를', '.active'],
+    ['#button的', '#button'],
+  ])('stops at the particle in %s', (input, expected) => {
+    expect(extract(input)).toEqual({ value: expected, length: expected.length });
+  });
+
+  it.each([
+    ['.', 'a bare sigil'],
+    ['#', 'a bare sigil'],
+    ['. active', 'a sigil followed by a space'],
+    ['.1col', 'a digit — CSS identifiers cannot start with one'],
+    ['button', 'no sigil at all'],
+  ])('declines %s (%s)', input => {
+    expect(ex.canExtract(input, 0)).toBe(false);
+  });
+
+  it('declines a CJK-only class name rather than claiming the sigil alone', () => {
+    // `.追加` would otherwise extract as a bare `.`; leaving it unclaimed lets
+    // the keyword/identifier extractors see the verb.
+    expect(ex.canExtract('.追加', 0)).toBe(false);
+  });
+});

package/src/core/tokenization/extractors.ts

@@ -34,6 +34,12 @@ import {
  * Method call handling:
  * - #dialog.showModal() → stops after #dialog (method call, not compound selector)
  * - #box.active → compound selector (no parens)
+ *
+ * NOTE: intentionally diverges from the semantic package's copy
+ * (packages/semantic/src/tokenizers/extractors/css-selector.ts), which also
+ * consumes pseudo-class/pseudo-element segments (#x:hover, .a:not(.b)). This
+ * legacy version is only used by BaseTokenizer.trySelector (no semantic call
+ * sites) and stays as-is.
  */
 export function extractCssSelector(input: string, startPos: number): string | null {
   if (startPos >= input.length) return null;

package/src/core/tokenization/token-utils.ts

@@ -237,6 +237,24 @@ export function isDigit(char: string): boolean {
   return /\d/.test(char);
 }
 
+/**
+ * Strip diacritical marks that are OPTIONAL in their script — currently Arabic
+ * harakat (U+064B–U+0652: fatha, kasra, damma, sukun, shadda…) and the
+ * superscript alif (U+0670).
+ *
+ * `بدّل` and `بَدِّل` are the same word; Arabic prose writes either. So any place
+ * that compares an Arabic surface form against a declared keyword has to compare
+ * stripped, or the same word fails to match itself.
+ *
+ * Deliberately Arabic-only. Latin diacritics are NOT optional — `obtén`, `récupère`
+ * and `vá` differ from their unaccented spellings in meaning or validity — and
+ * Hebrew niqqud (U+05B0–U+05BC) is out of range too, so this is inert for every
+ * other script.
+ */
+export function stripOptionalDiacritics(word: string): string {
+  return word.replace(/[ً-ْٰ]/g, '');
+}
+
 /**
  * Check if a character is an ASCII letter.
  */