@lokascript/i18n 2.5.1 → 2.7.0

package/src/grammar/types.ts

@@ -538,7 +538,28 @@ export function insertMarkers(
   const result: string[] = [];
 
   for (const element of elements) {
-    const marker = markers.find(m => m.role === element.role);
+    let marker = markers.find(m => m.role === element.role);
+
+    // A selector-shaped "event" is never a trigger. It's the dangling target of
+    // a locative `on` (`set @role to "alert" on #sr-announce`) that
+    // splitOnCommandBoundaries splits off as a headless pseudo-handler (set/put
+    // are deliberately NOT in ON_TARGET_COMMANDS — dual-destination collision).
+    // Emitting the event marker after it (ko 할 때, ja で, bn এ, tr de) plants a
+    // spurious mid-stream event anchor in the emission; suppress the marker and
+    // emit the bare selector, which body parsers skip harmlessly.
+    if (marker?.role === 'event' && /^[#.<@[]/.test(element.translated || element.value)) {
+      marker = undefined;
+    }
+
+    // A pronoun-valued "duration" is never a time expression. It's the target
+    // phrase of `take … for me` (en maps `for` → duration lexically), kept
+    // in-clause by splitOnCommandBoundaries' loop-head guard. Emitting the
+    // duration marker after it (bn জন্য — also bn's `for` loop keyword) anchors
+    // a spurious `for` command in the semantic parse; suppress the marker and
+    // emit the bare pronoun, which take-pattern matchers skip harmlessly.
+    if (marker?.role === 'duration' && /^(me|you|it)$/i.test(element.value)) {
+      marker = undefined;
+    }
 
     if (marker) {
       if (adpositionType === 'preposition') {

package/src/index.ts

@@ -220,6 +220,7 @@ export {
   turkishProfile,
   spanishProfile,
   indonesianProfile,
+  malayProfile,
   quechuaProfile,
   swahiliProfile,
   // Transformer

package/src/new-languages.test.ts

@@ -80,7 +80,7 @@ describe('New Language Support', () => {
       // TODO: Once grammar transformation is implemented, test for native word order
       expect(result).toContain('klik'); // click → klik
       expect(result).toContain('pada'); // on → pada
-      expect(result).toContain('ganti'); // toggle → ganti
+      expect(result).toContain('alihkan'); // toggle → alihkan (aligned to the semantic profile primary; `ganti` is swap's alternative)
     });
 
     it('should handle form interactions', () => {
@@ -98,7 +98,9 @@ describe('New Language Support', () => {
         from: 'en',
         to: 'id',
       });
-      expect(result).toContain('ambil'); // fetch → ambil
+      // fetch → muat (profile primary; 'ambil' is take's word — the old
+      // emission made every id fetch parse as take)
+      expect(result).toContain('muat');
       expect(result).toContain('taruh'); // put → taruh
       expect(result).toContain('hasil'); // result → hasil
     });
@@ -154,7 +156,9 @@ describe('New Language Support', () => {
         to: 'qu',
       });
       expect(result).toContain('niy'); // tell → niy
-      expect(result).toContain('aswan_kaylla'); // closest → aswan_kaylla
+      // closest → kaylla (single token the qu tokenizer recognizes; the old
+      // aswan_kaylla compound split as as+wan+_+kaylla and never parsed)
+      expect(result).toContain('kaylla');
     });
 
     it('should support pluralization', () => {

package/src/parser/parser-integration.test.ts

@@ -384,7 +384,11 @@ describe('KeywordProvider Integration', () => {
       expect(trKeywords.resolve('ve')).toBe('and');
       expect(trKeywords.resolve('veya')).toBe('or');
       expect(trKeywords.resolve('değil')).toBe('not');
-      expect(trKeywords.resolve('sonra')).toBe('then'); // dictionary uses 'sonra'
+      // 'sonra' is the positional `after` (`put X sonra Y`); `then` uses 'ardından'
+      // (disambiguated so the put-after position word no longer collides with then —
+      // mirrors zh then:'那么' vs after:'之后'). See put-tr-after / the tr then-set.
+      expect(trKeywords.resolve('sonra')).toBe('after');
+      expect(trKeywords.resolve('ardından')).toBe('then');
       expect(trKeywords.resolve('yoksa')).toBe('else');
     });
 

package/src/positional-keyword-drift.test.ts

@@ -0,0 +1,138 @@
+/**
+ * Dict positional emissions ↔ tokenizer recognition.
+ *
+ * The i18n dictionaries' `expressions` section declares what the transformer
+ * EMITS for positional/scope concepts (first/last/next/previous/closest/
+ * parent/random); the semantic tokenizers declare what the parser RECOGNIZES.
+ * Nothing else keeps the two in sync — Swahili's dict emitted `ijayo` for
+ * `next` while the tokenizer only knew `ifuatayo`, so `put X into next <sel>`
+ * failed to parse outright (fixed in #338).
+ *
+ * For every language and concept this test tokenizes the dict emission and
+ * requires it to normalize back to that concept. Same convention as
+ * command-primary-roles.test.ts: the test imports @lokascript/semantic, but
+ * tests don't ship in the bundle.
+ *
+ * KNOWN_DRIFT below is a burn-down list of the misalignments that existed when
+ * the test was introduced (mostly `random`, the `closest` compounds, and the
+ * ru/uk positional sets — see the table in the PR that added this). Each entry
+ * suppresses the exact-match requirement for one lang:concept pair. The test
+ * also fails when an entry STARTS passing, so fixes must remove their entry —
+ * the list can only shrink. Do NOT add new entries to silence a regression;
+ * new drift means a dict emission and a tokenizer disagree and one of them is
+ * wrong.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { getTokenizer } from '@lokascript/semantic';
+import { dictionaries } from './dictionaries';
+
+const POSITIONAL_CONCEPTS = [
+  'first',
+  'last',
+  'next',
+  'previous',
+  'closest',
+  'parent',
+  'random',
+] as const;
+
+/**
+ * Burn-down list (lang:concept). State as of introduction:
+ * - `random` is unrecognized by most tokenizers (no extras entry).
+ * - The `closest` superlatives/compounds (es máscercano, fr plusproche,
+ *   it piùvicino, pt mais_próximo, tr en_yakın, qu aswan_kaylla) split or
+ *   missed entirely — the tokenizer never matches multi-word/underscore
+ *   natives, so the old tokenizer entries were dead. Fixed by aligning each
+ *   dict to a single token the tokenizer recognizes (es cercano, fr proche,
+ *   it vicino, pt maispróximo, tr enyakın, qu kaylla).
+ * - ru/uk tokenizers carried only the feminine/neuter gendered variants — the
+ *   masculine nominative forms the dict emits were never listed; fixed.
+ * - qu next/previous were CROSS-MAPPED (qhipantin→last, ñawpaqnin→first —
+ *   morphology bound the prefixes); fixed with exact tokenizer entries.
+ * - de closest: FIXED (R2 wave 13). The dict used to emit `nächste` for both
+ *   next and closest, and the tokenizer normalizes `nächste`→next by design, so
+ *   closest was unrecoverable. The dict now emits the unambiguous
+ *   `nächstgelegene` for closest, which the tokenizer maps →closest (distinct
+ *   word, no shadowing of next).
+ * - bn last (শেষ) normalizes to `end` (the block terminator) — a polysemous
+ *   word claimed by the structural keyword. (sw had the same collision via
+ *   mwisho; fixed by emitting the distinct concatenated adjective `wamwisho`,
+ *   which the tokenizer reads as `last` — the saufsi/wennnicht/enyakın class.)
+ */
+const KNOWN_DRIFT = new Set<string>([
+  'ar:parent',
+  'ar:random',
+  'bn:last',
+  'bn:random',
+  'de:parent',
+  'de:random',
+  'es:random',
+  'fr:random',
+  'hi:random',
+  'id:random',
+  'it:parent',
+  'it:random',
+  'ja:random',
+  'ko:random',
+  'ms:random',
+  'pl:random',
+  'pt:random',
+  'qu:parent',
+  'qu:random',
+  'sw:random',
+  'th:random',
+  'tr:random',
+  'zh:random',
+]);
+
+function normalizeEmission(lang: string, emission: string): string | null {
+  let tokenizer: ReturnType<typeof getTokenizer>;
+  try {
+    tokenizer = getTokenizer(lang);
+  } catch {
+    return null; // no tokenizer for this language — out of scope
+  }
+  try {
+    const stream = tokenizer.tokenize(emission);
+    const token = stream.peek() as { normalized?: string; value: string } | undefined | null;
+    if (!token) return '(no-token)';
+    return token.normalized ?? token.value;
+  } catch {
+    return '(tokenize-error)';
+  }
+}
+
+describe('dict positional emissions are recognized by the tokenizer', () => {
+  for (const [lang, dict] of Object.entries(dictionaries)) {
+    if (lang === 'en') continue;
+    const expressions = dict.expressions ?? {};
+    for (const concept of POSITIONAL_CONCEPTS) {
+      const emission = expressions[concept];
+      if (!emission) continue;
+      const key = `${lang}:${concept}`;
+      const expectedDrift = KNOWN_DRIFT.has(key);
+
+      it(`[${key}] '${emission}' ${expectedDrift ? 'is on the burn-down list' : `normalizes to '${concept}'`}`, () => {
+        const normalized = normalizeEmission(lang, emission);
+        if (normalized === null) return; // no tokenizer registered
+        const aligned = normalized === concept;
+        if (expectedDrift) {
+          expect(
+            aligned,
+            `[${key}] '${emission}' now normalizes to '${concept}' — the drift is fixed; ` +
+              `remove '${key}' from KNOWN_DRIFT in positional-keyword-drift.test.ts (the list only shrinks).`
+          ).toBe(false);
+        } else {
+          expect(
+            aligned,
+            `[${key}] dict emits '${emission}' but the ${lang} tokenizer normalizes it to ` +
+              `'${normalized}', not '${concept}'. The transformer emits words the parser can't ` +
+              `read as positionals (the sw 'ijayo' bug class, #338). Align the dict emission to ` +
+              `a word the tokenizer recognizes, or teach the tokenizer the dict's word.`
+          ).toBe(true);
+        }
+      });
+    }
+  }
+});