@keymanapp/kmc-model 17.0.85-alpha

package/test/test-default-apply-case.ts

@@ -0,0 +1,128 @@
+
+import 'mocha';
+import { assert } from 'chai';
+import { defaultApplyCasing } from '../src/model-defaults.js';
+
+describe('The default applyCasing() function', function () {
+  // // --------
+  // // Definitions toward SMP testing.
+  // // Ref: https://unicode.org/charts/nameslist/n_1D400.html
+  // // Useful for tests related to strings with supplementary pairs.
+  // let u = function(code: number): string {
+  //   var H = Math.floor((code - 0x10000) / 0x400) + 0xD800;
+  //   var L = (code - 0x10000) % 0x400 + 0xDC00;
+
+  //   return String.fromCharCode(H, L);
+  // }
+
+  // Are there any known default-handled SMP cases?
+  // If not... we could make the defaultApplyCase function simpler by not worrying about SMP.
+  //
+  // let smp_a = u(0x1d5ba);  // MATHEMATICAL SANS-SERIF SMALL A
+  // let smp_p = u(0x1d5c9);
+  // let smp_l = u(0x1d5c5);
+  // let smp_e = u(0x1d5be);
+
+  // let smp_A = u(0x1d5a0);  // MATHEMATICAL SANS_SERIF CAPITAL A
+  // let smp_P = u(0x1d5af);
+  // let smp_L = u(0x1d5ab);
+  // let smp_E = u(0x1d5a4);
+
+  // // Unfortunately... the default JS .toUpperCase() implementation doesn't actually
+  // // map the 'SMALL' versions to the 'CAPITAL' versions.
+  // // ---------
+
+  describe('case:  \'lower\'', function() {
+    const testCases: [string, string][] = [
+      // Note:  not written the Turkish way.  Turns out 'İ'.toLowerCase() decomposes the result,
+      // which would have made for a fairly yucky test.
+      ['Istanbul', 'istanbul'],
+
+      // The DEFAULT function is NOT responsible for understanding the Turkish
+      // case regarding the lowercasing of:
+      // 'I' U+0048 LATIN CAPITAL LETTER I to 'ı' U+0131 LATIN SMALL LETTER DOTLESS I
+      // For Turkic languages, the recommendation is to make a
+      // custom applyCasing function:
+      ['DİYARBAKIR', 'di̇yarbakir'], // The 'i̇' is the decomposed result alluded to for the previous case.
+
+      // full-width romaji has corresponding capitalized versions:
+      ['ＡＥＳＴＨＥＴＩＣ', 'ａｅｓｔｈｅｔｉｃ'],
+
+      // "skýlos" is Greek for dog 🇬🇷🐶
+      // starts with an 's' and ends with an 's'
+      // which are DIFFERENT CHARACTERS in lowercased Greek!
+      ['ΣΚΥΛΟΣ', 'σκυλος'],
+
+      // Uncased syntax and numbers should pass through unscathed:
+      ['1234.?!', '1234.?!']
+    ];
+
+    for (let [input, expected] of testCases) {
+      it(`should lowercase '${input}' as '${expected}'`, function() {
+        assert.equal(defaultApplyCasing('lower', input), expected);
+      });
+    }
+  });
+
+  describe('case:  \'upper\'', function() {
+    const testCases: [string, string][] = [
+      // Inverse of the corresponding 'lower' test.
+      ['istanbul', 'ISTANBUL'],
+
+      // The DEFAULT function is NOT responsible for understanding the Turkish
+      // case regarding the uppercasing of:
+      // 'ı' U+0131 LATIN SMALL LETTER DOTLESS I to 'I' U+0048 LATIN CAPITAL LETTER I
+      // For Turkic languages, the recommendation is to make a
+      // custom applyCasing function:
+      ['diyarbakır', 'DIYARBAKIR'], // The 'i̇' is the decomposed result alluded to for the previous case.
+
+      // full-width romaji has corresponding capitalized versions:
+      ['ａｅｓｔｈｅｔｉｃ', 'ＡＥＳＴＨＥＴＩＣ'],
+
+      // "skýlos" is Greek for dog 🇬🇷🐶
+      // starts with an 's' and ends with an 's'
+      // which are DIFFERENT CHARACTERS in lowercased Greek!
+      ['σκυλος', 'ΣΚΥΛΟΣ'],
+
+      // Uncased syntax and numbers should pass through unscathed:
+      ['1234.?!', '1234.?!']
+    ];
+
+    for (let [input, expected] of testCases) {
+      it(`should uppercase '${input}' as '${expected}'`, function() {
+        assert.equal(defaultApplyCasing('upper', input), expected);
+      });
+    }
+  });
+
+  describe('case:  \'initial\'', function() {
+    const testCases: [string, string][] = [
+      // Inverse of the corresponding 'lower' test.
+      ['istanbul', 'Istanbul'],
+
+      // The DEFAULT function is NOT responsible for understanding the Turkish
+      // case regarding the uppercasing of:
+      // 'ı' U+0131 LATIN SMALL LETTER DOTLESS I to 'I' U+0048 LATIN CAPITAL LETTER I
+      // For Turkic languages, the recommendation is to make a
+      // custom applyCasing function:
+      ['diyarbakır', 'Diyarbakır'], // The 'i̇' is the decomposed result alluded to for the previous case.
+
+      // full-width romaji has corresponding capitalized versions:
+      ['ａｅｓｔｈｅｔｉｃ', 'Ａｅｓｔｈｅｔｉｃ'],
+
+      // "skýlos" is Greek for dog 🇬🇷🐶
+      // starts with an 's' and ends with an 's'
+      // which are DIFFERENT CHARACTERS in lowercased Greek!
+      ['σκυλος', 'Σκυλος'],
+
+      // Uncased syntax and numbers should pass through unscathed:
+      ['1234.?!', '1234.?!']
+    ];
+
+    for (let [input, expected] of testCases) {
+      it(`should initial-case '${input}' as '${expected}'`, function() {
+        assert.equal(defaultApplyCasing('initial', input), expected);
+      });
+    }
+  });
+});

package/test/test-default-search-term-to-key.ts

@@ -0,0 +1,181 @@
+
+import 'mocha';
+import {assert} from 'chai';
+import { defaultSearchTermToKey,
+         defaultCasedSearchTermToKey,
+         defaultApplyCasing } from '../src/model-defaults.js';
+
+
+describe('The default searchTermToKey() function', function () {
+  describe('languageUsesCasing: false', function() {
+    const testCases: [string, string][] = [
+      // "İstanbul" has a U+0130 LATIN CAPITAL LETTER I WITH DOT ABOVE.
+      // Without any casing operations, only the I should be altered.
+      ['İstanbul', 'Istanbul'],
+
+      // Similarly...
+      ['DİYARBAKIR', 'DIYARBAKIR'],
+
+      // "skýlos" is Greek for dog 🇬🇷🐶
+      // starts with an 's' and ends with an 's'
+      // which are DIFFERENT CHARACTERS in lowercased Greek!
+      ['σκύλος', 'σκυλος'],
+      ['ΣΚΥΛΟΣ', 'ΣΚΥΛΟΣ'],
+
+      // full-width romaji is compatibility-canonical with ASCII characters:
+      ['ａｅｓｔｈｅｔｉｃ', 'aesthetic'],
+
+      // U+212B ANGSTROM SIGN (Å)
+      // U+00C5 LATIN CAPITAL LETTER A WITH RING ABOVE (Å)
+      // and should both normalize to 'A'
+      ['\u212B', 'A'],
+      ['\u00C5', 'A'],
+
+      // We should not fall for U+037E GREEK QUESTION MARK's trolling:
+      ['\u037e', ';'],
+
+      // Test presentational forms of Arabic:
+      // U+FE8D ARABIC LETTER ALEF ISOLATED FORM -> U+0627 ARABIC LETTER ALEF
+      // U+FEDF ARABIC LETTER LAM INITIAL FORM -> U+0644 ARABIC LETTER LAM
+      // U+FED8 ARABIC LETTER QAF MEDIAL FORM -> U+0642 ARABIC LETTER QAF
+      // U+FEEC ARABIC LETTER HEH MEDIAL FORM -> U+0647 ARABIC LETTER HEH
+      // U+FEEE ARABIC LETTER WAW FINAL FORM -> U+0648 ARABIC LETTER WAW
+      // U+FE93 ARABIC LETTER TEH MARBUTA ISOLATED FORM -> U+0629 ARABIC LETTER TEH MARBUTA
+      ['\uFE8D\uFEDF\uFED8\uFEEC\uFEEE\uFE93', '\u0627\u0644\u0642\u0647\u0648\u0629'],
+
+      // Combine both NFKD **AND** knocking off diacritics:
+      // U+01C4 LATIN CAPITAL LETTER DZ WITH CARON (DŽ) -> <U+0064, U+007A> (dz)
+      ['DŽ', 'DZ'],
+    ];
+
+    for (let [input, expected] of testCases) {
+      it(`should normalize '${input}' to '${expected}'`, function() {
+        assert.equal(defaultSearchTermToKey(input), expected);
+      });
+    }
+  });
+
+  describe('languageUsesCasing:  true (custom applyCasing, inverts lower- & upper- casing)', function() {
+    const testCases: [string, string][] = [
+      // The DEFAULT function is NOT responsible for understanding the Turkish
+      // distinction between U+0130 LATIN CAPITAL LETTER I WITH DOT ABOVE and
+      // 'I' U+0048 LATIN CAPITAL LETTER I.
+      // For Turkic languages, the recommendation is thus to make a
+      // custom searchTermToKey function.
+      ['İstanbul', 'ISTANBUL'],
+      ['DİYARBAKIR', 'DIYARBAKIR'],
+
+      // "skýlos" is Greek for dog 🇬🇷🐶
+      // starts with an 's' and ends with an 's'
+      // which are DIFFERENT CHARACTERS in lowercased Greek!
+      ['σκύλος', 'ΣΚΥΛΟΣ'],
+      ['σκυλοσ', 'ΣΚΥΛΟΣ'],
+
+      // full-width romaji is compatibility-canonical with ASCII characters:
+      ['ａｅｓｔｈｅｔｉｃ', 'AESTHETIC'],
+
+      // U+212B ANGSTROM SIGN (Å)
+      // U+00C5 LATIN CAPITAL LETTER A WITH RING ABOVE (Å)
+      // and should both normalize to 'a'
+      ['\u212B', 'A'],
+      ['\u00C5', 'A'],
+
+      // We should not fall for U+037E GREEK QUESTION MARK's trolling:
+      ['\u037e', ';'],
+
+      // Test presentational forms of Arabic:
+      // U+FE8D ARABIC LETTER ALEF ISOLATED FORM -> U+0627 ARABIC LETTER ALEF
+      // U+FEDF ARABIC LETTER LAM INITIAL FORM -> U+0644 ARABIC LETTER LAM
+      // U+FED8 ARABIC LETTER QAF MEDIAL FORM -> U+0642 ARABIC LETTER QAF
+      // U+FEEC ARABIC LETTER HEH MEDIAL FORM -> U+0647 ARABIC LETTER HEH
+      // U+FEEE ARABIC LETTER WAW FINAL FORM -> U+0648 ARABIC LETTER WAW
+      // U+FE93 ARABIC LETTER TEH MARBUTA ISOLATED FORM -> U+0629 ARABIC LETTER TEH MARBUTA
+      ['\uFE8D\uFEDF\uFED8\uFEEC\uFEEE\uFE93', '\u0627\u0644\u0642\u0647\u0648\u0629'],
+
+      // Combine both NFKD **AND** knocking off diacritics:
+      // U+01C4 LATIN CAPITAL LETTER DZ WITH CARON (DŽ) -> <U+0064, U+007A> (dz)
+      ['DŽ', 'DZ'],
+    ];
+
+    // While a Turkish-based test would be nice, Turkish needs custom keying,
+    // as U+0130's default handling is... not ideal in Turkish.
+    //
+    // Instead, we can get a simple-enough test with inverted casing.
+    let customCasing = function(caseToApply: CasingForm,
+                                text: string,
+                                defaultApplyCasing: CasingFunction): string {
+      switch(caseToApply) {
+        case 'lower':
+            return text.toUpperCase();
+        case 'upper':
+            return text.toLowerCase();
+        case 'initial':
+            return customCasing('upper', text.charAt(0), defaultApplyCasing) + text.substr(1);
+        default:
+            return text;
+        }
+    }
+
+    let customCasingClosure = function(caseToApply: CasingForm, text: string): string {
+      return customCasing(caseToApply, text, defaultApplyCasing);
+    }
+
+    for (let [input, expected] of testCases) {
+      it(`should normalize '${input}' to '${expected}'`, function() {
+        assert.equal(defaultCasedSearchTermToKey(input, customCasingClosure as CasingFunction), expected);
+      });
+    }
+  });
+
+  describe('languageUsesCasing:  undefined (legacy 12.0 / 13.0 mode, uses `defaultApplyCasing`)', function() {
+    const testCases: [string, string][] = [
+      // "İstanbul" has a U+0130 LATIN CAPITAL LETTER I WITH DOT ABOVE.
+      // This should be lowercased.
+      ['İstanbul', 'istanbul'],
+
+      // The DEFAULT function is NOT responsible for understanding the Turkish
+      // case regarding the lowercasing of:
+      // 'I' U+0048 LATIN CAPITAL LETTER I to 'ı' U+0131 LATIN SMALL LETTER DOTLESS I
+      // For Turkic languages, the recommendation is to make a
+      // custom searchTermToKey function:
+      ['DİYARBAKIR', 'diyarbakir'],
+
+      // "skýlos" is Greek for dog 🇬🇷🐶
+      // starts with an 's' and ends with an 's'
+      // which are DIFFERENT CHARACTERS in lowercased Greek!
+      ['σκύλος', 'σκυλος'],
+      ['ΣΚΥΛΟΣ', 'σκυλοσ'],
+
+      // full-width romaji is compatibility-canonical with ASCII characters:
+      ['ａｅｓｔｈｅｔｉｃ', 'aesthetic'],
+
+      // U+212B ANGSTROM SIGN (Å)
+      // U+00C5 LATIN CAPITAL LETTER A WITH RING ABOVE (Å)
+      // and should both normalize to 'a'
+      ['\u212B', 'a'],
+      ['\u00C5', 'a'],
+
+      // We should not fall for U+037E GREEK QUESTION MARK's trolling:
+      ['\u037e', ';'],
+
+      // Test presentational forms of Arabic:
+      // U+FE8D ARABIC LETTER ALEF ISOLATED FORM -> U+0627 ARABIC LETTER ALEF
+      // U+FEDF ARABIC LETTER LAM INITIAL FORM -> U+0644 ARABIC LETTER LAM
+      // U+FED8 ARABIC LETTER QAF MEDIAL FORM -> U+0642 ARABIC LETTER QAF
+      // U+FEEC ARABIC LETTER HEH MEDIAL FORM -> U+0647 ARABIC LETTER HEH
+      // U+FEEE ARABIC LETTER WAW FINAL FORM -> U+0648 ARABIC LETTER WAW
+      // U+FE93 ARABIC LETTER TEH MARBUTA ISOLATED FORM -> U+0629 ARABIC LETTER TEH MARBUTA
+      ['\uFE8D\uFEDF\uFED8\uFEEC\uFEEE\uFE93', '\u0627\u0644\u0642\u0647\u0648\u0629'],
+
+      // Combine both NFKD **AND** knocking off diacritics:
+      // U+01C4 LATIN CAPITAL LETTER DZ WITH CARON (DŽ) -> <U+0064, U+007A> (dz)
+      ['DŽ', 'dz'],
+    ];
+
+    for (let [input, expected] of testCases) {
+      it(`should normalize '${input}' to '${expected}'`, function() {
+        assert.equal(defaultCasedSearchTermToKey(input, defaultApplyCasing), expected);
+      });
+    }
+  });
+});

package/test/test-error-logger.ts

@@ -0,0 +1,38 @@
+import { MAX_MESSAGES, KeymanCompilerError, log } from "../src/model-compiler-errors.js";
+import { LogHoarder } from "./helpers/index.js";
+import { assert } from "chai";
+
+describe('error logger', function () {
+  beforeEach(function () {
+    this.logHoarder = (new LogHoarder).install()
+  })
+
+  afterEach(function () {
+    this.logHoarder.uninstall();
+    delete this.logHoarder;
+  })
+
+  it('should stop logging messages **after** a maximum', function () {
+    for (let i = 0; i < MAX_MESSAGES; i++) {
+      log(KeymanCompilerError.CWARN_DuplicateWordInSameFile, "fake error");
+    }
+
+    // We've logged *just enough messages. This error should not be found:
+    assert.isFalse(this.logHoarder.hasSeenCode(
+      KeymanCompilerError.CWARN_TooManyErrorsOrWarnings
+    ));
+
+    // Log just one too many:
+    log(KeymanCompilerError.CWARN_DuplicateWordInSameFile, "fake error");
+
+    assert.isTrue(this.logHoarder.hasSeenCode(
+      KeymanCompilerError.CWARN_TooManyErrorsOrWarnings
+    ));
+
+    // Log a DIFFERENT error -- it should not appear in the log
+    log(KeymanCompilerError.CWARN_MixedNormalizationForms, "fake error");
+    assert.isFalse(this.logHoarder.hasSeenCode(
+      KeymanCompilerError.CWARN_MixedNormalizationForms
+    ));
+  })
+})

package/test/test-join-word-breaker.ts

@@ -0,0 +1,97 @@
+import { assert } from "chai";
+import defaultWordBreaker from './wordbreakers/default-wordbreaker-esm.js';
+import {decorateWithJoin} from '../src/join-word-breaker-decorator.js';
+
+describe('The join word breaker decorator', function () {
+  it('should decorate an existing word breaker', function () {
+    let breakWords = decorateWithJoin(defaultWordBreaker, ['-']);
+    assert.isFunction(breakWords);
+  });
+
+  const TEST_CASES: [string, string[], string[], string[]][] = [
+    /* input,      joiners,  default breaks,        breaks with joins */
+    // Original test case from https://github.com/keymanapp/keyman/issues/2753
+    ['khui-chhùi', ['-'],   ["khui", "-", "chhùi"], ["khui-chhùi"]],
+
+    // Plains Cree SRO:
+    ['ê-kotiskâwêyâhk', ['-'], ['ê', '-', 'kotiskâwêyâhk'], ['ê-kotiskâwêyâhk']],
+
+    // Edge cases:
+
+    // Joiner alone:
+    ['-', ['-'], ['-'], ['-']],
+    // Joiner at the end:
+    ['ni-', ['-'], ['ni', '-'], ['ni-']],
+    // Joiner at the end:
+    ['-ân', ['-'], ['-', 'ân'], ['-ân']],
+
+    // This was my guiding test case:
+    [
+      "-yâhk ê-nitawi-kotiskâwêyâhk ni-",
+      ["-"],
+      ["-", "yâhk", "ê", "-", "nitawi", "-", "kotiskâwêyâhk", "ni", "-"],
+      ["-yâhk", "ê-nitawi-kotiskâwêyâhk", "ni-"]
+    ],
+
+    // Do not perform any joins:
+    ["hello world", ["-"], ["hello", "world"], ["hello", "world"]],
+
+    // Joining using multiple joiners
+    [
+      "Email: no-body@example.com",
+      ["@", "-"],
+      ["Email", ":", "no", "-", "body", "@", "example.com"],
+      ["Email", ":", "no-body@example.com"]
+    ],
+
+    // Joining with two or more joiners in a row
+    [
+      "nobody@@example.com",
+      ["@"],
+      ["nobody", "@", "@", "example.com"],
+      ["nobody@@example.com"]
+    ],
+
+    // it should NOT join non-contiguous spans:
+    [
+      "this- is -bad",
+      ["-"],
+      ["this", "-", "is", "-", "bad"],
+      ["this-", "is", "-bad"]
+    ],
+
+    // different but adjacent joiners
+    [
+      "I made the kawé:-conjugator.",
+      ["-", ":"],
+      ["I", "made", "the", "kawé", ":", "-", "conjugator", "."],
+      ["I", "made", "the", "kawé:-conjugator", "."]
+    ],
+
+    // 3+ joiners in a row
+    [
+      // NB: – is U+2001 EN DASH
+      "This language is nut–=💠¤~ty!",
+      ["~", "–", "¤", "=", "💠"],
+      ["This", "language", "is", "nut", "–", "=", "💠", "¤", "~", "ty", "!"],
+      ["This", "language", "is", "nut–=💠¤~ty", "!"],
+    ],
+  ]
+
+  for (let [phrase, joiners, unjoined, expected] of TEST_CASES) {
+    it(`should break «${[phrase]}» as [${expected.join(' ;; ')}]`, function () {
+      let breakWords = decorateWithJoin(defaultWordBreaker, joiners);
+      let unjoinedResult = defaultWordBreaker(phrase).map(onlyText);
+      assert.deepEqual(unjoinedResult, unjoined);
+      let actualResult = breakWords(phrase).map(onlyText);
+      assert.deepEqual(actualResult, expected);
+    });
+  }
+
+  /**
+   * Get just the text from a span.
+   */
+  function onlyText(span: Span) {
+    return span.text;
+  }
+});

package/test/test-model-definitions.ts

@@ -0,0 +1,198 @@
+import 'mocha';
+import { assert } from 'chai';
+import { ModelDefinitions } from '../src/model-definitions.js';
+import { LexicalModelSource } from '../src/lexical-model.js';
+
+describe('Model definition pseudoclosures', function () {
+  describe('14.0 defaults', function() {
+    describe('languageUsesCasing == true', function() {
+      // We don't need a complete spec for this, given the (currently) limited range of what
+      // the ModelPseudoclosure covers.
+      let modelSource: LexicalModelSource = {
+        languageUsesCasing: true,
+        sources: [],
+        format: 'trie-1.0'
+      };
+
+      let pseudoclosure = new ModelDefinitions(modelSource);
+
+      const testCases: [string, string, string][] = [
+        // Note:  not written the Turkish way.  Turns out 'İ'.toLowerCase() decomposes the result,
+        // which would have made for a fairly yucky test.
+        ['Istanbul', 'istanbul', 'istanbul'],
+
+        // The DEFAULT function is NOT responsible for understanding the Turkish
+        // case regarding the lowercasing of:
+        // 'I' U+0048 LATIN CAPITAL LETTER I to 'ı' U+0131 LATIN SMALL LETTER DOTLESS I
+        // For Turkic languages, the recommendation is to make a
+        // custom applyCasing function:
+        ['DİYARBAKIR', 'di̇yarbakir', 'diyarbakir'], // The 'i̇' is the decomposed result alluded to for the previous case.
+
+        // full-width romaji has corresponding lowercased versions:
+        ['ＡＥＳＴＨＥＴＩＣ', 'ａｅｓｔｈｅｔｉｃ', 'aesthetic'],
+
+        // "skýlos" is Greek for dog 🇬🇷🐶
+        // starts with an 's' and ends with an 's'
+        // which are DIFFERENT CHARACTERS in lowercased Greek!
+        ['σκύλος', 'σκύλος', 'σκυλος'],
+        ['ΣΚΥΛΟΣ', 'σκυλος', 'σκυλοσ'],  // the keyed version after lowercasing doesn't know how
+                                         // to make the distinction.  Both 'Σ's have the same char-code.
+
+        // Uncased syntax and numbers should pass through unscathed:
+        ['1234.?!', '1234.?!', '1234.?!'],
+        ['”', '”', '"'],
+        ["‘", "‘", "'"]
+      ];
+
+      for (let [input, cased, keyed] of testCases) {
+        it(`should case '${input}' as '${cased}'`, function() {
+          assert.equal(pseudoclosure.applyCasing('lower', input), cased);
+        });
+
+        it(`should key  '${input}' as '${keyed}'`, function() {
+          assert.equal(pseudoclosure.searchTermToKey(input), keyed);
+        });
+      }
+    });
+
+    describe('languageUsesCasing == false', function() {
+      // We don't need a complete spec for this, given the (currently) limited range of what
+      // the ModelPseudoclosure covers.
+      let modelSource: LexicalModelSource = {
+        languageUsesCasing: false,
+        sources: [],
+        format: 'trie-1.0'
+      };
+
+      let pseudoclosure = new ModelDefinitions(modelSource);
+
+      const testCases: [string, string][] = [
+        // Note:  not written the Turkish way.  Turns out 'İ'.toLowerCase() decomposes the result,
+        // which would have made for a fairly yucky test.
+        ['Istanbul', 'Istanbul'],
+
+        ['DİYARBAKIR', 'DIYARBAKIR'],
+
+        // full-width romaji has corresponding capitalized versions:
+        ['ＡＥＳＴＨＥＴＩＣ', 'AESTHETIC'],
+
+        // "skýlos" is Greek for dog 🇬🇷🐶
+        // starts with an 's' and ends with an 's'
+        // which are DIFFERENT CHARACTERS in lowercased Greek!
+        ['σκύλος', 'σκυλος'],
+        ['ΣΚΥΛΟΣ', 'ΣΚΥΛΟΣ'],  // the keyed version after lowercasing doesn't know how
+                               // to make the distinction.  Both 'Σ's have the same char-code.
+
+        // Uncased syntax and numbers should pass through unscathed:
+        ['1234.?!', '1234.?!'],
+
+        ['”', '"'],
+        ["‘", "'"]
+      ];
+
+      for (let [input, keyed] of testCases) {
+        it(`should key  '${input}' as '${keyed}'`, function() {
+          assert.equal(pseudoclosure.searchTermToKey(input), keyed);
+        });
+      }
+    });
+  });
+  describe('Pre 14.0 defaults (languageUsesCasing == undefined)', function() {
+    // We don't need a complete spec for this, given the (currently) limited range of what
+    // the ModelPseudoclosure covers.
+    let modelSource: LexicalModelSource = {
+      sources: [],
+      format: 'trie-1.0'
+    };
+
+    let pseudoclosure = new ModelDefinitions(modelSource);
+
+    const testCases: [string, string][] = [
+      // Note:  not written the Turkish way.  Turns out 'İ'.toLowerCase() decomposes the result,
+      // which would have made for a fairly yucky test.
+      ['Istanbul', 'istanbul'],
+
+      ['DİYARBAKIR', 'diyarbakir'],
+
+      // full-width romaji has corresponding capitalized versions:
+      ['ＡＥＳＴＨＥＴＩＣ', 'aesthetic'],
+
+      // "skýlos" is Greek for dog 🇬🇷🐶
+      // starts with an 's' and ends with an 's'
+      // which are DIFFERENT CHARACTERS in lowercased Greek!
+      ['σκύλος', 'σκυλος'],
+      ['ΣΚΥΛΟΣ', 'σκυλοσ'],  // the keyed version after lowercasing doesn't know how
+                             // to make the distinction.  Both 'Σ's have the same char-code.
+
+      // Uncased syntax and numbers should pass through unscathed:
+      ['1234.?!', '1234.?!']
+    ];
+
+    for (let [input, keyed] of testCases) {
+      it(`should key  '${input}' as '${keyed}'`, function() {
+        assert.equal(pseudoclosure.searchTermToKey(input), keyed);
+      });
+    }
+  });
+
+  describe('Model-defined applyCasing + (dependent) searchTermToKey', function() {
+    // Note:  this test only implements enough Turkish-related stuff to facilitate
+    // a functional test.  Not guaranteed to be sufficient for actual Turkish use.
+    let turkishCasing = function(form: CasingForm, text: string, defaultApplyCasing: (form: CasingForm, text: string) => string): string {
+      switch(form) {
+      case 'lower':
+        return defaultApplyCasing(form, text
+            .replace(/I/g, 'ı')
+            .replace(/İ/g, 'i'));
+      case 'upper':
+        return defaultApplyCasing(form, text
+            .replace(/ı/g, 'I')
+            .replace(/i/g, 'İ'));
+      case 'initial':
+        return turkishCasing('upper', text.charAt(0), defaultApplyCasing) + text.substr(1);
+      default:
+        return text;
+      }
+    }
+
+    let modelSource: LexicalModelSource = {
+      languageUsesCasing: true,
+      applyCasing: turkishCasing,
+      searchTermToKey: function(wordform: string, applyCasing: CasingFunction): string {
+        return Array.from(wordform
+          .normalize('NFC')  // Mostly to avoid decomposing 'İ'
+        ) // end of `Array.from`
+        .map(function(c) { return applyCasing('lower', c)})  // Will use custom `applyCasing` definition!
+        .join('');
+      },
+      sources: [],
+      format: 'trie-1.0'
+    };
+
+    let pseudoclosure = new ModelDefinitions(modelSource);
+
+    const testCases: [string, string, string][] = [
+      ['İstanbul', 'istanbul', 'istanbul'],
+
+      // The DEFAULT function is NOT responsible for understanding the Turkish
+      // case regarding the lowercasing of:
+      // 'I' U+0048 LATIN CAPITAL LETTER I to 'ı' U+0131 LATIN SMALL LETTER DOTLESS I
+      // For Turkic languages, the recommendation is to make a
+      // custom applyCasing function:
+      ['DİYARBAKIR', 'diyarbakır', 'diyarbakır'],
+
+      // Uncased syntax and numbers should pass through unscathed:
+      ['1234.?!', '1234.?!', '1234.?!']
+    ];
+
+    for (let [input, cased, keyed] of testCases) {
+      it(`should case '${input}' as '${cased}'`, function() {
+        assert.equal(pseudoclosure.applyCasing('lower', input), cased);
+      });
+
+      it(`should key  '${input}' as '${keyed}'`, function() {
+        assert.equal(pseudoclosure.searchTermToKey(input), keyed);
+      });
+    }
+  });
+});

package/test/test-override-script-defaults.ts

@@ -0,0 +1,33 @@
+import { assert } from "chai";
+import defaultWordBreaker from './wordbreakers/default-wordbreaker-esm.js';
+import {decorateWithScriptOverrides} from '../src/script-overrides-decorator.js';
+
+const THIN_SPACE = "\u2009";
+
+describe('The script overrides word breaker decorator', function () {
+  it('should decorate an existing word breaker', function () {
+    let breakWords = decorateWithScriptOverrides(defaultWordBreaker, 'break-words-at-spaces')
+    assert.isFunction(breakWords);
+  });
+
+  // I do not read this script or language so I have NO idea what this says
+  // ¯\_(ツ)_/¯
+  const phraseComponents = ["ຈາກ", THIN_SPACE, "ກ໌ນິ", THIN_SPACE, "ສນາ", THIN_SPACE, "ເກ໌າະ", THIN_SPACE, "ຢັອຫ", THIN_SPACE, "ລະ", THIN_SPACE, "ບຣອມ", THIN_SPACE, "ເຢາະ", ","];
+  const phraseSpans = phraseComponents.filter(span => span !== THIN_SPACE);
+  const phrase = phraseComponents.join("");
+  const expectedNumSpans = phraseSpans.length;
+
+  it(`should break «${[phrase]}» as ${expectedNumSpans} spans`, function () {
+    let breakWords = decorateWithScriptOverrides(defaultWordBreaker, 'break-words-at-spaces');
+    let defaultResult = defaultWordBreaker(phrase);
+
+    assert.isAbove(defaultResult.length, expectedNumSpans);
+    let actualResult = breakWords(phrase);
+    assert.lengthOf(actualResult, expectedNumSpans);
+    assert.deepEqual(actualResult.map(grabText), phraseSpans);
+  });
+
+  function grabText(span: Span) {
+    return span.text;
+  }
+});