@keymanapp/kmc-model 17.0.85-alpha → 17.0.87-alpha

package/test/test-parse-wordlist.ts

@@ -1,127 +0,0 @@
-import {parseWordListFromContents, parseWordListFromFilename, WordList} from '../src/build-trie.js';
-import {assert} from 'chai';
-import 'mocha';
-import { makePathToFixture, LogHoarder } from './helpers/index.js';
-import { KeymanCompilerError } from '../src/model-compiler-errors.js';
-
-const BOM = '\ufeff';
-const SENCOTEN_WORDLIST = {
-  'TŦE': 13644,
-  'E': 9134,
-  'SEN': 4816,
-  'Ȼ': 3479,
-  'SW̱': 2621,
-  'NIȽ': 2314,
-  'U¸': 2298,
-  'I¸': 1988,
-  'ȻSE': 1925,
-  'I': 1884
-};
-
-describe('parsing a word list', function () {
-  beforeEach(function () {
-    this.logHoarder = (new LogHoarder).install()
-  })
-
-  afterEach(function () {
-    this.logHoarder.uninstall();
-    delete this.logHoarder;
-  })
-
-  it('should remove the UTF-8 byte order mark from files', function () {
-    let word = 'hello';
-    let count = 1;
-    let expected: WordList = {};
-    expected[word] = count;
-
-    let file = `# this is a comment\n${word}\t${count}`;
-    let withoutBOM: WordList = {};
-    parseWordListFromContents(withoutBOM, file);
-    assert.deepEqual(withoutBOM, expected, "expected regular file to parse properly");
-    assert.isFalse(this.logHoarder.hasSeenWarnings());
-
-    let withBOM: WordList = {};
-    parseWordListFromContents(withBOM, `${BOM}${file}`)
-    assert.deepEqual(withBOM, expected, "expected BOM to be ignored");
-    assert.isFalse(this.logHoarder.hasSeenWarnings());
-  });
-
-  it('should read word lists in UTF-8', function () {
-    // N.B.: this is the format exported by Google Drive when selecting "TSV".
-    const filename = makePathToFixture('example.qaa.sencoten', 'wordlist.tsv');
-    let wordlist: WordList = {};
-    parseWordListFromFilename(wordlist, filename);
-
-    assert.deepEqual(wordlist, SENCOTEN_WORDLIST);
-    assert.isFalse(this.logHoarder.hasSeenWarnings());
-  });
-
-  it('should read word lists in UTF-16 little-endian (with BOM)', function () {
-    // N.B.: this is the format exported by MS Excel when selecting
-    // "UTF-16" text (tested on Excel for macOS).
-    const filename = makePathToFixture('example.qaa.utf16le', 'wordlist.txt');
-    let wordlist: WordList = {};
-    parseWordListFromFilename(wordlist, filename);
-
-    assert.deepEqual(wordlist, SENCOTEN_WORDLIST);
-    assert.isFalse(this.logHoarder.hasSeenWarnings());
-  });
-
-  it('should NOT read word lists in UTF-16 big-endian (with BOM)', function () {
-    // N.B.: Does anything output this format...?
-    const filename = makePathToFixture('example.qaa.utf16be', 'wordlist.txt');
-    let wordlist: WordList = {};
-    assert.throws(() => {
-      parseWordListFromFilename(wordlist, filename);
-    }, 'UTF-16BE is unsupported');
-  });
-
-  it('should merge duplicate entries in a wordlist', function () {
-    // Tests that we merge NFC+NFD entries and identical entries, trimming whitespace
-    // Note building the wordlist from an array to make clear that we have unnormalised inputs
-    const words = [
-      'hello',       //1
-      'hello\u0301', //2, NFD helló
-      'hell\u00f3',  //3, NFC helló
-      ' hello ',     //4, expect to trim whitespace
-      'hello'];      //5
-
-    const expected: WordList = {
-        'hello': 10,     /* 1+4+5 trimmed and identical */
-        'hell\u00f3': 5, /* 2+3 normalised to NFC */
-    };
-
-    // Build a wordlist from the array
-    let file = `# this is a comment\n`;
-    for(let i = 0; i < words.length; i++) {
-      file += `${words[i]}\t${i+1}\n`;
-    }
-    let repeatedWords: WordList = {};
-    parseWordListFromContents(repeatedWords, file);
-
-    assert.deepEqual(repeatedWords, expected);
-
-    assert.isTrue(this.logHoarder.hasSeenWarnings());
-    // hello has been seen multiple times:
-    assert.isTrue(this.logHoarder.hasSeenCode(KeymanCompilerError.CWARN_DuplicateWordInSameFile));
-    // helló and hello + U+0301 have both been seen:
-    assert.isTrue(this.logHoarder.hasSeenCode(KeymanCompilerError.CWARN_MixedNormalizationForms));
-
-    // Let's parse another file:
-
-    this.logHoarder.clear();
-    // Now, parse a DIFFERENT file, but with an NFD entry.
-    parseWordListFromContents(repeatedWords, "hello\u0301\t5\n");
-    assert.isTrue(this.logHoarder.hasSeenWarnings())
-    // hello + U+0301 (NFD) has been seen, but...
-    assert.isTrue(this.logHoarder.hasSeenCode(KeymanCompilerError.CWARN_MixedNormalizationForms));
-    // BUT! We have not seen a duplicate **within the same file**
-    assert.isFalse(this.logHoarder.hasSeenCode(KeymanCompilerError.CWARN_DuplicateWordInSameFile));
-
-    assert.deepEqual(repeatedWords, {
-      hello: expected['hello'],
-      // should have seen more of this entry:
-      "hell\u00f3": expected["hell\u00f3"] + 5,
-    });
-  });
-});

package/test/test-punctuation.ts

@@ -1,35 +0,0 @@
-import LexicalModelCompiler from '../src/lexical-model-compiler.js';
-import {assert} from 'chai';
-import 'mocha';
-
-import { makePathToFixture, compileModelSourceCode } from './helpers/index.js';
-
-describe('LexicalModelCompiler', function () {
-  describe('specifying punctuation', function () {
-    const MODEL_ID = 'example.qaa.trivial';
-    const PATH = makePathToFixture(MODEL_ID);
-
-    it('should compile punctuation into the generated code', function () {
-      let compiler = new LexicalModelCompiler;
-      let code = compiler.generateLexicalModelCode(MODEL_ID, {
-        format: 'trie-1.0',
-        sources: ['wordlist.tsv'],
-        punctuation: {
-          quotesForKeepSuggestion: { open: `«`, close: `»`},
-          insertAfterWord: " " , // OGHAM SPACE MARK
-        }
-      }, PATH) as string;
-
-      // Check that the punctuation actually made into the code:
-      assert.match(code, /«/);
-      assert.match(code, /»/);
-      // Ensure we inserted that OGHAM SPACE MARK!
-      assert.match(code, /\u1680/);
-
-      // Make sure it compiles!
-      let compilation = compileModelSourceCode(code);
-      assert.isFalse(compilation.hasSyntaxError);
-      assert.isNotNull(compilation.exportedModel);
-    });
-  })
-});

package/test/tsconfig.json

@@ -1,22 +0,0 @@
-{
-  "extends": "../../kmc/tsconfig.kmc-base.json",
-
-  "compilerOptions": {
-      "rootDir": ".",
-      "rootDirs": ["./", "../src/"],
-      "outDir": "../build/test",
-      "esModuleInterop": true,
-      "moduleResolution": "node16",
-      "allowSyntheticDefaultImports": true,
-      "baseUrl": ".",
-  },
-  "include": [
-      "**/test-*.ts",
-      "*.ts",
-      "helpers/*.ts",
-      "wordbreakers/*.ts"
-  ],
-  "references": [
-      { "path": "../" },
-    ]
-}

package/test/wordbreakers/README.md

@@ -1,3 +0,0 @@
-Wordbreakers ES Module format
-
-TODO: once we move common/models/wordbreakers to ESM, eliminate this.