@keymanapp/kmc-model 17.0.85-alpha

package/build/test/helpers/index.js

@@ -0,0 +1,160 @@
+/// <reference types="@keymanapp/models-types" />
+/**
+ * Helpers and utilities for the Mocha tests.
+ */
+import * as path from 'path';
+import { fileURLToPath } from 'url';
+import { assert } from 'chai';
+import { redirectLogMessagesTo, resetLogMessageHandler, LogLevel } from '../../src/model-compiler-errors.js';
+/**
+ * Builds a path to the fixture with the given path components.
+ *
+ * e.g., makePathToFixture('example.qaa.trivial')
+ * e.g., makePathToFixture('example.qaa.trivial', 'model.ts')
+ *
+ * @param components One or more path components.
+ */
+export function makePathToFixture(...components) {
+    return fileURLToPath(new URL(path.join('..', '..', '..', 'test', 'fixtures', ...components), import.meta.url));
+}
+/**
+ * Given source code of a model (as produced by the compiler), this evaluates
+ * it!
+ *
+ * That way, you can determine if the source code was syntactically-valid,
+ * whether it raise an exception during construction, and you can even inspect
+ * the resultant LexicalModel for yourself!
+ *
+ * @param code
+ */
+export function compileModelSourceCode(code) {
+    assert.typeOf(code, 'string');
+    let error = null;
+    let exportedModel = null;
+    let hasSyntaxError = false;
+    let modelConstructorName = null;
+    let module = null;
+    try {
+        module = new Function('LMLayerWorker', 'models', 'wordBreakers', code);
+    }
+    catch (err) {
+        if (err instanceof SyntaxError) {
+            hasSyntaxError = true;
+        }
+        else {
+            throw err;
+        }
+    }
+    // Module had some sort of syntax error:
+    if (module === null) {
+        return {
+            error, exportedModel, hasSyntaxError, modelConstructorName
+        };
+    }
+    let fakeLMLayerWorker = {
+        loadModel(model) {
+            exportedModel = model;
+        }
+    };
+    // We expect the compiled model to reach into the `models` namespace,
+    // and get some sort of model constructor. Problem is, we don't know
+    // what model constructors are defined!
+    // This proxy allows us to intercept
+    //    models.*
+    // meaning that when the compiled code tries to do this:
+    //
+    //    new models.WhateverModel('foo', 'bar')
+    //
+    // ...we can intercept the name "WhateverModel" and assign it to
+    // `modelConstructorName`.
+    let modelsNamespace = new Proxy({}, {
+        get(_target, property) {
+            if (typeof property !== 'string')
+                throw new Error(`Don't know how to handle non-string property: ${String(property)}`);
+            return class DummyModel {
+                constructor() {
+                    modelConstructorName = property;
+                }
+            };
+        }
+    });
+    // We expect the compiled model to reach into the `wordBreaker` namespace,
+    // and maybe get a function. Problem is, we don't know
+    // what functions are available!
+    // This proxy allows us to intercept
+    //    wordBreakers.*
+    // meaning that when the compiled code tries to do this:
+    //
+    //    wordBreaker["someWordBreaker"]
+    //
+    // ...we can intercept the name "someWordBreaker".
+    let wordBreakerNamespace = new Proxy({}, {
+        get(_target, property) {
+            if (typeof property !== 'string')
+                throw new Error(`Don't know how to handle non-string property: ${String(property)}`);
+            return function dummyWordBreaker() {
+            };
+        }
+    });
+    try {
+        module(fakeLMLayerWorker, modelsNamespace, wordBreakerNamespace);
+    }
+    catch (err) {
+        error = err;
+    }
+    return {
+        error, exportedModel, hasSyntaxError, modelConstructorName
+    };
+}
+/**
+ * Enables one to query log messages after they have been logged.
+ */
+export class LogHoarder {
+    messages = [];
+    /**
+     * Get rid of all log messages seen.
+     */
+    clear() {
+        this.messages = [];
+    }
+    /**
+     * Hoards a log message for later perusal.
+     */
+    handleLog(log) {
+        this.messages.push(log);
+    }
+    /**
+     * Has an error message with this code been witnessed?
+     */
+    hasSeenCode(code) {
+        return Boolean(this.messages.find(log => log.code === code));
+    }
+    /**
+     * Have any warnings been logged?
+     */
+    hasSeenWarnings() {
+        return this.messages
+            .filter(log => log.level === LogLevel.CERR_WARNING)
+            .length > 0;
+    }
+    /**
+     * Overrides the global log handler, allowing one to browse log messages
+     * later.
+     *
+     * Remember to uninstall the log handler afterwards!
+     */
+    install() {
+        redirectLogMessagesTo(this.handleLog.bind(this));
+        return this;
+    }
+    /**
+     * Return the log message handler to its default.
+     *
+     * Note: You MUST uninstall the hoarder after use!
+     * It's recommended you put this in an afterEach() callback.
+     */
+    uninstall() {
+        resetLogMessageHandler();
+    }
+}
+//# sourceMappingURL=index.js.map

package/build/test/helpers/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../test/helpers/index.ts"],"names":[],"mappings":"AAAA,iDAAiD;AAEjD;;GAEG;AACH,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AACpC,OAAO,EAAC,MAAM,EAAC,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAkC,qBAAqB,EAAE,sBAAsB,EAAE,QAAQ,EAAC,MAAM,oCAAoC,CAAC;AAS5I;;;;;;;GAOG;AACF,MAAM,UAAU,iBAAiB,CAAC,GAAG,UAAoB;IACxD,OAAO,aAAa,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,UAAU,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AACjH,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,sBAAsB,CAAC,IAAY;IACjD,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAE9B,IAAI,KAAK,GAAiB,IAAI,CAAC;IAC/B,IAAI,aAAa,GAAkB,IAAI,CAAC;IACxC,IAAI,cAAc,GAAG,KAAK,CAAC;IAC3B,IAAI,oBAAoB,GAAkB,IAAI,CAAC;IAE/C,IAAI,MAAM,GAAsB,IAAI,CAAC;IACrC,IAAI;QACF,MAAM,GAAG,IAAI,QAAQ,CAAC,eAAe,EAAE,QAAQ,EAAE,cAAc,EAAE,IAAI,CAAe,CAAC;KACtF;IAAC,OAAO,GAAG,EAAE;QACZ,IAAI,GAAG,YAAY,WAAW,EAAE;YAC9B,cAAc,GAAG,IAAI,CAAC;SACvB;aAAM;YACL,MAAM,GAAG,CAAC;SACX;KACF;IAED,wCAAwC;IACxC,IAAI,MAAM,KAAK,IAAI,EAAE;QACnB,OAAO;YACL,KAAK,EAAE,aAAa,EAAE,cAAc,EAAE,oBAAoB;SAC3D,CAAC;KACH;IAED,IAAI,iBAAiB,GAAG;QACtB,SAAS,CAAC,KAAa;YACrB,aAAa,GAAG,KAAK,CAAC;QACxB,CAAC;KACF,CAAC;IACF,qEAAqE;IACrE,oEAAoE;IACpE,uCAAuC;IACvC,oCAAoC;IACpC,cAAc;IACd,wDAAwD;IACxD,EAAE;IACF,4CAA4C;IAC5C,EAAE;IACF,gEAAgE;IAChE,0BAA0B;IAC1B,IAAI,eAAe,GAAG,IAAI,KAAK,CAAC,EAAE,EAAE;QAClC,GAAG,CAAC,OAAO,EAAE,QAAQ;YACnB,IAAI,OAAO,QAAQ,KAAK,QAAQ;gBAC9B,MAAM,IAAI,KAAK,CAAC,iDAAiD,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YACvF,OAAO,MAAM,UAAU;gBACrB;oBACE,oBAAoB,GAAG,QAAkB,CAAC;gBAC5C,CAAC;aACF,CAAA;QACH,CAAC;KACF,CAAC,CAAC;IAEH,0EAA0E;IAC1E,sDAAsD;IACtD,gCAAgC;IAChC,oCAAoC;IACpC,oBAAoB;IACpB,wDAAwD;IACxD,EAAE;IACF,oCAAoC;IACpC,EAAE;IACF,kDAAkD;IAClD,IAAI,oBAAoB,GAAG,IAAI,KAAK,CAAC,EAAE,EAAE;QACvC,GAAG,CAAC,OAAO,EAAE,QAAQ;YACnB,IAAI,OAAO,QAAQ,KAAK,QAAQ;gBAC9B,MAAM,IAAI,KAAK,CAAC,iDAAiD,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YACvF,OAAO,SAAS,gBAAgB;YAChC,CAAC,CAAA;QACH,CAAC;KACF,CAAC,CAAC;IAEH,IAAI;QACF,MAAM,CAAC,iBAAiB,EAAE,eAAe,EAAE,oBAAoB,CAAC,CAAC;KAClE;IAAC,OAAO,GAAG,EAAE;QACZ,KAAK,GAAG,GAAG,CAAC;KACb;IAED,OAAO;QACL,KAAK,EAAE,aAAa,EAAE,cAAc,EAAE,oBAAoB;KAC3D,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,OAAO,UAAU;IACb,QAAQ,GAAiB,EAAE,CAAC;IAEpC;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,GAAe;QACvB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,IAAyB;QACnC,OAAO,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED;;OAEG;IACH,eAAe;QACb,OAAO,IAAI,CAAC,QAAQ;aACjB,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,KAAK,QAAQ,CAAC,YAAY,CAAC;aAClD,MAAM,GAAG,CAAC,CAAC;IAChB,CAAC;IAED;;;;;OAKG;IACH,OAAO;QACL,qBAAqB,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACjD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;OAKG;IACH,SAAS;QACP,sBAAsB,EAAE,CAAA;IAC1B,CAAC;CACF"}

package/build/test/test-compile-model-with-pseudoclosure.d.ts

@@ -0,0 +1,2 @@
+import 'mocha';
+//# sourceMappingURL=test-compile-model-with-pseudoclosure.d.ts.map

package/build/test/test-compile-model-with-pseudoclosure.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-compile-model-with-pseudoclosure.d.ts","sourceRoot":"","sources":["../../test/test-compile-model-with-pseudoclosure.ts"],"names":[],"mappings":"AAEA,OAAO,OAAO,CAAC"}

package/build/test/test-compile-model-with-pseudoclosure.js

@@ -0,0 +1,200 @@
+import LexicalModelCompiler from '../src/lexical-model-compiler.js';
+import { assert } from 'chai';
+import 'mocha';
+import { makePathToFixture, compileModelSourceCode } from './helpers/index.js';
+describe('LexicalModelCompiler - pseudoclosure compilation + use', function () {
+    const MODEL_ID = 'example.qaa.trivial';
+    const PATH = makePathToFixture(MODEL_ID);
+    describe('specifying custom methods: applyCasing and searchTermToKey', function () {
+        let casingWithPrependedSymbols = function (casingName, text, defaultApplyCasing) {
+            switch (casingName) {
+                // Use of symbols, and of the `casingName` name, exist to serve as regex targets.
+                case 'lower':
+                    return '-' + defaultApplyCasing(casingName, text);
+                case 'upper':
+                    return '+' + defaultApplyCasing(casingName, text);
+                case 'initial':
+                    return '^' + defaultApplyCasing(casingName, text);
+                default:
+                    return defaultApplyCasing(casingName, text);
+            }
+        };
+        it('variant 1:  applyCasing prepends symbols, searchTermToKey removes them', function () {
+            let compiler = new LexicalModelCompiler;
+            let code = compiler.generateLexicalModelCode(MODEL_ID, {
+                format: 'trie-1.0',
+                sources: ['wordlist.tsv'],
+                languageUsesCasing: true,
+                applyCasing: casingWithPrependedSymbols,
+                // Parameter name `rawSearchTerm` selected for uniqueness, regex matching test target.
+                searchTermToKey: function (rawSearchTerm, applyCasing) {
+                    // Strips any applyCasing symbols ('-', '+', '^') out of the compiled Trie.
+                    // We should be able to test that they do _not_ occur within internal nodes of the Trie.
+                    return applyCasing('lower', rawSearchTerm)
+                        // We replace all prepended applyCasing symbols with a separate symbol not
+                        // otherwise seen within the model.
+                        .replace(/^-/, '§')
+                        .replace(/^\+/, '§')
+                        .replace(/^\^/, '§');
+                }
+            }, PATH);
+            // Check that pseudoclosure forwarding is set up correctly.
+            assert.match(code, /\.model\.searchTermToKey/);
+            assert.match(code, /\.model\.applyCasing/);
+            // Check that the methods actually made into the code; use our custom parameter names:
+            assert.match(code, /casingName/);
+            assert.match(code, /rawSearchTerm/);
+            // Check that the prepended symbols from applyCasing also appear in the code:
+            assert.match(code, /'-'/);
+            assert.match(code, /'\+'/);
+            assert.match(code, /'\^'/);
+            assert.match(code, /§/);
+            let modelInitIndex = code.indexOf('LMLayerWorker.loadModel');
+            let modelInitCode = code.substring(modelInitIndex);
+            // Chop off the IIFE terminator.  Not strictly necessary, but whatever.
+            modelInitCode = modelInitCode.substring(0, modelInitCode.lastIndexOf('\n})();'));
+            // Check that the prepended symbols from applyCasing do not appear in the Trie.
+            assert.notMatch(modelInitCode, /['"]-['"]/);
+            assert.notMatch(modelInitCode, /['"]\+['"]/);
+            assert.notMatch(modelInitCode, /['"]\^['"]/);
+            // Instead, our custom keyer should ensure that the following symbol DOES appear.
+            // Verifies that the compiler uses the custom searchTermToKey definition.
+            assert.match(modelInitCode, /['"]§['"]/);
+            // Make sure it compiles!
+            let compilation = compileModelSourceCode(code);
+            assert.isFalse(compilation.hasSyntaxError);
+            assert.isNotNull(compilation.exportedModel);
+        });
+        it('variant 2:  applyCasing prepends symbols, searchTermToKey keeps them', function () {
+            let compiler = new LexicalModelCompiler;
+            let code = compiler.generateLexicalModelCode(MODEL_ID, {
+                format: 'trie-1.0',
+                sources: ['wordlist.tsv'],
+                languageUsesCasing: true,
+                applyCasing: casingWithPrependedSymbols,
+                // Parameter name `rawSearchTerm` selected for uniqueness, regex matching test target.
+                searchTermToKey: function (rawSearchTerm, applyCasing) {
+                    // Strips any applyCasing symbols ('-', '+', '^') out of the compiled Trie.
+                    // We should be able to test that they do _not_ occur within internal nodes of the Trie.
+                    return applyCasing('lower', rawSearchTerm);
+                }
+            }, PATH);
+            // Check that pseudoclosure forwarding is set up correctly.
+            assert.match(code, /\.model\.searchTermToKey/);
+            assert.match(code, /\.model\.applyCasing/);
+            // Check that the methods actually made into the code; use our custom parameter names:
+            assert.match(code, /casingName/);
+            assert.match(code, /rawSearchTerm/);
+            // Check that the prepended symbols from applyCasing also appear in the code:
+            assert.match(code, /'-'/);
+            assert.match(code, /'\+'/);
+            assert.match(code, /'\^'/);
+            let modelInitIndex = code.indexOf('LMLayerWorker.loadModel');
+            let modelInitCode = code.substring(modelInitIndex);
+            // Chop off the IIFE terminator.  Not strictly necessary, but whatever.
+            modelInitCode = modelInitCode.substring(0, modelInitCode.lastIndexOf('\n})();'));
+            // Check that the prepended lowercase "-" DOES appear within the Trie, as keying
+            // does not remove it in this variant.  Verifies that the compiler actually
+            // used the custom applyCasing definition!
+            assert.match(modelInitCode, /['"]-['"]/);
+            // Make sure it compiles!
+            let compilation = compileModelSourceCode(code);
+            assert.isFalse(compilation.hasSyntaxError);
+            assert.isNotNull(compilation.exportedModel);
+        });
+        it('variant 3:  applyCasing prepends symbols, default searchTermToKey', function () {
+            let compiler = new LexicalModelCompiler;
+            let code = compiler.generateLexicalModelCode(MODEL_ID, {
+                format: 'trie-1.0',
+                sources: ['wordlist.tsv'],
+                languageUsesCasing: true,
+                applyCasing: casingWithPrependedSymbols
+            }, PATH);
+            // Check that pseudoclosure forwarding is set up correctly.
+            assert.match(code, /\.model\.searchTermToKey/);
+            assert.match(code, /\.model\.applyCasing/);
+            // Check that the methods actually made into the code; use our custom parameter names:
+            assert.match(code, /casingName/);
+            assert.match(code, /function defaultCasedSearchTermToKey/);
+            // Check that the prepended symbols from applyCasing also appear in the code:
+            assert.match(code, /'-'/);
+            assert.match(code, /'\+'/);
+            assert.match(code, /'\^'/);
+            let modelInitIndex = code.indexOf('LMLayerWorker.loadModel');
+            let modelInitCode = code.substring(modelInitIndex);
+            // Chop off the IIFE terminator.  Not strictly necessary, but whatever.
+            modelInitCode = modelInitCode.substring(0, modelInitCode.lastIndexOf('\n})();'));
+            // Check that the prepended lowercase "-" DOES appear within the Trie, as keying
+            // does not remove it in this variant.  Verifies that the compiler actually
+            // used the custom applyCasing definition!
+            assert.match(modelInitCode, /['"]-['"]/);
+            // Make sure it compiles!
+            let compilation = compileModelSourceCode(code);
+            assert.isFalse(compilation.hasSyntaxError);
+            assert.isNotNull(compilation.exportedModel);
+        });
+    });
+    describe('relying on default applyCasing + searchTermToKey', function () {
+        it('languageUsesCasing: true', function () {
+            let compiler = new LexicalModelCompiler;
+            let code = compiler.generateLexicalModelCode(MODEL_ID, {
+                format: 'trie-1.0',
+                sources: ['wordlist.tsv'],
+                languageUsesCasing: true, // applyCasing should appear!
+            }, PATH);
+            // Does the cased version of the search-term default appear?
+            assert.match(code, /function defaultCasedSearchTermToKey/);
+            assert.match(code, /\.model\.searchTermToKey/);
+            // Does the default casing function appear?
+            assert.match(code, /function defaultApplyCasing/);
+            assert.match(code, /\.defaults\.applyCasing/);
+            assert.match(code, /languageUsesCasing/);
+            // Make sure it compiles!
+            let compilation = compileModelSourceCode(code);
+            assert.isFalse(compilation.hasSyntaxError);
+            assert.isNotNull(compilation.exportedModel);
+        });
+        it('languageUsesCasing: false', function () {
+            let compiler = new LexicalModelCompiler;
+            let code = compiler.generateLexicalModelCode(MODEL_ID, {
+                format: 'trie-1.0',
+                sources: ['wordlist.tsv'],
+                languageUsesCasing: false, // applyCasing should not appear!
+            }, PATH);
+            // Does the cased version of the search-term default appear?
+            assert.match(code, /function defaultSearchTermToKey/);
+            assert.match(code, /\.model\.searchTermToKey/);
+            // Does the default casing function appear?
+            // Since `languageUsesCasing` is explicitly `false`, there should be no usage.
+            assert.notMatch(code, /function defaultApplyCasing/);
+            assert.notMatch(code, /\.defaults\.applyCasing/);
+            assert.match(code, /languageUsesCasing/);
+            // Make sure it compiles!
+            let compilation = compileModelSourceCode(code);
+            assert.isFalse(compilation.hasSyntaxError);
+            assert.isNotNull(compilation.exportedModel);
+        });
+        it('languageUsesCasing: undefined', function () {
+            let compiler = new LexicalModelCompiler;
+            let code = compiler.generateLexicalModelCode(MODEL_ID, {
+                format: 'trie-1.0',
+                sources: ['wordlist.tsv']
+                // languageUsesCasing: undefined
+            }, PATH);
+            // Does the cased version of the search-term default appear?
+            // Needed to match 14.0 defaults.
+            assert.match(code, /function defaultCasedSearchTermToKey/);
+            assert.match(code, /\.model\.searchTermToKey/);
+            // Does the default casing function appear?
+            assert.match(code, /function defaultApplyCasing/);
+            assert.match(code, /\.defaults\.applyCasing/);
+            // Did the compilation process explictly leave `languageUsesCasing` undefined?
+            assert.notMatch(code, /languageUsesCasing/);
+            // Make sure it compiles!
+            let compilation = compileModelSourceCode(code);
+            assert.isFalse(compilation.hasSyntaxError);
+            assert.isNotNull(compilation.exportedModel);
+        });
+    });
+});
+//# sourceMappingURL=test-compile-model-with-pseudoclosure.js.map

package/build/test/test-compile-model-with-pseudoclosure.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-compile-model-with-pseudoclosure.js","sourceRoot":"","sources":["../../test/test-compile-model-with-pseudoclosure.ts"],"names":[],"mappings":"AAAA,OAAO,oBAAoB,MAAM,kCAAkC,CAAC;AACpE,OAAO,EAAC,MAAM,EAAC,MAAM,MAAM,CAAC;AAC5B,OAAO,OAAO,CAAC;AAEf,OAAO,EAAE,iBAAiB,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAE/E,QAAQ,CAAC,wDAAwD,EAAE;IACjE,MAAM,QAAQ,GAAG,qBAAqB,CAAC;IACvC,MAAM,IAAI,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAEzC,QAAQ,CAAC,4DAA4D,EAAE;QACrE,IAAI,0BAA0B,GAAmB,UAAS,UAAsB,EAAE,IAAY,EAAE,kBAAkC;YAChI,QAAO,UAAU,EAAE;gBACjB,iFAAiF;gBACjF,KAAK,OAAO;oBACV,OAAO,GAAG,GAAG,kBAAkB,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;gBACpD,KAAK,OAAO;oBACV,OAAO,GAAG,GAAG,kBAAkB,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;gBACpD,KAAK,SAAS;oBACZ,OAAO,GAAG,GAAG,kBAAkB,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;gBACpD;oBACE,OAAO,kBAAkB,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;aAC/C;QACH,CAAC,CAAC;QAEF,EAAE,CAAC,wEAAwE,EAAE;YAC3E,IAAI,QAAQ,GAAG,IAAI,oBAAoB,CAAC;YACxC,IAAI,IAAI,GAAG,QAAQ,CAAC,wBAAwB,CAAC,QAAQ,EAAE;gBACrD,MAAM,EAAE,UAAU;gBAClB,OAAO,EAAE,CAAC,cAAc,CAAC;gBACzB,kBAAkB,EAAE,IAAI;gBACxB,WAAW,EAAE,0BAA0B;gBACvC,sFAAsF;gBACtF,eAAe,EAAE,UAAS,aAAqB,EAAE,WAA2B;oBAC1E,2EAA2E;oBAC3E,wFAAwF;oBACxF,OAAO,WAAW,CAAC,OAAO,EAAE,aAAa,CAAC;wBACxC,0EAA0E;wBAC1E,mCAAmC;yBAClC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC;yBAClB,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;yBACnB,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;gBACzB,CAAC;aACF,EAAE,IAAI,CAAW,CAAC;YAEnB,2DAA2D;YAC3D,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,0BAA0B,CAAC,CAAC;YAC/C,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,sBAAsB,CAAC,CAAC;YAE3C,sFAAsF;YACtF,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;YACjC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC;YAEpC,6EAA6E;YAC7E,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC1B,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC3B,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC3B,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAExB,IAAI,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,yBAAyB,CAAC,CAAC;YAC7D,IAAI,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;YACnD,uEAAuE;YACvE,aAAa,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC,EAAE,aAAa,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAA;YAEhF,+EAA+E;YAC/E,MAAM,CAAC,QAAQ,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;YAC5C,MAAM,CAAC,QAAQ,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;YAC7C,MAAM,CAAC,QAAQ,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;YAE7C,iFAAiF;YACjF,yEAAyE;YACzE,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;YAEzC,yBAAyB;YACzB,IAAI,WAAW,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;YAC/C,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC;YAC3C,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sEAAsE,EAAE;YACzE,IAAI,QAAQ,GAAG,IAAI,oBAAoB,CAAC;YACxC,IAAI,IAAI,GAAG,QAAQ,CAAC,wBAAwB,CAAC,QAAQ,EAAE;gBACrD,MAAM,EAAE,UAAU;gBAClB,OAAO,EAAE,CAAC,cAAc,CAAC;gBACzB,kBAAkB,EAAE,IAAI;gBACxB,WAAW,EAAE,0BAA0B;gBACvC,sFAAsF;gBACtF,eAAe,EAAE,UAAS,aAAqB,EAAE,WAA2B;oBAC1E,2EAA2E;oBAC3E,wFAAwF;oBACxF,OAAO,WAAW,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;gBAC7C,CAAC;aACF,EAAE,IAAI,CAAW,CAAC;YAEnB,2DAA2D;YAC3D,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,0BAA0B,CAAC,CAAC;YAC/C,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,sBAAsB,CAAC,CAAC;YAE3C,sFAAsF;YACtF,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;YACjC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC;YAEpC,6EAA6E;YAC7E,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC1B,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC3B,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAE3B,IAAI,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,yBAAyB,CAAC,CAAC;YAC7D,IAAI,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;YACnD,uEAAuE;YACvE,aAAa,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC,EAAE,aAAa,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAA;YAEhF,gFAAgF;YAChF,2EAA2E;YAC3E,0CAA0C;YAC1C,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;YAEzC,yBAAyB;YACzB,IAAI,WAAW,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;YAC/C,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC;YAC3C,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mEAAmE,EAAE;YACtE,IAAI,QAAQ,GAAG,IAAI,oBAAoB,CAAC;YACxC,IAAI,IAAI,GAAG,QAAQ,CAAC,wBAAwB,CAAC,QAAQ,EAAE;gBACrD,MAAM,EAAE,UAAU;gBAClB,OAAO,EAAE,CAAC,cAAc,CAAC;gBACzB,kBAAkB,EAAE,IAAI;gBACxB,WAAW,EAAE,0BAA0B;aACxC,EAAE,IAAI,CAAW,CAAC;YAEnB,2DAA2D;YAC3D,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,0BAA0B,CAAC,CAAC;YAC/C,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,sBAAsB,CAAC,CAAC;YAE3C,sFAAsF;YACtF,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;YACjC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,sCAAsC,CAAC,CAAC;YAE3D,6EAA6E;YAC7E,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC1B,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC3B,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAE3B,IAAI,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,yBAAyB,CAAC,CAAC;YAC7D,IAAI,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;YACnD,uEAAuE;YACvE,aAAa,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC,EAAE,aAAa,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAA;YAEhF,gFAAgF;YAChF,2EAA2E;YAC3E,0CAA0C;YAC1C,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;YAEzC,yBAAyB;YACzB,IAAI,WAAW,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;YAC/C,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC;YAC3C,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,kDAAkD,EAAE;QAC3D,EAAE,CAAC,0BAA0B,EAAE;YAC7B,IAAI,QAAQ,GAAG,IAAI,oBAAoB,CAAC;YACxC,IAAI,IAAI,GAAG,QAAQ,CAAC,wBAAwB,CAAC,QAAQ,EAAE;gBACrD,MAAM,EAAE,UAAU;gBAClB,OAAO,EAAE,CAAC,cAAc,CAAC;gBACzB,kBAAkB,EAAE,IAAI,EAAE,6BAA6B;aACxD,EAAE,IAAI,CAAW,CAAC;YAEnB,4DAA4D;YAC5D,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,sCAAsC,CAAC,CAAC;YAC3D,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,0BAA0B,CAAC,CAAC;YAE/C,2CAA2C;YAC3C,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,6BAA6B,CAAC,CAAC;YAClD,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,yBAAyB,CAAC,CAAC;YAE9C,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;YAEzC,yBAAyB;YACzB,IAAI,WAAW,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;YAC/C,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC;YAC3C,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2BAA2B,EAAE;YAC9B,IAAI,QAAQ,GAAG,IAAI,oBAAoB,CAAC;YACxC,IAAI,IAAI,GAAG,QAAQ,CAAC,wBAAwB,CAAC,QAAQ,EAAE;gBACrD,MAAM,EAAE,UAAU;gBAClB,OAAO,EAAE,CAAC,cAAc,CAAC;gBACzB,kBAAkB,EAAE,KAAK,EAAE,iCAAiC;aAC7D,EAAE,IAAI,CAAW,CAAC;YAEnB,4DAA4D;YAC5D,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,iCAAiC,CAAC,CAAC;YACtD,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,0BAA0B,CAAC,CAAC;YAE/C,2CAA2C;YAC3C,8EAA8E;YAC9E,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,6BAA6B,CAAC,CAAC;YACrD,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,yBAAyB,CAAC,CAAC;YAEjD,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;YAEzC,yBAAyB;YACzB,IAAI,WAAW,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;YAC/C,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC;YAC3C,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE;YAClC,IAAI,QAAQ,GAAG,IAAI,oBAAoB,CAAC;YACxC,IAAI,IAAI,GAAG,QAAQ,CAAC,wBAAwB,CAAC,QAAQ,EAAE;gBACrD,MAAM,EAAE,UAAU;gBAClB,OAAO,EAAE,CAAC,cAAc,CAAC;gBACzB,gCAAgC;aACjC,EAAE,IAAI,CAAW,CAAC;YAEnB,4DAA4D;YAC5D,iCAAiC;YACjC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,sCAAsC,CAAC,CAAC;YAC3D,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,0BAA0B,CAAC,CAAC;YAE/C,2CAA2C;YAC3C,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,6BAA6B,CAAC,CAAC;YAClD,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,yBAAyB,CAAC,CAAC;YAE9C,8EAA8E;YAC9E,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;YAE5C,yBAAyB;YACzB,IAAI,WAAW,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;YAC/C,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC;YAC3C,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAC"}

package/build/test/test-compile-model.d.ts

@@ -0,0 +1,2 @@
+import 'mocha';
+//# sourceMappingURL=test-compile-model.d.ts.map

package/build/test/test-compile-model.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-compile-model.d.ts","sourceRoot":"","sources":["../../test/test-compile-model.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,CAAC"}

package/build/test/test-compile-model.js

@@ -0,0 +1,30 @@
+import 'mocha';
+import { assert } from 'chai';
+import { compileModel } from '../src/main.js';
+import { makePathToFixture, compileModelSourceCode } from './helpers/index.js';
+describe('compileModel', function () {
+    // Try to compile ALL of the correct models.
+    const MODELS = [
+        'example.qaa.sencoten',
+        'example.qaa.trivial',
+        'example.qaa.utf16le',
+        'example.qaa.wordbreaker',
+        'example.qaa.joinwordbreaker',
+        'example.qaa.scriptusesspaces',
+    ];
+    for (let modelID of MODELS) {
+        let modelPath = makePathToFixture(modelID, `${modelID}.model.ts`);
+        it(`should compile ${modelID}`, function () {
+            let code = compileModel(modelPath);
+            let r;
+            assert.doesNotThrow(() => {
+                r = compileModelSourceCode(code);
+            });
+            let compilation = r;
+            assert.isFalse(compilation.hasSyntaxError, 'model code had syntax error');
+            assert.isNull(compilation.error, `compilation error: ${compilation.error}`);
+            assert.equal(compilation.modelConstructorName, 'TrieModel');
+        });
+    }
+});
+//# sourceMappingURL=test-compile-model.js.map

package/build/test/test-compile-model.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-compile-model.js","sourceRoot":"","sources":["../../test/test-compile-model.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,CAAC;AACf,OAAO,EAAC,MAAM,EAAC,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAC,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAC,iBAAiB,EAAE,sBAAsB,EAAoB,MAAM,oBAAoB,CAAC;AAEhG,QAAQ,CAAC,cAAc,EAAE;IACvB,4CAA4C;IAC5C,MAAM,MAAM,GAAG;QACb,sBAAsB;QACtB,qBAAqB;QACrB,qBAAqB;QACrB,yBAAyB;QACzB,6BAA6B;QAC7B,8BAA8B;KAC/B,CAAC;IAEF,KAAK,IAAI,OAAO,IAAI,MAAM,EAAE;QAC1B,IAAI,SAAS,GAAG,iBAAiB,CAAC,OAAO,EAAE,GAAG,OAAO,WAAW,CAAC,CAAC;QAElE,EAAE,CAAC,kBAAkB,OAAO,EAAE,EAAE;YAC9B,IAAI,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;YACnC,IAAI,CAAU,CAAC;YACf,MAAM,CAAC,YAAY,CAAC,GAAG,EAAE;gBACvB,CAAC,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;YACnC,CAAC,CAAC,CAAC;YACH,IAAI,WAAW,GAAG,CAAsB,CAAC;YAEzC,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,cAAc,EAAE,6BAA6B,CAAC,CAAC;YAC1E,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE,sBAAsB,WAAW,CAAC,KAAK,EAAE,CAAC,CAAC;YAC5E,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,oBAAoB,EAAE,WAAW,CAAC,CAAC;QAC9D,CAAC,CAAC,CAAC;KACJ;AACH,CAAC,CAAC,CAAC"}

package/build/test/test-compile-trie.d.ts

@@ -0,0 +1,2 @@
+import 'mocha';
+//# sourceMappingURL=test-compile-trie.d.ts.map

package/build/test/test-compile-trie.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-compile-trie.d.ts","sourceRoot":"","sources":["../../test/test-compile-trie.ts"],"names":[],"mappings":"AAEA,OAAO,OAAO,CAAC"}

package/build/test/test-compile-trie.js

@@ -0,0 +1,125 @@
+import LexicalModelCompiler from '../src/lexical-model-compiler.js';
+import { assert } from 'chai';
+import 'mocha';
+import { makePathToFixture, compileModelSourceCode } from './helpers/index.js';
+import { createTrieDataStructure } from '../src/build-trie.js';
+describe('LexicalModelCompiler', function () {
+    describe('#generateLexicalModelCode', function () {
+        it('should compile a trivial word list', function () {
+            const MODEL_ID = 'example.qaa.trivial';
+            const PATH = makePathToFixture(MODEL_ID);
+            let compiler = new LexicalModelCompiler;
+            let code = compiler.generateLexicalModelCode(MODEL_ID, {
+                format: 'trie-1.0',
+                sources: ['wordlist.tsv']
+            }, PATH);
+            let result = compileModelSourceCode(code);
+            assert.isFalse(result.hasSyntaxError);
+            assert.isNotNull(result.exportedModel);
+            assert.equal(result.modelConstructorName, 'TrieModel');
+            // Sanity check: the word list has three total unweighted words, with a
+            // total weight of 3!
+            assert.match(code, /\btotalWeight\b["']?:\s*3\b/);
+            // Sanity check: the word breaker is a property of the object.
+            assert.match(code, /\bwordBreaker\b["']?:\s*wordBreakers\b/);
+        });
+        it('should compile a word list exported by Microsoft Excel', function () {
+            const MODEL_ID = 'example.qaa.utf16le';
+            const PATH = makePathToFixture(MODEL_ID);
+            let compiler = new LexicalModelCompiler;
+            let code = compiler.generateLexicalModelCode(MODEL_ID, {
+                format: 'trie-1.0',
+                sources: ['wordlist.txt']
+            }, PATH);
+            let result = compileModelSourceCode(code);
+            assert.isFalse(result.hasSyntaxError);
+            assert.isNotNull(result.exportedModel);
+            assert.equal(result.modelConstructorName, 'TrieModel');
+            // Sanity check: the word list has three total unweighted words, with a
+            // total weight of 44,103!
+            assert.match(code, /\btotalWeight\b["']?:\s*44103\b/);
+        });
+    });
+    it('should compile a word list with a custom word breaking function', function () {
+        const MODEL_ID = 'example.qaa.trivial';
+        const PATH = makePathToFixture(MODEL_ID);
+        let compiler = new LexicalModelCompiler;
+        let code = compiler.generateLexicalModelCode(MODEL_ID, {
+            format: 'trie-1.0',
+            sources: ['wordlist.tsv'],
+            // This is a possible word breaking function:
+            wordBreaker(phrase) {
+                return [];
+            }
+        }, PATH);
+        let result = compileModelSourceCode(code);
+        assert.isFalse(result.hasSyntaxError, `Syntax error in ${code}`);
+        assert.isNotNull(result.exportedModel);
+        assert.equal(result.modelConstructorName, 'TrieModel');
+        // Sanity check: the word breaker is a property of the object.
+        assert.match(code, /\bwordBreaker\b["']?:\s+function\b/);
+    });
+    it('should not generate unpaired surrogate code units', function () {
+        const MODEL_ID = 'example.qaa.smp';
+        const PATH = makePathToFixture(MODEL_ID);
+        let compiler = new LexicalModelCompiler;
+        let code = compiler.generateLexicalModelCode(MODEL_ID, {
+            format: 'trie-1.0',
+            sources: ['wordlist.tsv']
+        }, PATH);
+        let result = compileModelSourceCode(code);
+        assert.isFalse(result.hasSyntaxError);
+        assert.isNotNull(result.exportedModel);
+        assert.equal(result.modelConstructorName, 'TrieModel');
+        // Test every character in the string to make sure we don't have
+        // unpaired surrogates which destroy everything.
+        // We can assume that the first and last chars are not SMP
+        for (var i = 1; i < code.length - 1; i++) {
+            assert.notEqual(0xFFFD, code.charCodeAt(i));
+            if (code.charCodeAt(i) >= 0xD800 && code.charCodeAt(i) < 0xDC00) {
+                assert.isTrue((code.charCodeAt(i + 1) >= 0xDC00 && code.charCodeAt(i + 1) < 0xE000), 'Unpaired lead surrogate U+' + code.charCodeAt(i).toString(16) + ' at position ' + i + ' of \'' + code + '\'');
+            }
+            else if (code.charCodeAt(i) >= 0xDC00 && code.charCodeAt(i) < 0xE000) {
+                assert.isTrue((code.charCodeAt(i - 1) >= 0xD800 && code.charCodeAt(i - 1) < 0xDC00), 'Unpaired trail surrogate U+' + code.charCodeAt(i).toString(16) + ' at position ' + i + ' of \'' + code + '\'');
+            }
+        }
+        // Sanity check: the word list has three total unweighted words, with a
+        // total weight of 27,596!
+        assert.match(code, /\btotalWeight\b["']?:\s*27596\b/);
+    });
+    it('should include the source code of its search term to key function', function () {
+        const MODEL_ID = 'example.qaa.trivial';
+        const PATH = makePathToFixture(MODEL_ID);
+        let compiler = new LexicalModelCompiler;
+        let code = compiler.generateLexicalModelCode(MODEL_ID, {
+            format: 'trie-1.0',
+            sources: ['wordlist.tsv']
+            // NOTE: we intentionally OMIT the searchTermToKey function
+            // so that the default can be provided.
+        }, PATH);
+        assert.match(code, /(["']|)searchTermToKey\1:\s*function\b/, 'expected to find searchTermToKey specified as a function');
+    });
+});
+describe('createTrieDataStructure()', function () {
+    const WORDLIST_FILENAME = makePathToFixture('example.qaa.trivial', 'wordlist.tsv');
+    it('must be given an explicit searchTermToKey function', function () {
+        assert.throws(function () {
+            createTrieDataStructure([WORDLIST_FILENAME]);
+        }, TypeError);
+    });
+    it('uses the provided searchTermToKey function', function () {
+        // check if the expected key is in the resultant data structure.
+        // N.B., we assume the wordlist contains the wordform "turtles"
+        let lowercaseSourceCode = createTrieDataStructure([WORDLIST_FILENAME], (wf) => {
+            return wf.toLowerCase();
+        });
+        assert.match(lowercaseSourceCode, /"key":\s*"turtles"/);
+        assert.notMatch(lowercaseSourceCode, /"key":\s*"TURTLES"/);
+        let uppercaseSourceCode = createTrieDataStructure([WORDLIST_FILENAME], (wf) => {
+            return wf.toUpperCase();
+        });
+        assert.match(uppercaseSourceCode, /"key":\s*"TURTLES"/);
+        assert.notMatch(uppercaseSourceCode, /"key":\s*"turtles"/);
+    });
+});
+//# sourceMappingURL=test-compile-trie.js.map

package/build/test/test-compile-trie.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-compile-trie.js","sourceRoot":"","sources":["../../test/test-compile-trie.ts"],"names":[],"mappings":"AAAA,OAAO,oBAAoB,MAAM,kCAAkC,CAAC;AACpE,OAAO,EAAC,MAAM,EAAC,MAAM,MAAM,CAAC;AAC5B,OAAO,OAAO,CAAC;AAEf,OAAO,EAAC,iBAAiB,EAAE,sBAAsB,EAAC,MAAM,oBAAoB,CAAC;AAC7E,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAE/D,QAAQ,CAAC,sBAAsB,EAAE;IAC/B,QAAQ,CAAC,2BAA2B,EAAE;QACpC,EAAE,CAAC,oCAAoC,EAAE;YACvC,MAAM,QAAQ,GAAG,qBAAqB,CAAC;YACvC,MAAM,IAAI,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;YAEzC,IAAI,QAAQ,GAAG,IAAI,oBAAoB,CAAC;YACxC,IAAI,IAAI,GAAG,QAAQ,CAAC,wBAAwB,CAAC,QAAQ,EAAE;gBACrD,MAAM,EAAE,UAAU;gBAClB,OAAO,EAAE,CAAC,cAAc,CAAC;aAC1B,EAAE,IAAI,CAAW,CAAC;YAEnB,IAAI,MAAM,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;YAC1C,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;YACtC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACvC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,oBAAoB,EAAE,WAAW,CAAC,CAAC;YAEvD,uEAAuE;YACvE,qBAAqB;YACrB,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,6BAA6B,CAAC,CAAC;YAElD,8DAA8D;YAC9D,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,wCAAwC,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wDAAwD,EAAE;YAC3D,MAAM,QAAQ,GAAG,qBAAqB,CAAC;YACvC,MAAM,IAAI,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;YAEzC,IAAI,QAAQ,GAAG,IAAI,oBAAoB,CAAC;YACxC,IAAI,IAAI,GAAG,QAAQ,CAAC,wBAAwB,CAAC,QAAQ,EAAE;gBACrD,MAAM,EAAE,UAAU;gBAClB,OAAO,EAAE,CAAC,cAAc,CAAC;aAC1B,EAAE,IAAI,CAAW,CAAC;YAEnB,IAAI,MAAM,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;YAC1C,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;YACtC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACvC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,oBAAoB,EAAE,WAAW,CAAC,CAAC;YAEvD,uEAAuE;YACvE,0BAA0B;YAC1B,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,iCAAiC,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iEAAiE,EAAE;QACpE,MAAM,QAAQ,GAAG,qBAAqB,CAAC;QACvC,MAAM,IAAI,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAEzC,IAAI,QAAQ,GAAG,IAAI,oBAAoB,CAAC;QACxC,IAAI,IAAI,GAAG,QAAQ,CAAC,wBAAwB,CAAC,QAAQ,EAAE;YACrD,MAAM,EAAE,UAAU;YAClB,OAAO,EAAE,CAAC,cAAc,CAAC;YACzB,6CAA6C;YAC7C,WAAW,CAAC,MAAc;gBACxB,OAAO,EAAE,CAAC;YACZ,CAAC;SACF,EAAE,IAAI,CAAW,CAAC;QAEnB,IAAI,MAAM,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,EAAE,mBAAmB,IAAI,EAAE,CAAC,CAAC;QACjE,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QACvC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,oBAAoB,EAAE,WAAW,CAAC,CAAC;QAEvD,8DAA8D;QAC9D,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,oCAAoC,CAAC,CAAC;IAC3D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE;QACtD,MAAM,QAAQ,GAAG,iBAAiB,CAAC;QACnC,MAAM,IAAI,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAEzC,IAAI,QAAQ,GAAG,IAAI,oBAAoB,CAAC;QACxC,IAAI,IAAI,GAAG,QAAQ,CAAC,wBAAwB,CAAC,QAAQ,EAAE;YACrD,MAAM,EAAE,UAAU;YAClB,OAAO,EAAE,CAAC,cAAc,CAAC;SAC1B,EAAE,IAAI,CAAW,CAAC;QAEnB,IAAI,MAAM,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;QACtC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QACvC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,oBAAoB,EAAE,WAAW,CAAC,CAAC;QAEvD,gEAAgE;QAChE,gDAAgD;QAChD,0DAA0D;QAC1D,KAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE;YACvC,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;YAC5C,IAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,MAAM,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,MAAM,EAAE;gBAC9D,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,GAAC,CAAC,CAAC,IAAI,MAAM,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,GAAC,CAAC,CAAC,GAAG,MAAM,CAAC,EAC7E,4BAA4B,GAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAC,eAAe,GAAC,CAAC,GAAC,QAAQ,GAAC,IAAI,GAAC,IAAI,CAAC,CAAC;aACtG;iBAAM,IAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,MAAM,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,MAAM,EAAE;gBACrE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,GAAC,CAAC,CAAC,IAAI,MAAM,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,GAAC,CAAC,CAAC,GAAG,MAAM,CAAC,EAC7E,6BAA6B,GAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAC,eAAe,GAAC,CAAC,GAAC,QAAQ,GAAC,IAAI,GAAC,IAAI,CAAC,CAAC;aACvG;SACF;QAED,uEAAuE;QACvE,0BAA0B;QAC1B,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,iCAAiC,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mEAAmE,EAAE;QACtE,MAAM,QAAQ,GAAG,qBAAqB,CAAC;QACvC,MAAM,IAAI,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,QAAQ,GAAG,IAAI,oBAAoB,CAAC;QACxC,IAAI,IAAI,GAAG,QAAQ,CAAC,wBAAwB,CAAC,QAAQ,EAAE;YACrD,MAAM,EAAE,UAAU;YAClB,OAAO,EAAE,CAAC,cAAc,CAAC;YACzB,2DAA2D;YAC3D,uCAAuC;SACxC,EAAE,IAAI,CAAW,CAAC;QAEnB,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,wCAAwC,EACzD,0DAA0D,CAC3D,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,2BAA2B,EAAE;IACpC,MAAM,iBAAiB,GAAG,iBAAiB,CAAC,qBAAqB,EAAE,cAAc,CAAC,CAAC;IAEnF,EAAE,CAAC,oDAAoD,EAAE;QACvD,MAAM,CAAC,MAAM,CAAC;YACZ,uBAAuB,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC;QAC/C,CAAC,EAAE,SAAS,CAAC,CAAA;IACf,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4CAA4C,EAAE;QAC/C,gEAAgE;QAChE,+DAA+D;QAC/D,IAAI,mBAAmB,GAAG,uBAAuB,CAAC,CAAC,iBAAiB,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE;YAC5E,OAAO,EAAE,CAAC,WAAW,EAAE,CAAA;QACzB,CAAC,CAAC,CAAA;QACF,MAAM,CAAC,KAAK,CAAC,mBAAmB,EAAE,oBAAoB,CAAC,CAAC;QACxD,MAAM,CAAC,QAAQ,CAAC,mBAAmB,EAAE,oBAAoB,CAAC,CAAC;QAE3D,IAAI,mBAAmB,GAAG,uBAAuB,CAAC,CAAC,iBAAiB,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE;YAC5E,OAAO,EAAE,CAAC,WAAW,EAAE,CAAA;QACzB,CAAC,CAAC,CAAA;QACF,MAAM,CAAC,KAAK,CAAC,mBAAmB,EAAE,oBAAoB,CAAC,CAAC;QACxD,MAAM,CAAC,QAAQ,CAAC,mBAAmB,EAAE,oBAAoB,CAAC,CAAC;IAC7D,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAC"}

package/build/test/test-default-apply-case.d.ts

@@ -0,0 +1,2 @@
+import 'mocha';
+//# sourceMappingURL=test-default-apply-case.d.ts.map

package/build/test/test-default-apply-case.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-default-apply-case.d.ts","sourceRoot":"","sources":["../../test/test-default-apply-case.ts"],"names":[],"mappings":"AACA,OAAO,OAAO,CAAC"}