@keymanapp/kmc-model 17.0.85-alpha

package/build/src/build-trie.d.ts

@@ -0,0 +1,40 @@
+/**
+ * A word list is (conceptually) an array of pairs: the concrete word form itself + a
+ * non-negative count.
+ *
+ * Since each word should only appear once within the list, we represent it with
+ * an associative array pattern keyed by the wordform.
+ */
+export type WordList = {
+    [wordform: string]: number;
+};
+/**
+ * Returns a data structure that can be loaded by the TrieModel.
+ *
+ * It implements a **weighted** trie, whose indices (paths down the trie) are
+ * generated by a search key, and not concrete wordforms themselves.
+ *
+ * @param sourceFiles an array of source files that will be read to generate the trie.
+ */
+export declare function createTrieDataStructure(filenames: string[], searchTermToKey?: (wf: string) => string): string;
+/**
+ * Parses a word list from a file, merging duplicate entries.
+ *
+ * The word list may be encoded in:
+ *
+ *  - UTF-8, with or without BOM [exported by most software]
+ *  - UTF-16, little endian, with BOM [exported by Microsoft Excel]
+ *
+ * @param wordlist word list to merge entries into (may have existing entries)
+ * @param filename filename of the word list
+ */
+export declare function parseWordListFromFilename(wordlist: WordList, filename: string): void;
+/**
+ * Parses a word list from a string. The string should have multiple lines
+ * with LF or CRLF line terminators.
+ *
+ * @param wordlist word list to merge entries into (may have existing entries)
+ * @param filename filename of the word list
+ */
+export declare function parseWordListFromContents(wordlist: WordList, contents: string): void;
+//# sourceMappingURL=build-trie.d.ts.map

package/build/src/build-trie.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"build-trie.d.ts","sourceRoot":"","sources":["../../src/build-trie.ts"],"names":[],"mappings":"AAMA;;;;;;GAMG;AACH,MAAM,MAAM,QAAQ,GAAG;IAAC,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAA;CAAC,CAAC;AAEpD;;;;;;;GAOG;AACH,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,eAAe,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM,CAU7G;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,CAEpF;AAED;;;;;;GAMG;AACH,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,CAEpF"}

package/build/src/build-trie.js

@@ -0,0 +1,362 @@
+import { readFileSync } from "fs";
+import { log, KeymanCompilerError } from "./model-compiler-errors.js";
+// Supports LF or CRLF line terminators.
+const NEWLINE_SEPARATOR = /\u000d?\u000a/;
+/**
+ * Returns a data structure that can be loaded by the TrieModel.
+ *
+ * It implements a **weighted** trie, whose indices (paths down the trie) are
+ * generated by a search key, and not concrete wordforms themselves.
+ *
+ * @param sourceFiles an array of source files that will be read to generate the trie.
+ */
+export function createTrieDataStructure(filenames, searchTermToKey) {
+    if (typeof searchTermToKey !== "function") {
+        throw new TypeError("searchTermToKey must be explicitly specified");
+    }
+    // Make one big word list out of all of the filenames provided.
+    let wordlist = {};
+    filenames.forEach(filename => parseWordListFromFilename(wordlist, filename));
+    let trie = Trie.buildTrie(wordlist, searchTermToKey);
+    return JSON.stringify(trie);
+}
+/**
+ * Parses a word list from a file, merging duplicate entries.
+ *
+ * The word list may be encoded in:
+ *
+ *  - UTF-8, with or without BOM [exported by most software]
+ *  - UTF-16, little endian, with BOM [exported by Microsoft Excel]
+ *
+ * @param wordlist word list to merge entries into (may have existing entries)
+ * @param filename filename of the word list
+ */
+export function parseWordListFromFilename(wordlist, filename) {
+    _parseWordList(wordlist, new WordListFromFilename(filename));
+}
+/**
+ * Parses a word list from a string. The string should have multiple lines
+ * with LF or CRLF line terminators.
+ *
+ * @param wordlist word list to merge entries into (may have existing entries)
+ * @param filename filename of the word list
+ */
+export function parseWordListFromContents(wordlist, contents) {
+    _parseWordList(wordlist, new WordListFromMemory(contents));
+}
+/**
+ * Reads a tab-separated values file into a word list. This function converts all
+ * entries into NFC and merges duplicate entries across wordlists. Duplication is
+ * on the basis of character-for-character equality after normalisation to NFC.
+ *
+ * Format specification:
+ *
+ *  - the file is a UTF-8 encoded text file.
+ *  - new lines are either LF or CRLF.
+ *  - the file MAY start with the UTF-8 byte-order mark (BOM); that is, if the
+ *    first three bytes of the file are EF BB BF, these will be interepreted as
+ *    the BOM and will be ignored.
+ *  - the file either consists of a comment or an entry.
+ *  - comment lines MUST start with the '#' character on the very first column.
+ *  - entries are one to three columns, separated by the (horizontal) tab
+ *    character.
+ *  - column 1 (REQUIRED): the wordform: can have any character except tab, CR,
+ *    LF. Surrounding whitespace characters are trimmed.
+ *  - column 2 (optional): the count: a non-negative integer specifying how many
+ *    times this entry has appeared in the corpus. Blank means 'indeterminate';
+ *    commas are permissible in the digits.
+ *  - column 3 (optional): comment: an informative comment, ignored by the tool.
+ *
+ * @param wordlist word list to merge entries into (may have existing entries)
+ * @param contents contents of the file to import
+ */
+function _parseWordList(wordlist, source) {
+    const TAB = "\t";
+    let wordsSeenInThisFile = new Set();
+    for (let [lineno, line] of source.lines()) {
+        // Remove the byte-order mark (BOM) from the beginning of the string.
+        // Because `contents` can be the concatenation of several files, we have to remove
+        // the BOM from every possible start of file -- i.e., beginning of every line.
+        line = line.replace(/^\uFEFF/, '').trim();
+        if (line.startsWith('#') || line === "") {
+            continue; // skip comments and empty lines
+        }
+        // The third column is the comment. Always ignored!
+        let [wordform, countText] = line.split(TAB);
+        // Clean the word form.
+        let original = wordform;
+        wordform = wordform.normalize('NFC');
+        if (original !== wordform) {
+            // Mixed normalization forms are yucky! Warn about it.
+            log(KeymanCompilerError.CWARN_MixedNormalizationForms, `“${wordform}” is not in Unicode NFC. Automatically converting to NFC.`, { filename: source.name, lineno });
+        }
+        wordform = wordform.trim();
+        countText = (countText || '').trim().replace(/,/g, '');
+        let count = parseInt(countText, 10);
+        // When parsing a decimal integer fails (e.g., blank or something else):
+        if (!isFinite(count) || count < 0) {
+            // TODO: is this the right thing to do?
+            // Treat it like a hapax legonmenom -- it exist, but only once.
+            count = 1;
+        }
+        if (wordsSeenInThisFile.has(wordform)) {
+            // The same word seen across multiple files is fine,
+            // but a word seen multiple times in one file is a problem!
+            log(KeymanCompilerError.CWARN_DuplicateWordInSameFile, `duplicate word “${wordform}” found in same file; summing counts`, { filename: source.name, lineno });
+        }
+        wordsSeenInThisFile.add(wordform);
+        wordlist[wordform] = (isNaN(wordlist[wordform]) ? 0 : wordlist[wordform] || 0) + count;
+    }
+}
+class WordListFromMemory {
+    name = '<memory>';
+    _contents;
+    constructor(contents) {
+        this._contents = contents;
+    }
+    *lines() {
+        yield* enumerateLines(this._contents.split(NEWLINE_SEPARATOR));
+    }
+}
+class WordListFromFilename {
+    name;
+    constructor(filename) {
+        this.name = filename;
+    }
+    *lines() {
+        let contents = readFileSync(this.name, detectEncoding(this.name));
+        yield* enumerateLines(contents.split(NEWLINE_SEPARATOR));
+    }
+}
+/**
+ * Yields pairs of [lineno, line], given an Array of lines.
+ */
+function* enumerateLines(lines) {
+    let i = 1;
+    for (let line of lines) {
+        yield [i, line];
+        i++;
+    }
+}
+var Trie;
+(function (Trie_1) {
+    /**
+     * A sentinel value for when an internal node has contents and requires an
+     * "internal" leaf. That is, this internal node has content. Instead of placing
+     * entries as children in an internal node, a "fake" leaf is created, and its
+     * key is this special internal value.
+     *
+     * The value is a valid Unicode BMP code point, but it is a "non-character".
+     * Unicode will never assign semantics to these characters, as they are
+     * intended to be used internally as sentinel values.
+     */
+    const INTERNAL_VALUE = '\uFDD0';
+    /**
+     * Builds a trie from a word list.
+     *
+     * @param wordlist    The wordlist with non-negative weights.
+     * @param keyFunction Function that converts word forms into indexed search keys
+     * @returns A JSON-serialiable object that can be given to the TrieModel constructor.
+     */
+    function buildTrie(wordlist, keyFunction) {
+        let root = new Trie(keyFunction).buildFromWordList(wordlist).root;
+        return {
+            totalWeight: sumWeights(root),
+            root: root
+        };
+    }
+    Trie_1.buildTrie = buildTrie;
+    /**
+     * Wrapper class for the trie and its nodes and wordform to search
+     */
+    class Trie {
+        root = createRootNode();
+        toKey;
+        constructor(wordform2key) {
+            this.toKey = wordform2key;
+        }
+        /**
+         * Populates the trie with the contents of an entire wordlist.
+         * @param words a list of word and count pairs.
+         */
+        buildFromWordList(words) {
+            for (let [wordform, weight] of Object.entries(words)) {
+                let key = this.toKey(wordform);
+                addUnsorted(this.root, { key, weight, content: wordform }, 0);
+            }
+            sortTrie(this.root);
+            return this;
+        }
+    }
+    // "Constructors"
+    function createRootNode() {
+        return {
+            type: 'leaf',
+            weight: 0,
+            entries: []
+        };
+    }
+    // Implement Trie creation.
+    /**
+     * Adds an entry to the trie.
+     *
+     * Note that the trie will likely be unsorted after the add occurs. Before
+     * performing a lookup on the trie, use call sortTrie() on the root note!
+     *
+     * @param node Which node should the entry be added to?
+     * @param entry the wordform/weight/key to add to the trie
+     * @param index the index in the key and also the trie depth. Should be set to
+     *              zero when adding onto the root node of the trie.
+     */
+    function addUnsorted(node, entry, index = 0) {
+        // Each node stores the MAXIMUM weight out of all of its decesdents, to
+        // enable a greedy search through the trie.
+        node.weight = Math.max(node.weight, entry.weight);
+        // When should a leaf become an interior node?
+        // When it already has a value, but the key of the current value is longer
+        // than the prefix.
+        if (node.type === 'leaf' && index < entry.key.length && node.entries.length >= 1) {
+            convertLeafToInternalNode(node, index);
+        }
+        if (node.type === 'leaf') {
+            // The key matches this leaf node, so add yet another entry.
+            addItemToLeaf(node, entry);
+        }
+        else {
+            // Push the node down to a lower node.
+            addItemToInternalNode(node, entry, index);
+        }
+        node.unsorted = true;
+    }
+    /**
+     * Adds an item to the internal node at a given depth.
+     * @param item
+     * @param index
+     */
+    function addItemToInternalNode(node, item, index) {
+        let char = item.key[index];
+        if (!node.children[char]) {
+            node.children[char] = createRootNode();
+            node.values.push(char);
+        }
+        addUnsorted(node.children[char], item, index + 1);
+    }
+    function addItemToLeaf(leaf, item) {
+        leaf.entries.push(item);
+    }
+    /**
+     * Mutates the given Leaf to turn it into an InternalNode.
+     *
+     * NOTE: the node passed in will be DESTRUCTIVELY CHANGED into a different
+     * type when passed into this function!
+     *
+     * @param depth depth of the trie at this level.
+     */
+    function convertLeafToInternalNode(leaf, depth) {
+        let entries = leaf.entries;
+        // Alias the current node, as the desired type.
+        let internal = leaf;
+        internal.type = 'internal';
+        delete leaf.entries;
+        internal.values = [];
+        internal.children = {};
+        // Convert the old values array into the format for interior nodes.
+        for (let item of entries) {
+            let char;
+            if (depth < item.key.length) {
+                char = item.key[depth];
+            }
+            else {
+                char = INTERNAL_VALUE;
+            }
+            if (!internal.children[char]) {
+                internal.children[char] = createRootNode();
+                internal.values.push(char);
+            }
+            addUnsorted(internal.children[char], item, depth + 1);
+        }
+        internal.unsorted = true;
+    }
+    /**
+     * Recursively sort the trie, in descending order of weight.
+     * @param node any node in the trie
+     */
+    function sortTrie(node) {
+        if (node.type === 'leaf') {
+            if (!node.unsorted) {
+                return;
+            }
+            node.entries.sort(function (a, b) { return b.weight - a.weight; });
+        }
+        else {
+            // We MUST recurse and sort children before returning.
+            for (let char of node.values) {
+                sortTrie(node.children[char]);
+            }
+            if (!node.unsorted) {
+                return;
+            }
+            node.values.sort((a, b) => {
+                return node.children[b].weight - node.children[a].weight;
+            });
+        }
+        delete node.unsorted;
+    }
+    /**
+     * O(n) recursive traversal to sum the total weight of all leaves in the
+     * trie, starting at the provided node.
+     *
+     * @param node The node to start summing weights.
+     */
+    function sumWeights(node) {
+        let val;
+        if (node.type === 'leaf') {
+            val = node.entries
+                .map(entry => entry.weight)
+                //.map(entry => isNaN(entry.weight) ? 1 : entry.weight)
+                .reduce((acc, count) => acc + count, 0);
+        }
+        else {
+            val = Object.keys(node.children)
+                .map((key) => sumWeights(node.children[key]))
+                .reduce((acc, count) => acc + count, 0);
+        }
+        if (isNaN(val)) {
+            console.error("Unexpected NaN has appeared!");
+        }
+        return val;
+    }
+})(Trie || (Trie = {}));
+/**
+ * Detects the encoding of a text file.
+ *
+ * Supported encodings are:
+ *
+ *  - UTF-8, with or without BOM
+ *  - UTF-16, little endian, with BOM
+ *
+ * UTF-16 in big endian is explicitly NOT supported! The reason is two-fold:
+ * 1) Node does not support it without resorting to an external library (or
+ * swapping every byte in the file!); and 2) I'm not sure anything actually
+ * outputs in this format anyway!
+ *
+ * @param filename filename of the file to detect encoding
+ */
+function detectEncoding(filename) {
+    let buffer = readFileSync(filename);
+    // Note: BOM is U+FEFF
+    // In little endian, this is 0xFF 0xFE
+    if (buffer[0] == 0xFF && buffer[1] == 0xFE) {
+        return 'utf16le';
+    }
+    else if (buffer[0] == 0xFE && buffer[1] == 0xFF) {
+        // Big Endian, is NOT supported because Node does not support it (???)
+        // See: https://stackoverflow.com/a/14551669/6626414
+        throw new Error('UTF-16BE is unsupported');
+    }
+    else {
+        // Assume its in UTF-8, with or without a BOM.
+        return 'utf8';
+    }
+}
+//# sourceMappingURL=build-trie.js.map

package/build/src/build-trie.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"build-trie.js","sourceRoot":"","sources":["../../src/build-trie.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EAAE,GAAG,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAEtE,wCAAwC;AACxC,MAAM,iBAAiB,GAAG,eAAe,CAAC;AAW1C;;;;;;;GAOG;AACH,MAAM,UAAU,uBAAuB,CAAC,SAAmB,EAAE,eAAwC;IACnG,IAAI,OAAO,eAAe,KAAK,UAAU,EAAE;QACzC,MAAM,IAAI,SAAS,CAAC,8CAA8C,CAAC,CAAA;KACpE;IACD,+DAA+D;IAC/D,IAAI,QAAQ,GAAa,EAAE,CAAC;IAC5B,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,yBAAyB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;IAE7E,IAAI,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,eAAuC,CAAC,CAAC;IAC7E,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,yBAAyB,CAAC,QAAkB,EAAE,QAAgB;IAC5E,cAAc,CAAC,QAAQ,EAAE,IAAI,oBAAoB,CAAC,QAAQ,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,yBAAyB,CAAC,QAAkB,EAAE,QAAgB;IAC5E,cAAc,CAAC,QAAQ,EAAE,IAAI,kBAAkB,CAAC,QAAQ,CAAC,CAAC,CAAC;AAC7D,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,SAAS,cAAc,CAAC,QAAkB,EAAE,MAAuB;IACjE,MAAM,GAAG,GAAG,IAAI,CAAC;IAEjB,IAAI,mBAAmB,GAAG,IAAI,GAAG,EAAU,CAAC;IAE5C,KAAK,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,KAAK,EAAE,EAAE;QACzC,qEAAqE;QACrE,kFAAkF;QAClF,8EAA8E;QAC9E,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAE1C,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,KAAK,EAAE,EAAE;YACvC,SAAS,CAAC,gCAAgC;SAC3C;QAED,mDAAmD;QACnD,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAE5C,uBAAuB;QACvB,IAAI,QAAQ,GAAG,QAAQ,CAAC;QAExB,QAAQ,GAAG,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACrC,IAAI,QAAQ,KAAK,QAAQ,EAAE;YACzB,sDAAsD;YACtD,GAAG,CACD,mBAAmB,CAAC,6BAA6B,EACjD,IAAI,QAAQ,2DAA2D,EACvE,EAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAC,CAChC,CAAA;SACF;QAED,QAAQ,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAA;QAE1B,SAAS,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACvD,IAAI,KAAK,GAAG,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAEpC,wEAAwE;QACxE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE;YACjC,uCAAuC;YACvC,+DAA+D;YAC/D,KAAK,GAAG,CAAC,CAAC;SACX;QAED,IAAI,mBAAmB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;YACrC,oDAAoD;YACpD,2DAA2D;YAC3D,GAAG,CACD,mBAAmB,CAAC,6BAA6B,EACjD,mBAAmB,QAAQ,sCAAsC,EACjE,EAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAC,CAChC,CAAA;SACF;QACD,mBAAmB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAElC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC;KACxF;AACH,CAAC;AASD,MAAM,kBAAkB;IACb,IAAI,GAAG,UAAU,CAAC;IACV,SAAS,CAAS;IAEnC,YAAY,QAAgB;QAC1B,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;IAC5B,CAAC;IAED,CAAC,KAAK;QACJ,KAAM,CAAC,CAAA,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC;IACjE,CAAC;CACF;AAED,MAAM,oBAAoB;IACf,IAAI,CAAS;IACtB,YAAY,QAAgB;QAC1B,IAAI,CAAC,IAAI,GAAG,QAAQ,CAAC;IACvB,CAAC;IAED,CAAC,KAAK;QACJ,IAAI,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAClE,KAAM,CAAC,CAAA,cAAc,CAAC,QAAQ,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC;IAC3D,CAAC;CACF;AAED;;GAEG;AACH,QAAQ,CAAC,CAAC,cAAc,CAAC,KAAe;IACpC,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,KAAK,IAAI,IAAI,IAAI,KAAK,EAAE;QACtB,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAChB,CAAC,EAAE,CAAC;KACL;AACL,CAAC;AAED,IAAU,IAAI,CAmSb;AAnSD,WAAU,MAAI;IAsFZ;;;;;;;;;OASG;IACH,MAAM,cAAc,GAAG,QAAQ,CAAC;IAEhC;;;;;;OAMG;IACH,SAAgB,SAAS,CAAC,QAAkB,EAAE,WAA4B;QACxE,IAAI,IAAI,GAAG,IAAI,IAAI,CAAC,WAAW,CAAC,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC;QAClE,OAAO;YACL,WAAW,EAAE,UAAU,CAAC,IAAI,CAAC;YAC7B,IAAI,EAAE,IAAI;SACX,CAAA;IACH,CAAC;IANe,gBAAS,YAMxB,CAAA;IAED;;OAEG;IACH,MAAM,IAAI;QACC,IAAI,GAAG,cAAc,EAAE,CAAC;QACjC,KAAK,CAAkB;QACvB,YAAY,YAA6B;YACvC,IAAI,CAAC,KAAK,GAAG,YAAY,CAAC;QAC5B,CAAC;QAED;;;WAGG;QACH,iBAAiB,CAAC,KAAe;YAC/B,KAAK,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;gBACpD,IAAI,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBAC/B,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC;aAC/D;YACD,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpB,OAAO,IAAI,CAAC;QACd,CAAC;KACF;IAED,iBAAiB;IACjB,SAAS,cAAc;QACrB,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,MAAM,EAAE,CAAC;YACT,OAAO,EAAE,EAAE;SACZ,CAAC;IACJ,CAAC;IAED,2BAA2B;IAE3B;;;;;;;;;;OAUG;IACH,SAAS,WAAW,CAAC,IAAU,EAAE,KAAY,EAAE,QAAgB,CAAC;QAC9D,uEAAuE;QACvE,2CAA2C;QAC3C,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAElD,8CAA8C;QAC9C,0EAA0E;QAC1E,mBAAmB;QACnB,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE;YAChF,yBAAyB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;SACxC;QAED,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE;YACxB,4DAA4D;YAC5D,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;SAC5B;aAAM;YACL,sCAAsC;YACtC,qBAAqB,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;SAC3C;QAED,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACvB,CAAC;IAED;;;;OAIG;IACH,SAAS,qBAAqB,CAAC,IAAkB,EAAE,IAAW,EAAE,KAAa;QAC3E,IAAI,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC3B,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;YACxB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,cAAc,EAAE,CAAC;YACvC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACxB;QACD,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IACpD,CAAC;IAED,SAAS,aAAa,CAAC,IAAU,EAAE,IAAW;QAC5C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED;;;;;;;OAOG;IACH,SAAS,yBAAyB,CAAC,IAAU,EAAE,KAAa;QAC1D,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAE3B,+CAA+C;QAC/C,IAAI,QAAQ,GAAc,IAAqB,CAAC;QAChD,QAAQ,CAAC,IAAI,GAAG,UAAU,CAAC;QAE3B,OAAO,IAAI,CAAC,OAAO,CAAC;QACpB,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC;QACrB,QAAQ,CAAC,QAAQ,GAAG,EAAE,CAAC;QAEvB,mEAAmE;QACnE,KAAK,IAAI,IAAI,IAAI,OAAO,EAAE;YACxB,IAAI,IAAY,CAAC;YACjB,IAAI,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE;gBAC3B,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;aACxB;iBAAM;gBACL,IAAI,GAAG,cAAc,CAAC;aACvB;YAED,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;gBAC5B,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,cAAc,EAAE,CAAC;gBAC3C,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aAC5B;YACD,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;SACvD;QAED,QAAQ,CAAC,QAAQ,GAAG,IAAI,CAAC;IAC3B,CAAC;IAED;;;OAGG;IACH,SAAS,QAAQ,CAAC,IAAU;QAC1B,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE;YACxB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE;gBAClB,OAAO;aACR;YAED,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;SACpE;aAAM;YACL,sDAAsD;YACtD,KAAK,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,EAAE;gBAC5B,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;aAC/B;YAED,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE;gBAClB,OAAO;aACR;YAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;gBACxB,OAAO,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YAC3D,CAAC,CAAC,CAAC;SACJ;QAED,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED;;;;;OAKG;IACH,SAAS,UAAU,CAAC,IAAU;QAC5B,IAAI,GAAW,CAAC;QAChB,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE;YACxB,GAAG,GAAG,IAAI,CAAC,OAAO;iBACf,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;gBAC3B,uDAAuD;iBACtD,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC;SAC3C;aAAM;YACL,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC;iBAC7B,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;iBAC5C,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC;SAC3C;QAED,IAAG,KAAK,CAAC,GAAG,CAAC,EAAE;YACb,OAAO,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;SAC/C;QACD,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC,EAnSS,IAAI,KAAJ,IAAI,QAmSb;AAED;;;;;;;;;;;;;;GAcG;AACH,SAAS,cAAc,CAAC,QAAgB;IACtC,IAAI,MAAM,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IACpC,sBAAsB;IACtB,sCAAsC;IACtC,IAAI,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE;QAC1C,OAAO,SAAS,CAAC;KAClB;SAAM,IAAI,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE;QACjD,sEAAsE;QACtE,oDAAoD;QACpD,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAA;KAC3C;SAAM;QACL,8CAA8C;QAC9C,OAAO,MAAM,CAAC;KACf;AACH,CAAC"}

package/build/src/join-word-breaker-decorator.d.ts

@@ -0,0 +1,10 @@
+/// <reference types="@keymanapp/models-types" />
+/**
+ * Returns a word breaker that joins spans of an existing word breaker.
+ * Spans are joined if they are connected by a delimiter.
+ *
+ * @param breaker The word breaker whose results will be decorated.
+ * @param joiners What delimiters should be used to join spans.
+ */
+export declare function decorateWithJoin(breaker: WordBreakingFunction, joiners: string[]): WordBreakingFunction;
+//# sourceMappingURL=join-word-breaker-decorator.d.ts.map

package/build/src/join-word-breaker-decorator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"join-word-breaker-decorator.d.ts","sourceRoot":"","sources":["../../src/join-word-breaker-decorator.ts"],"names":[],"mappings":";AAEA;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,oBAAoB,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,oBAAoB,CA4HvG"}

package/build/src/join-word-breaker-decorator.js

@@ -0,0 +1,121 @@
+/// <reference types="@keymanapp/models-types" />
+/**
+ * Returns a word breaker that joins spans of an existing word breaker.
+ * Spans are joined if they are connected by a delimiter.
+ *
+ * @param breaker The word breaker whose results will be decorated.
+ * @param joiners What delimiters should be used to join spans.
+ */
+export function decorateWithJoin(breaker, joiners) {
+    // Make a copy so that if the original array is accidentally mutated, it
+    // won't affect the joiner.
+    const delimiters = joiners.concat();
+    return function (input) {
+        let originalSpans = breaker(input);
+        // Implements a finite-state transducer (FST) where:
+        //  - Transductions are pushed onto a stack
+        //  - There are three states:
+        //    - empty stack (initial state)
+        //    - unjoined
+        //    - joined
+        // - all three states are accepting states
+        // - there is NO backtracking on the input
+        //   (hence the for-loop over the input tape)
+        // - each state is a JavaScript callback (function)
+        let state = emptyStack;
+        let stack = [];
+        for (let span of originalSpans) {
+            state = state(span);
+        }
+        return stack;
+        /******************* States *******************/
+        function emptyStack(span) {
+            stack.push(span);
+            if (isJoiner(span)) {
+                return joined;
+            }
+            else {
+                return unjoined;
+            }
+        }
+        function unjoined(span) {
+            // NB: stack has at least one span in it
+            if (isJoiner(span)) {
+                if (spansAreBackToBack(lastFrom(stack), span)) {
+                    concatLastSpanInStackWith(span);
+                }
+                else {
+                    // Spans are non-contiguous, so don't join them!
+                    stack.push(span);
+                }
+                return joined;
+            }
+            else {
+                // Span cannot be joined
+                stack.push(span);
+                return unjoined;
+            }
+        }
+        function joined(span) {
+            // NB: stack has at least one span in it
+            if (!spansAreBackToBack(lastFrom(stack), span)) {
+                // Spans are non-contiguous and cannot be joined:
+                stack.push(span);
+                return unjoined;
+            }
+            // Spans are contiguous
+            concatLastSpanInStackWith(span);
+            if (isJoiner(span)) {
+                return joined;
+            }
+            else {
+                return unjoined;
+            }
+        }
+        /****************** Helpers ******************/
+        function concatLastSpanInStackWith(span) {
+            let lastIndex = stack.length - 1;
+            let top = stack[lastIndex];
+            let joinedSpan = concatenateSpans(top, span);
+            stack[lastIndex] = joinedSpan;
+        }
+    };
+    function isJoiner(span) {
+        return includes(delimiters, span.text);
+    }
+    /**
+     * Returns true when the spans are contiguous.
+     * Order matters when calling this function!
+     */
+    function spansAreBackToBack(former, latter) {
+        return former.end === latter.start;
+    }
+    function concatenateSpans(former, latter) {
+        if (latter.start !== former.end) {
+            throw new Error(`Cannot concatenate non-contiguous spans: ${JSON.stringify(former)}/${JSON.stringify(latter)}`);
+        }
+        return {
+            start: former.start,
+            end: latter.end,
+            length: former.length + latter.length,
+            text: former.text + latter.text
+        };
+    }
+    /**
+     * When Array.prototype.include() doesn't exist:
+     */
+    function includes(haystack, needle) {
+        for (let item of haystack) {
+            if (item === needle)
+                return true;
+        }
+        return false;
+    }
+    /**
+     * Get the last element from the array.
+     */
+    function lastFrom(array) {
+        return array[array.length - 1];
+    }
+}
+//# sourceMappingURL=join-word-breaker-decorator.js.map

package/build/src/join-word-breaker-decorator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"join-word-breaker-decorator.js","sourceRoot":"","sources":["../../src/join-word-breaker-decorator.ts"],"names":[],"mappings":"AAAA,iDAAiD;AAEjD;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAA6B,EAAE,OAAiB;IAC/E,wEAAwE;IACxE,2BAA2B;IAC3B,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAEpC,OAAO,UAAU,KAAa;QAC5B,IAAI,aAAa,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;QAEnC,oDAAoD;QACpD,2CAA2C;QAC3C,6BAA6B;QAC7B,mCAAmC;QACnC,gBAAgB;QAChB,cAAc;QACd,0CAA0C;QAC1C,0CAA0C;QAC1C,6CAA6C;QAC7C,mDAAmD;QACnD,IAAI,KAAK,GAAG,UAAU,CAAC;QACvB,IAAI,KAAK,GAAW,EAAE,CAAC;QACvB,KAAK,IAAI,IAAI,IAAI,aAAa,EAAE;YAC9B,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;SACrB;QAED,OAAO,KAAK,CAAC;QAEb,gDAAgD;QAChD,SAAS,UAAU,CAAC,IAAU;YAC5B,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAEjB,IAAI,QAAQ,CAAC,IAAI,CAAC,EAAE;gBAClB,OAAO,MAAM,CAAC;aACf;iBAAM;gBACL,OAAO,QAAQ,CAAA;aAChB;QACH,CAAC;QAED,SAAS,QAAQ,CAAC,IAAU;YAC1B,wCAAwC;YACxC,IAAI,QAAQ,CAAC,IAAI,CAAC,EAAE;gBAClB,IAAI,kBAAkB,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,EAAE;oBAC7C,yBAAyB,CAAC,IAAI,CAAC,CAAC;iBACjC;qBAAM;oBACL,gDAAgD;oBAChD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBAClB;gBACD,OAAO,MAAM,CAAC;aAEf;iBAAM;gBACL,wBAAwB;gBACxB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACjB,OAAO,QAAQ,CAAC;aACjB;QACH,CAAC;QAED,SAAS,MAAM,CAAC,IAAU;YACxB,wCAAwC;YACxC,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,EAAE;gBAC9C,iDAAiD;gBACjD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACjB,OAAO,QAAQ,CAAC;aACjB;YAED,uBAAuB;YACvB,yBAAyB,CAAC,IAAI,CAAC,CAAC;YAChC,IAAI,QAAQ,CAAC,IAAI,CAAC,EAAE;gBAClB,OAAO,MAAM,CAAC;aACf;iBAAM;gBACL,OAAO,QAAQ,CAAC;aACjB;QACH,CAAC;QAED,+CAA+C;QAC/C,SAAS,yBAAyB,CAAC,IAAU;YAC3C,IAAI,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;YAEjC,IAAI,GAAG,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;YAC3B,IAAI,UAAU,GAAG,gBAAgB,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YAC7C,KAAK,CAAC,SAAS,CAAC,GAAG,UAAU,CAAC;QAChC,CAAC;IACH,CAAC,CAAA;IAED,SAAS,QAAQ,CAAC,IAAU;QAC1B,OAAO,QAAQ,CAAC,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACzC,CAAC;IAED;;;OAGG;IACH,SAAS,kBAAkB,CAAC,MAAY,EAAE,MAAY;QACpD,OAAO,MAAM,CAAC,GAAG,KAAK,MAAM,CAAC,KAAK,CAAC;IACrC,CAAC;IAED,SAAS,gBAAgB,CAAC,MAAY,EAAE,MAAY;QAClD,IAAI,MAAM,CAAC,KAAK,KAAK,MAAM,CAAC,GAAG,EAAE;YAC/B,MAAM,IAAI,KAAK,CAAC,4CAA4C,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;SACjH;QAED,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,MAAM,EAAE,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;YACrC,IAAI,EAAE,MAAM,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI;SAChC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,SAAS,QAAQ,CAAI,QAAa,EAAE,MAAS;QAC3C,KAAK,IAAI,IAAI,IAAI,QAAQ,EAAE;YACzB,IAAI,IAAI,KAAK,MAAM;gBACjB,OAAO,IAAI,CAAC;SACf;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACH,SAAS,QAAQ,CAAI,KAAU;QAC7B,OAAO,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACjC,CAAC;AACH,CAAC"}

package/build/src/lexical-model-compiler.d.ts

@@ -0,0 +1,19 @@
+/// <reference path="lexical-model.d.ts" />
+import { LexicalModelSource } from "./lexical-model.js";
+export default class LexicalModelCompiler {
+    /**
+     * Returns the generated code for the model that will ultimately be loaded by
+     * the LMLayer worker. This code contains all model parameters, and specifies
+     * word breakers and auxilary functions that may be required.
+     *
+     * @param model_id      The model ID. TODO: not sure if this is actually required!
+     * @param modelSource   A specification of the model to compile
+     * @param sourcePath    Where to find auxilary sources files
+     */
+    generateLexicalModelCode(model_id: string, modelSource: LexicalModelSource, sourcePath: string): string;
+    transpileSources(sources: Array<string>): Array<string>;
+    logError(s: string): void;
+}
+export declare class ModelSourceError extends Error {
+}
+//# sourceMappingURL=lexical-model-compiler.d.ts.map

package/build/src/lexical-model-compiler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"lexical-model-compiler.d.ts","sourceRoot":"","sources":["../../src/lexical-model-compiler.ts"],"names":[],"mappings":";AAaA,OAAO,EAAE,kBAAkB,EAA0C,MAAM,oBAAoB,CAAC;AAEhG,MAAM,CAAC,OAAO,OAAO,oBAAoB;IAEvC;;;;;;;;OAQG;IACH,wBAAwB,CAAC,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM;IAiE9F,gBAAgB,CAAC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IAUvD,QAAQ,CAAC,CAAC,EAAE,MAAM;CAGnB;AAED,qBAAa,gBAAiB,SAAQ,KAAK;CAC1C"}