@readium/shared 1.2.0

package/src/util/tokenizer/tokenize-english/README.md

@@ -0,0 +1,7 @@
+This is a copy of https://github.com/textlint-rule/rousseau/tree/master/packages/tokenize-english
+which is also available as the npm package `@textlint-rule/tokenize-english`.
+
+It has been included here because a slight modification needs to be made since
+the code does not run in JavaScript's strict mode due to a poor global variable declaration.
+It has also been modified to remove the lodash dependency so it doesn't have to be
+included directly in our dependencies.

package/src/util/tokenizer/tokenize-english/abbreviations.js

@@ -0,0 +1,52 @@
+export default [
+    "ie",
+    "eg",
+    "ext", // + number?
+    "Fig",
+    "fig",
+    "Figs",
+    "figs",
+    "et al",
+    "Co",
+    "Corp",
+    "Ave",
+    "Inc",
+    "Ex",
+    "Viz",
+    "vs",
+    "Vs",
+    "repr",
+    "Rep",
+    "Dem",
+    "trans",
+    "Vol",
+    "pp",
+    "rev",
+    "est",
+    "Ref",
+    "Refs",
+    "Eq",
+    "Eqs",
+    "Ch",
+    "Sec",
+    "Secs",
+    "mi",
+    "Dept",
+
+    "Univ",
+    "Nos",
+    "No",
+    "Mol",
+    "Cell",
+
+    "Miss", "Mrs", "Mr", "Ms",
+    "Prof", "Dr",
+    "Sgt", "Col", "Gen", "Rep", "Sen",'Gov', "Lt", "Maj", "Capt","St",
+
+    "Sr", "Jr", "jr", "Rev",
+    "PhD", "MD", "BA", "MA", "MM",
+    "BSc", "MSc",
+
+    "Jan","Feb","Mar","Apr","Jun","Jul","Aug","Sep","Sept","Oct","Nov","Dec",
+    "Sun","Mon","Tu","Tue","Tues","Wed","Th","Thu","Thur","Thurs","Fri","Sat"
+];

package/src/util/tokenizer/tokenize-english/index.d.ts

@@ -0,0 +1,3 @@
+export default function (tokenize: Tokenizer): {
+    sentences: () => (text: string) => Segment[];
+};

package/src/util/tokenizer/tokenize-english/index.js

@@ -0,0 +1,243 @@
+/*
+Sentence Boundary Detection (SBD)
+Split text into sentences with a vanilla rule based approach (i.e working ~95% of the time).
+
+Split a text based on period, question and exclamation marks.
+Skips (most) abbreviations (Mr., Mrs., PhD.)
+Skips numbers/currency
+Skips urls, websites, email addresses, phone nr.
+Counts ellipsis and ?! as single punctuation
+*/
+
+import utils from "./utils";
+import abbreviations from "./abbreviations";
+
+function isCapitalized(str) {
+    return /^[A-Z][a-z].*/.test(str) || isNumber(str);
+}
+
+// Start with opening quotes or capitalized letter
+function isSentenceStarter(str) {
+    return isCapitalized(str) || /``|"|'/.test(str.substring(0,2));
+}
+
+function isCommonAbbreviation(str) {
+    return ~abbreviations.indexOf(str.replace(/\W+/g, ''));
+}
+
+// This is going towards too much rule based
+function isTimeAbbreviation(word, next) {
+    if (word === "a.m." || word === "p.m.") {
+        var tmp = next.replace(/\W+/g, '').slice(-3).toLowerCase();
+
+        if (tmp === "day") {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+function isDottedAbbreviation(word) {
+    var matches = word.replace(/[\(\)\[\]\{\}]/g, '').match(/(.\.)*/);
+    return matches && matches[0].length > 0;
+}
+
+// TODO look for next words, if multiple capitalized -> not sentence ending
+function isCustomAbbreviation(str) {
+    if (str.length <= 3)
+        return true;
+
+    return isCapitalized(str);
+}
+
+// Uses current word count in sentence and next few words to check if it is
+// more likely an abbreviation + name or new sentence.
+function isNameAbbreviation(wordCount, words) {
+    if (words.length > 0) {
+        if (wordCount < 5 && words[0].length < 6 && isCapitalized(words[0])) {
+            return true;
+        }
+
+        var capitalized = words.filter(function(str) {
+            return /[A-Z]/.test(str.charAt(0));
+        });
+
+        return capitalized.length >= 3;
+    }
+
+    return false;
+}
+
+function isNumber(str, dotPos) {
+    if (dotPos) {
+        str = str.slice(dotPos-1, dotPos+2);
+    }
+
+    return !isNaN(Number(str));
+}
+
+// Phone number matching
+// http://stackoverflow.com/a/123666/951517
+function isPhoneNr(str) {
+    return str.match(/^(?:(?:\+?1\s*(?:[.-]\s*)?)?(?:\(\s*([2-9]1[02-9]|[2-9][02-8]1|[2-9][02-8][02-9])\s*\)|([2-9]1[02-9]|[2-9][02-8]1|[2-9][02-8][02-9]))\s*(?:[.-]\s*)?)?([2-9]1[02-9]|[2-9][02-9]1|[2-9][02-9]{2})\s*(?:[.-]\s*)?([0-9]{4})(?:\s*(?:#|x\.?|ext\.?|extension)\s*(\d+))?$/);
+}
+
+// Match urls / emails
+// http://stackoverflow.com/a/3809435/951517
+function isURL(str) {
+    return str.match(/[-a-zA-Z0-9@:%._\+~#=]{2,256}\.[a-z]{2,6}\b([-a-zA-Z0-9@:%_\+.~#?&//=]*)/);
+}
+
+// Starting a new sentence if beginning with capital letter
+// Exception: The word is enclosed in brackets
+function isConcatenated(word) {
+    var i = 0;
+
+    if ((i = word.indexOf(".")) > -1 ||
+        (i = word.indexOf("!")) > -1 ||
+        (i = word.indexOf("?")) > -1)
+    {
+        var c = word.charAt(i + 1);
+
+        // Check if the next word starts with a letter
+        if (c.match(/[a-zA-Z].*/)) {
+            return [word.slice(0, i), word.charAt(i), word.slice(i+1)];
+        }
+    }
+
+    return false;
+}
+
+function isBoundaryChar(word) {
+    return word === "." ||
+           word === "!" ||
+           word === "?";
+}
+
+// http://tech.grammarly.com/blog/posts/How-to-Split-Sentences.html
+function tokenizeSentences(tokenize, opts) {
+    opts = Object.assign(opts || {},  {
+        newlineBoundary: opts && opts.newlineBoundary || true
+    })
+
+    var splitRe = /\s/;
+
+    return tokenize.serie(
+        // Split into words
+        tokenize.re(splitRe, { split: true }),
+
+        // Merge words as sentences
+        tokenize.splitAndMerge(function(word, token, prev, next, i, tokens) {
+            var tmp;
+            var endOfSentence = [word, null];
+            var sentenceNotOver = word;
+
+            // Find the next word
+            var nextWord = tokens.slice(i+1).find(function(tok) {
+                return !splitRe.test(tok.value)
+            });
+
+            // Newline boundaries
+            if (word === '\n' && opts.newlineBoundary) return endOfSentence;
+
+            if (isBoundaryChar(word) ||
+                utils.endsWithChar(word, "?!"))
+            {
+                return endOfSentence;
+            }
+
+            // A dot might indicate the end sentences
+            // Exception: The next sentence starts with a word (non abbreviation)
+            //            that has a capital letter.
+            if (utils.endsWithChar(word, '.')) {
+                // Check if there is a next word
+                if (nextWord) {
+                    // This should be improved with machine learning
+
+                    // Single character abbr.
+                    if (word.length === 2 && isNaN(word.charAt(0)) && word.match(/[a-zA-Z]/)) {
+                        return sentenceNotOver;
+                    }
+
+                    // Common abbr. that often do not end sentences
+                    if (isCommonAbbreviation(word)) {
+                        return sentenceNotOver;
+                    }
+
+                    // Next word starts with capital word, but current sentence is
+                    // quite short
+                    if (isSentenceStarter(nextWord.value)) {
+                        if (isTimeAbbreviation(word, nextWord.value)) {
+                            return sentenceNotOver;
+                        }
+
+                        // Dealing with names at the start of sentences
+                        /*if (isNameAbbreviation(wordCount, words.slice(i, 6))) {
+                            return word;
+                        }*/
+
+                        if (isNumber(nextWord.value) && isCustomAbbreviation(word)) {
+                            return sentenceNotOver;
+                        }
+                    }
+                    else {
+                        // Skip ellipsis
+                        if (utils.endsWith(word, "..")) {
+                            return sentenceNotOver;
+                        }
+
+                        //// Skip abbreviations
+                        // Short words + dot or a dot after each letter
+                        if (isDottedAbbreviation(word) || isCustomAbbreviation(word)) {
+                            return sentenceNotOver;
+                        }
+                    }
+                }
+
+                return endOfSentence;
+            }
+
+            // Check if the word has a dot in it
+            let index;
+            if ((index = word.indexOf(".")) > -1) {
+                if (isNumber(word, index)) {
+                    return sentenceNotOver;
+                }
+
+                // Custom dotted abbreviations (like K.L.M or I.C.T)
+                if (isDottedAbbreviation(word)) {
+                    return sentenceNotOver;
+                }
+
+                // Skip urls / emails and the like
+                if (isURL(word) || isPhoneNr(word)) {
+                    return sentenceNotOver;
+                }
+            }
+
+            if (tmp = isConcatenated(word)) {
+                return [
+                    tmp[0]+tmp[1], null, tmp[2]
+                ];
+            }
+
+            return sentenceNotOver;
+        }),
+
+        // Filter empty sentences
+        tokenize.filter(function(sentence) {
+            if (sentence.trim() == '') return false;
+            return true;
+        })
+    );
+};
+
+// From lodash in original
+const partial = (fn, ...partialArgs) => (...args) => fn(...partialArgs, ...args);
+
+export default function(tokenize) {
+    return {
+        sentences: partial(tokenizeSentences, tokenize)
+    };
+};

package/src/util/tokenizer/tokenize-english/utils.js

@@ -0,0 +1,16 @@
+function endsWithChar(word, c) {
+    if (c.length > 1) {
+        return c.indexOf(word.slice(-1)) > -1;
+    }
+
+    return word.slice(-1) === c;
+}
+
+function endsWith(word, end) {
+    return word.slice(word.length - end.length) === end;
+}
+
+export default {
+    endsWith: endsWith,
+    endsWithChar: endsWithChar
+};

package/src/util/tokenizer/tokenize-text/README.MD

@@ -0,0 +1,6 @@
+This is a copy of https://github.com/textlint-rule/rousseau/tree/master/packages/tokenize-text
+which is also available as the npm package `@textlint-rule/tokenize-text`.
+
+It has been modified to remove the lodash dependency so it doesn't have to be
+included directly in our dependencies, since it becomes incredibly large with
+lodash, especially since the entire lodash lib was imported.

package/src/util/tokenizer/tokenize-text/index.d.ts

@@ -0,0 +1,52 @@
+// Only covers what's necessary to use these packages from TS
+
+export declare interface TextlintSegment {
+    value: string;
+    index: number;
+    offset: number;
+}
+
+declare class Tokenizer {
+    constructor(opts?: {
+        cacheGet?: (key: any) => any;
+        cacheSet?: (key: any, value: any) => void;
+    });
+
+    split(
+        fn: Function,
+        opts?: {
+            preserveProperties?: boolean;
+            cache?: Function;
+        }
+    ): (text: string | Object[], tok?: any) => any[];
+
+    debug(prefix?: string): (text: string, tok: any) => boolean;
+
+    re(re: RegExp, opts?: { split?: boolean }): Function;
+
+    splitAndMerge(
+        fn: Function,
+        opts?: {
+            mergeWith?: string;
+        }
+    ): (tokens: any[]) => any[];
+
+    filter(fn: Function): Function;
+
+    extend(fn: Function | Object): Function;
+
+    ifthen(condition: Function, then: Function): Function;
+
+    test(re: RegExp): Function;
+
+    flow(...fns: Function[]): Function;
+
+    serie(...fns: Function[]): Function;
+
+    merge: Function;
+    sections: Function;
+    words: Function;
+    characters: Function;
+}
+
+export default Tokenizer;

package/src/util/tokenizer/tokenize-text/index.js

@@ -0,0 +1,256 @@
+import tokenUtils from './tokens';
+
+var WORD_BOUNDARY_CHARS = '\t\r\n\u00A0 !\"#$%&()*+,\-.\\/:;<=>?@\[\\\]^_`{|}~';
+var WORD_BOUNDARY_REGEX = new RegExp('[' + WORD_BOUNDARY_CHARS + ']');
+var SPLIT_REGEX = new RegExp(
+'([^' + WORD_BOUNDARY_CHARS + ']+)');
+
+function Tokenizer(opts) {
+    if (!(this instanceof Tokenizer)) return new Tokenizer(opts);
+
+    this.opts = Object.assign({
+        cacheGet: function(key) { return null; },
+        cacheSet: function(key, value) { }
+    }, opts);
+}
+
+Tokenizer.prototype.split = function tokenizeSplit(fn, opts = {}) {
+    var that = this;
+    opts = Object.assign({ preserveProperties: true, cache: () => null }, opts);
+
+    return function(text, tok) {
+        if (arguments.length === 6) return fn.apply(null, arguments);
+
+        var prev;
+        var cacheId, cacheValue;
+
+        if(text === undefined) return [];
+
+        if (typeof text === "string") {
+            text = [{
+                value: text,
+                index: 0,
+                offset: text.length
+            }];
+        } else if (!Array.isArray(text)) {
+            text = [text];
+        }
+
+        cacheId = tokenUtils.tokensId(text, opts.cache());
+        if (cacheId) {
+            cacheValue = that.opts.cacheGet(cacheId);
+            if (cacheValue) {
+                return cacheValue;
+            }
+        }
+
+        var result = text.map(function(token, i) {
+            var next = text[i + 1];
+            var tokens = fn(
+                token.value,
+                Object.assign({}, token),
+                prev ? Object.assign({}, prev) : null,
+                next ? Object.assign({}, next) : null,
+                i,
+                text
+            ) || [];
+
+            tokens = tokenUtils.normalize(token, tokens);
+
+            if (opts.preserveProperties) {
+                var props = tokenUtils.properties(token);
+                tokens = tokens.map(function(_tok) {
+                    return Object.assign({}, _tok, props);
+                });
+            }
+
+            prev = token;
+
+            return tokens;
+        }).filter(Boolean).flat();
+
+        if (cacheId) {
+            that.opts.cacheSet(cacheId, result);
+        }
+
+        return result;
+    };
+};
+
+// Tokenize a text using a RegExp
+Tokenizer.prototype.re = function tokenizeRe(re, opts = {}) {
+    opts = Object.assign({ split: false }, opts);
+
+    return this.split(function(text, tok) {
+        var originalText = text;
+        var tokens = [];
+        var match;
+        var start = 0;
+        var lastIndex = 0;
+
+        while (match = re.exec(text)) {
+            // Index in the current text section
+            var index = match.index;
+
+            // Index in the original text
+            var absoluteIndex = start + index;
+
+            var value = match[0] || "";
+            var offset = value.length;
+
+            // If splitting, push missed text
+            if (opts.split && start < absoluteIndex) {
+                var beforeText = originalText.slice(start, absoluteIndex);
+                tokens.push({
+                    value: beforeText,
+                    index: start,
+                    offset: beforeText.length
+                });
+            }
+
+            tokens.push({
+                value: value,
+                index: absoluteIndex,
+                offset: offset,
+                match: match
+            });
+
+            text = text.slice(index + offset);
+            start = absoluteIndex + offset;
+        }
+
+        // If splitting, push left text
+        if (opts.split && text) {
+            tokens.push({
+                value: text,
+                index: start,
+                offset: text.length
+            });
+        }
+
+        return tokens;
+    }, {
+        cache: function() {
+            return re.toString();
+        }
+    });
+};
+
+// Split and merge tokens
+Tokenizer.prototype.splitAndMerge = function tokenizeSplitAndMerge(fn, opts = {}) {
+    var that = this;
+    opts = Object.assign({ mergeWith: '' }, opts);
+
+    return function(tokens) {
+        var result = [];
+        var accu = [];
+
+        function pushAccu() {
+            if (accu.length == 0) return;
+
+            // Merge accumulator into one token
+            var tok = tokenUtils.merge(accu, opts.mergeWith);
+
+            result.push(tok);
+            accu = [];
+        }
+
+        that.split(function(word, token) {
+            var toks = fn.apply(null, arguments);
+
+            // Normalize tokens
+            toks = tokenUtils.normalize(token, toks);
+
+            // Accumulate tokens and push to final results
+            toks.forEach(function(tok) {
+                if (tok === null) {
+                    pushAccu();
+                } else {
+                    accu.push(tok);
+                }
+            });
+        })(tokens);
+
+        // Push tokens left in accumulator
+        pushAccu();
+
+        return result;
+    };
+};
+
+// Filter when tokenising
+Tokenizer.prototype.filter = function tokenizeFilter(fn) {
+    return this.split(function(text, tok) {
+        if (fn.apply(null, arguments)) {
+            return {
+                value: tok.value,
+                index: 0,
+                offset: tok.offset
+            };
+        }
+        return undefined;
+    });
+};
+
+// Extend a token properties
+Tokenizer.prototype.extend = function tokenizeExtend(fn) {
+    return this.split(function(text, tok) {
+        var o = typeof fn === 'function' ? fn.apply(null, arguments) : fn;
+        return Object.assign({
+            value: tok.value,
+            index: 0,
+            offset: tok.offset
+        }, o);
+    });
+};
+
+// Condition for tokenizing flow
+Tokenizer.prototype.ifthen = function(condition, then) {
+    return this.split(function(text, tok) {
+        if (condition.apply(null, arguments)) {
+            return then.apply(null, arguments);
+        }
+        const {index, ...rest} = tok; // Omit 'index'
+        return rest;
+    });
+};
+
+// Filter by testing a regex
+Tokenizer.prototype.test = function tokenizeTest(re) {
+    return this.filter(function(text, tok) {
+        return re.test(text);
+    }, {
+        cache: re.toString()
+    });
+};
+
+// Process token by all arguments
+Tokenizer.prototype.flow = function tokenizeFlow(...args) {
+    const fn = args.reduce((acc, cur) => (...args) => cur(acc(...args)));
+    return this.split(fn);
+};
+
+// Group and process a token as a group
+Tokenizer.prototype.serie = function tokenizeFlow(...args) {
+    return args.reduce((acc, cur) => (...args) => cur(acc(...args)));
+};
+
+// Merge all tokens into one
+Tokenizer.prototype.merge = function() {
+    return this.splitAndMerge(token => [token]);
+};
+
+Tokenizer.prototype.sections = function() {
+    return this.re(/([^\n\.,;!?]+)/i, { split: false });
+};
+
+Tokenizer.prototype.words = function() {
+    return this.re(SPLIT_REGEX);
+};
+
+Tokenizer.prototype.characters = function() {
+    return this.re(/[^\s]/);
+};
+
+export default Tokenizer;
+