de-compromise 0.0.4 → 0.0.5

package/builds/de-compromise.cjs

@@ -53,6 +53,8 @@
   };
   var compute$9 = fns$4;
 
+  // wrappers for loops in javascript arrays
+
   const forEach = function (cb) {
     let ptrs = this.fullPointer;
     ptrs.forEach((ptr, i) => {
@@ -64,11 +66,14 @@
 
   const map = function (cb, empty) {
     let ptrs = this.fullPointer;
-    // let cache = this._cache || []
     let res = ptrs.map((ptr, i) => {
       let view = this.update([ptr]);
-      // view._cache = cache[i]
-      return cb(view, i)
+      let out = cb(view, i);
+      // if we returned nothing, return a view
+      if (out === undefined) {
+        return this.none()
+      }
+      return out
     });
     if (res.length === 0) {
       return empty || this.update([])
@@ -95,23 +100,18 @@
 
   const filter = function (cb) {
     let ptrs = this.fullPointer;
-    // let cache = this._cache || []
     ptrs = ptrs.filter((ptr, i) => {
       let view = this.update([ptr]);
-      // view._cache = cache[i]
       return cb(view, i)
     });
-    let res = this.update(ptrs); //TODO: keep caches automatically
-    // res._cache = ptrs.map(ptr => cache[ptr[0]])
+    let res = this.update(ptrs);
     return res
   };
 
   const find = function (cb) {
     let ptrs = this.fullPointer;
-    // let cache = this._cache || []
     let found = ptrs.find((ptr, i) => {
       let view = this.update([ptr]);
-      // view._cache = cache[i]
       return cb(view, i)
     });
     return this.update([found])
@@ -119,10 +119,8 @@
 
   const some = function (cb) {
     let ptrs = this.fullPointer;
-    // let cache = this._cache || []
     return ptrs.some((ptr, i) => {
       let view = this.update([ptr]);
-      // view._cache = cache[i]
       return cb(view, i)
     })
   };
@@ -176,14 +174,11 @@
     /** */
     eq: function (n) {
       let ptr = this.pointer;
-      let cache = this._cache || [];
       if (!ptr) {
         ptr = this.docs.map((_doc, i) => [i]);
       }
       if (ptr[n]) {
-        let view = this.update([ptr[n]]);
-        view._cache = cache[n];
-        return view
+        return this.update([ptr[n]])
       }
       return this.none()
     },
@@ -342,31 +337,27 @@
     update(pointer) {
       let m = new View(this.document, pointer);
       // send the cache down, too?
-      if (m._cache && pointer && pointer.length > 1) {
-        // only if it's full
+      if (this._cache && pointer && pointer.length > 0) {
+        // only keep cache if it's a full-sentence
         let cache = [];
-        pointer.forEach(ptr => {
+        pointer.forEach((ptr, i) => {
+          let [n, start, end] = ptr;
           if (ptr.length === 1) {
-            cache.push(m._cache[ptr[0]]);
+            cache[i] = this._cache[n];
+          } else if (start === 0 && this.document[n].length === end) {
+            cache[i] = this._cache[n];
           }
-          // let [n, start, end] = ptr
-          // if (start === 0 && this.document[n][end - 1] && !this.document[n][end]) {
-          //   console.log('=-=-=-= here -=-=-=-')
-          // }
         });
-        m._cache = cache;
+        if (cache.length > 0) {
+          m._cache = cache;
+        }
       }
       m.world = this.world;
       return m
     }
     // create a new View, from this one
     toView(pointer) {
-      if (pointer === undefined) {
-        pointer = this.pointer;
-      }
-      let m = new View(this.document, pointer);
-      // m._cache = this._cache // share this full thing
-      return m
+      return new View(this.document, pointer || this.pointer)
     }
     fromText(input) {
       const { methods } = this;
@@ -400,7 +391,7 @@
   Object.assign(View.prototype, api$d);
   var View$1 = View;
 
-  var version$1 = '14.3.1';
+  var version$1 = '14.4.0';
 
   const isObject$6 = function (item) {
     return item && typeof item === 'object' && !Array.isArray(item)
@@ -583,7 +574,6 @@
     return doc
   };
 
-
   /** extend compromise functionality */
   nlp.plugin = function (plugin) {
     extend$1(plugin, this._world, View$1, this);
@@ -859,7 +849,7 @@
       after 46-thousand sentences
 
   */
-  let start$1 = 0;
+  let index$2 = 0;
 
   const pad3 = (str) => {
     str = str.length < 3 ? '0' + str : str;
@@ -868,19 +858,17 @@
 
   const toId = function (term) {
     let [n, i] = term.index || [0, 0];
-    start$1 += 1;
-    var now = start$1;
-    now = parseInt(now, 10);
+    index$2 += 1;
 
-    //don't overflow time
-    now = now > 46655 ? 46655 : now;
+    //don't overflow index
+    index$2 = index$2 > 46655 ? 0 : index$2;
     //don't overflow sentences
-    n = n > 46655 ? 46655 : n;
+    n = n > 46655 ? 0 : n;
     // //don't overflow terms
-    i = i > 1294 ? 1294 : i;
+    i = i > 1294 ? 0 : i;
 
     // 3 digits for time
-    let id = pad3(now.toString(36));
+    let id = pad3(index$2.toString(36));
     // 3 digit  for sentence index (46k)
     id += pad3(n.toString(36));
 
@@ -939,6 +927,7 @@
 
   const insert = function (input, view, prepend) {
     const { document, world } = view;
+    view.uncache();
     // insert words at end of each doc
     let ptrs = view.fullPointer;
     let selfPtrs = view.fullPointer;
@@ -1027,6 +1016,7 @@
   fns$2.replaceWith = function (input, keep = {}) {
     let ptrs = this.fullPointer;
     let main = this;
+    this.uncache();
     if (typeof input === 'function') {
       return replaceByFn(main, input)
     }
@@ -1172,6 +1162,7 @@
     /** */
     remove: function (reg) {
       const { indexN } = this.methods.one.pointer;
+      this.uncache();
       // two modes:
       //  - a. remove self, from full parent
       let self = this.all();
@@ -1203,7 +1194,6 @@
         this.ptrs = [];
         return self.none()
       }
-      // self._cache = null
       let res = self.toView(ptrs); //return new document
       return res
     },
@@ -1318,7 +1308,7 @@
   methods$h.deHyphenate = methods$h.dehyphenate;
   methods$h.toQuotation = methods$h.toQuotations;
 
-  var whitespace$1 = methods$h;
+  var whitespace = methods$h;
 
   /** alphabetical order */
   const alpha = (a, b) => {
@@ -1410,6 +1400,7 @@
   /** re-arrange the order of the matches (in place) */
   const sort = function (input) {
     let { docs, pointer } = this;
+    this.uncache();
     if (typeof input === 'function') {
       return customSort(this, input)
     }
@@ -1449,6 +1440,9 @@
     let ptrs = this.pointer || this.docs.map((_d, n) => [n]);
     ptrs = [].concat(ptrs);
     ptrs = ptrs.reverse();
+    if (this._cache) {
+      this._cache = this._cache.reverse();
+    }
     return this.update(ptrs)
   };
 
@@ -1541,7 +1535,7 @@
   };
   var harden$1 = { harden, soften };
 
-  const methods$f = Object.assign({}, caseFns, insert$1, replace, remove, whitespace$1, sort$1, concat, harden$1);
+  const methods$f = Object.assign({}, caseFns, insert$1, replace, remove, whitespace, sort$1, concat, harden$1);
 
   const addAPI$2 = function (View) {
     Object.assign(View.prototype, methods$f);
@@ -1793,6 +1787,34 @@
   };
   var numberRange$1 = numberRange;
 
+  const numUnit = /^([0-9.,+-]+)([a-z°²³µ/]+)$/i;
+
+  const notUnit = new Set([
+    'st',
+    'nd',
+    'rd',
+    'th',
+    'am',
+    'pm',
+    'max'
+  ]);
+
+  const numberUnit = function (terms, i) {
+    let term = terms[i];
+    let parts = term.text.match(numUnit);
+    if (parts !== null) {
+      // is it a recognized unit, like 'km'?
+      let unit = parts[2].toLowerCase().trim();
+      // don't split '3rd'
+      if (notUnit.has(unit)) {
+        return null
+      }
+      return [parts[1], unit] //split it
+    }
+    return null
+  };
+  var numberUnit$1 = numberUnit;
+
   const byApostrophe = /'/;
   const numDash = /^[0-9][^-–—]*[-–—].*?[0-9]/;
 
@@ -1809,10 +1831,6 @@
       end += 1;
     }
     tmp.ptrs = [[0, start, end]];
-    tmp.compute('lexicon');
-    if (tmp.world.compute.preTagger) {
-      tmp.compute('preTagger');
-    }
   };
 
   const byEnd = {
@@ -1854,7 +1872,7 @@
 
   const toDocs = function (words, view) {
     let doc = view.fromText(words.join(' '));
-    doc.compute('id');
+    doc.compute(['id', 'alias']);
     return doc.docs[0]
   };
 
@@ -1863,6 +1881,7 @@
     let { world, document } = view;
     const { model, methods } = world;
     let list = model.one.contractions || [];
+    new Set(model.one.units || []);
     // each sentence
     document.forEach((terms, n) => {
       // loop through terms backwards
@@ -1898,10 +1917,18 @@
             methods.one.setTag(words, 'NumberRange', world);//add custom tag
             // is it a time-range, like '5-9pm'
             if (words[2] && words[2].tags.has('Time')) {
-              methods.one.setTag([words[0]], 'Time', world);
+              methods.one.setTag([words[0]], 'Time', world, null, 'time-range');
             }
             reTag(document[n], view, i, words.length);
           }
+          continue
+        }
+        // split-apart '4km'
+        words = numberUnit$1(terms, i);
+        if (words) {
+          words = toDocs(words, view);
+          splice(document, [n, i], words);
+          methods.one.setTag([words[1]], 'Unit', world, null, 'contraction-unit');
         }
       }
     });
@@ -2002,7 +2029,7 @@
 
   // tag any words in our lexicon - even if it hasn't been filled-up yet
   // rest of pre-tagger is in ./two/preTagger
-  const firstPass$1 = function (view) {
+  const lexicon$5 = function (view) {
     const world = view.world;
     view.docs.forEach(terms => {
       for (let i = 0; i < terms.length; i += 1) {
@@ -2017,7 +2044,7 @@
   };
 
   var compute$4 = {
-    lexicon: firstPass$1
+    lexicon: lexicon$5
   };
 
   // derive clever things from our lexicon key-value pairs
@@ -2026,12 +2053,12 @@
     let lex = {};
     // console.log('start:', Object.keys(lex).length)
     let _multi = {};
-
     // go through each word in this key-value obj:
     Object.keys(words).forEach(word => {
       let tag = words[word];
       // normalize lexicon a little bit
       word = word.toLowerCase().trim();
+      word = word.replace(/'s\b/, '');
       // cache multi-word terms
       let split = word.split(/ /);
       if (split.length > 1) {
@@ -2290,11 +2317,13 @@
   /** pre-compile a list of matches to lookup */
   const lib$4 = {
     /** turn an array or object into a compressed trie*/
-    compile: function (input) {
+    buildTrie: function (input) {
       const trie = build(input, this.world());
       return compress$1(trie)
     }
   };
+  // add alias
+  lib$4.compile = lib$4.buildTrie;
 
   var lookup = {
     api: api$a,
@@ -2348,6 +2377,19 @@
   //   return false
   // }
 
+  const parseRegs = function (regs, opts, world) {
+    const one = world.methods.one;
+    if (typeof regs === 'number') {
+      regs = String(regs);
+    }
+    // support param as string
+    if (typeof regs === 'string') {
+      regs = one.killUnicode(regs, world);
+      regs = one.parseMatch(regs, opts, world);
+    }
+    return regs
+  };
+
   const match$2 = function (regs, group, opts) {
     const one = this.methods.one;
     // support param as view object
@@ -2358,25 +2400,12 @@
     if (isNet(regs)) {
       return this.sweep(regs, { tagger: false }).view.settle()
     }
-    // support param as string
-    if (typeof regs === 'string') {
-      regs = one.killUnicode(regs, this.world);
-      regs = one.parseMatch(regs, opts, this.world);
-    }
+    regs = parseRegs(regs, opts, this.world);
     let todo = { regs, group };
     let res = one.match(this.docs, todo, this._cache);
     let { ptrs, byGroup } = fixPointers(res, this.fullPointer);
     let view = this.toView(ptrs);
     view._groups = byGroup;
-    // try to keep some of the cache
-    // if (this._cache) {
-    //   view._cache = view.ptrs.map(ptr => {
-    //     if (isFull(ptr, this.document)) {
-    //       return this._cache[ptr[0]]
-    //     }
-    //     return null
-    //   })
-    // }
     return view
   };
 
@@ -2390,10 +2419,7 @@
     if (isNet(regs)) {
       return this.sweep(regs, { tagger: false, matchOne: true }).view
     }
-    if (typeof regs === 'string') {
-      regs = one.killUnicode(regs, this.world);
-      regs = one.parseMatch(regs, opts, this.world);
-    }
+    regs = parseRegs(regs, opts, this.world);
     let todo = { regs, group, justOne: true };
     let res = one.match(this.docs, todo, this._cache);
     let { ptrs, byGroup } = fixPointers(res, this.fullPointer);
@@ -2413,10 +2439,7 @@
     if (isNet(regs)) {
       return this.sweep(regs, { tagger: false }).view.found
     }
-    if (typeof regs === 'string') {
-      regs = one.killUnicode(regs, this.world);
-      regs = one.parseMatch(regs, opts, this.world);
-    }
+    regs = parseRegs(regs, opts, this.world);
     let todo = { regs, group, justOne: true };
     let ptrs = one.match(this.docs, todo, this._cache).ptrs;
     return ptrs.length > 0
@@ -2434,10 +2457,7 @@
       let m = this.sweep(regs, { tagger: false }).view.settle();
       return this.if(m)//recurse with result
     }
-    if (typeof regs === 'string') {
-      regs = one.killUnicode(regs, this.world);
-      regs = one.parseMatch(regs, opts, this.world);
-    }
+    regs = parseRegs(regs, opts, this.world);
     let todo = { regs, group, justOne: true };
     let ptrs = this.fullPointer;
     let cache = this._cache || [];
@@ -2467,10 +2487,7 @@
       return this.ifNo(m)
     }
     // otherwise parse the match string
-    if (typeof regs === 'string') {
-      regs = one.killUnicode(regs, this.world);
-      regs = one.parseMatch(regs, opts, this.world);
-    }
+    regs = parseRegs(regs, opts, this.world);
     let cache = this._cache || [];
     let view = this.filter((m, i) => {
       let todo = { regs, group, justOne: true };
@@ -2849,14 +2866,24 @@
         return obj
       }
 
-      //machine/sense overloaded
+      //root/sense overloaded
       if (start(w) === '{' && end(w) === '}') {
         w = stripBoth(w);
+        obj.id = w;
+        obj.root = w;
         if (/\//.test(w)) {
-          obj.sense = w;
-          obj.greedy = true;
-        } else {
-          obj.machine = w;
+          let split = obj.root.split(/\//);
+          obj.root = split[0];
+          obj.pos = split[1];
+          if (obj.pos === 'adj') {
+            obj.pos = 'Adjective';
+          }
+          // titlecase
+          obj.pos = obj.pos.charAt(0).toUpperCase() + obj.pos.substr(1).toLowerCase();
+          // add sense-number too
+          if (split[2] !== undefined) {
+            obj.num = split[2];
+          }
         }
         return obj
       }
@@ -2965,6 +2992,60 @@
   };
   var splitHyphens$2 = splitHyphens$1;
 
+  const addVerbs = function (token, world) {
+    let { verbConjugate } = world.methods.two.transform;
+    let res = verbConjugate(token.root, world.model);
+    delete res.FutureTense;
+    return Object.values(res).filter(str => str)
+  };
+
+  const addNoun = function (token, world) {
+    let { nounToPlural } = world.methods.two.transform;
+    let res = [token.root];
+    res.push(nounToPlural(token.root, world.model));
+    return res
+  };
+
+  const addAdjective = function (token, world) {
+    let { adjToSuperlative, adjToComparative, adjToAdverb } = world.methods.two.transform;
+    let res = [token.root];
+    res.push(adjToSuperlative(token.root, world.model));
+    res.push(adjToComparative(token.root, world.model));
+    res.push(adjToAdverb(token.root, world.model));
+    return res
+  };
+
+  // turn '{walk}' into 'walking', 'walked', etc
+  const inflectRoot = function (regs, world) {
+    // do we have compromise/two?
+    if (world.methods.two && world.methods.two.transform) {
+      regs = regs.map(token => {
+        // a reg to convert '{foo}'
+        if (token.root) {
+          let choices = [];
+          if (!token.pos || token.pos === 'Verb') {
+            choices = choices.concat(addVerbs(token, world));
+          }
+          if (!token.pos || token.pos === 'Noun') {
+            choices = choices.concat(addNoun(token, world));
+          }
+          // don't run these by default
+          if (!token.pos || token.pos === 'Adjective') {
+            choices = choices.concat(addAdjective(token, world));
+          }
+          choices = choices.filter(str => str);
+          if (choices.length > 0) {
+            token.operator = 'or';
+            token.fastOr = new Set(choices);
+          }
+        }
+        return token
+      });
+    }
+    return regs
+  };
+  var inflectRoot$1 = inflectRoot;
+
   // name any [unnamed] capture-groups with a number
   const nameGroups = function (regs) {
     let index = 0;
@@ -3073,6 +3154,8 @@
     tokens = tokens.map(str => parseToken$1(str, opts));
     // '~re-do~'
     tokens = splitHyphens$2(tokens, world);
+    // '{walk}'
+    tokens = inflectRoot$1(tokens, world);
     //clean up anything weird
     tokens = postProcess$1(tokens);
     // console.log(tokens)
@@ -3328,6 +3411,10 @@
     }
     // support optimized (one|two)
     if (reg.fastOr !== undefined) {
+      // {work/verb} must be a verb
+      if (reg.pos && !term.tags.has(reg.pos)) {
+        return null
+      }
       return reg.fastOr.has(term.implicit) || reg.fastOr.has(term.normal) || reg.fastOr.has(term.text) || reg.fastOr.has(term.machine)
     }
     //support slower (one|two)
@@ -4601,6 +4688,9 @@
     if (method === 'normal') {
       return this.text('normal')
     }
+    if (method === 'root') {
+      return this.text('root')
+    }
     if (method === 'machine' || method === 'reduced') {
       return this.text('machine')
     }
@@ -5240,7 +5330,7 @@
     return { wants, count }
   };
 
-  const parse$1 = function (matches, world) {
+  const parse$2 = function (matches, world) {
     const parseMatch = world.methods.one.parseMatch;
     matches.forEach(obj => {
       obj.regs = parseMatch(obj.match, {}, world);
@@ -5259,12 +5349,12 @@
     return matches
   };
 
-  var parse$2 = parse$1;
+  var parse$3 = parse$2;
 
   // do some indexing on the list of matches
-  const compile = function (matches, world) {
+  const buildNet = function (matches, world) {
     // turn match-syntax into json
-    matches = parse$2(matches, world);
+    matches = parse$3(matches, world);
 
     // collect by wants and needs
     let hooks = {};
@@ -5300,7 +5390,7 @@
     }
   };
 
-  var buildNet = compile;
+  var buildNet$1 = buildNet;
 
   // for each cached-sentence, find a list of possible matches
   const getHooks = function (docCaches, hooks) {
@@ -5456,7 +5546,7 @@
       console.log(`\n\n  \x1b[32m→ ${list.length} post-tagger:\x1b[0m`); //eslint-disable-line
     }
     return list.map(todo => {
-      if (!todo.tag && !todo.chunk) {
+      if (!todo.tag && !todo.chunk && !todo.unTag) {
         return
       }
       let reason = todo.reason || todo.match;
@@ -5493,7 +5583,7 @@
   var bulkTagger = tagger$3;
 
   var methods$3 = {
-    buildNet,
+    buildNet: buildNet$1,
     bulkMatch,
     bulkTagger
   };
@@ -5549,7 +5639,7 @@
     }
     // finally, add our tag
     term.tags.add(tag);
-    // now it's dirty
+    // now it's dirty?
     term.dirty = true;
     // add a chunk too, if it's easy
     addChunk(term, tag);
@@ -5964,71 +6054,20 @@
     }
     return all
   };
-  var basicSplit$1 = basicSplit;
-
-  const isAcronym$2 = /[ .][A-Z]\.? *$/i;
-  const hasEllipse = /(?:\u2026|\.{2,}) *$/;
-  const hasLetter$1 = /\p{L}/u;
-
-  /** does this look like a sentence? */
-  const isSentence = function (str, abbrevs) {
-    // must have a letter
-    if (hasLetter$1.test(str) === false) {
-      return false
-    }
-    // check for 'F.B.I.'
-    if (isAcronym$2.test(str) === true) {
-      return false
-    }
-    //check for '...'
-    if (hasEllipse.test(str) === true) {
-      return false
-    }
-    let txt = str.replace(/[.!?\u203D\u2E18\u203C\u2047-\u2049] *$/, '');
-    let words = txt.split(' ');
-    let lastWord = words[words.length - 1].toLowerCase();
-    // check for 'Mr.'
-    if (abbrevs.hasOwnProperty(lastWord) === true) {
-      return false
-    }
-    // //check for jeopardy!
-    // if (blacklist.hasOwnProperty(lastWord)) {
-    //   return false
-    // }
-    return true
-  };
-  var isSentence$1 = isSentence;
+  var simpleSplit = basicSplit;
 
-  //(Rule-based sentence boundary segmentation) - chop given text into its proper sentences.
-  // Ignore periods/questions/exclamations used in acronyms/abbreviations/numbers, etc.
-  //regs-
-  const hasSomething = /\S/;
-  const startWhitespace = /^\s+/;
-  const hasLetter = /[a-z0-9\u00C0-\u00FF\u00a9\u00ae\u2000-\u3300\ud000-\udfff]/i;
+  const hasLetter$2 = /[a-z0-9\u00C0-\u00FF\u00a9\u00ae\u2000-\u3300\ud000-\udfff]/i;
+  const hasSomething$1 = /\S/;
 
-  const splitSentences = function (text, model) {
-    let abbrevs = model.one.abbreviations || new Set();
-    text = text || '';
-    text = String(text);
-    let sentences = [];
-    // First do a greedy-split..
+  const notEmpty = function (splits) {
     let chunks = [];
-    // Ensure it 'smells like' a sentence
-    if (!text || typeof text !== 'string' || hasSomething.test(text) === false) {
-      return sentences
-    }
-    // cleanup unicode-spaces
-    text = text.replace('\xa0', ' ');
-    // Start somewhere:
-    let splits = basicSplit$1(text);
-    // Filter-out the crap ones
     for (let i = 0; i < splits.length; i++) {
       let s = splits[i];
       if (s === undefined || s === '') {
         continue
       }
       //this is meaningful whitespace
-      if (hasSomething.test(s) === false || hasLetter.test(s) === false) {
+      if (hasSomething$1.test(s) === false || hasLetter$2.test(s) === false) {
         //add it to the last one
         if (chunks[chunks.length - 1]) {
           chunks[chunks.length - 1] += s;
@@ -6042,12 +6081,20 @@
       //else, only whitespace, no terms, no sentence
       chunks.push(s);
     }
-    //detection of non-sentence chunks:
-    //loop through these chunks, and join the non-sentence chunks back together..
+    return chunks
+  };
+  var simpleMerge = notEmpty;
+
+  //loop through these chunks, and join the non-sentence chunks back together..
+  const smartMerge = function (chunks, world) {
+    const isSentence = world.methods.one.tokenize.isSentence;
+    const abbrevs = world.model.one.abbreviations || new Set();
+
+    let sentences = [];
     for (let i = 0; i < chunks.length; i++) {
       let c = chunks[i];
       //should this chunk be combined with the next one?
-      if (chunks[i + 1] && isSentence$1(c, abbrevs) === false) {
+      if (chunks[i + 1] && isSentence(c, abbrevs) === false) {
         chunks[i + 1] = c + (chunks[i + 1] || '');
       } else if (c && c.length > 0) {
         //this chunk is a proper sentence..
@@ -6055,6 +6102,31 @@
         chunks[i] = '';
       }
     }
+    return sentences
+  };
+  var smartMerge$1 = smartMerge;
+
+  //(Rule-based sentence boundary segmentation) - chop given text into its proper sentences.
+  // Ignore periods/questions/exclamations used in acronyms/abbreviations/numbers, etc.
+  //regs-
+  const hasSomething = /\S/;
+  const startWhitespace = /^\s+/;
+
+  const splitSentences = function (text, world) {
+    text = text || '';
+    text = String(text);
+    // Ensure it 'smells like' a sentence
+    if (!text || typeof text !== 'string' || hasSomething.test(text) === false) {
+      return []
+    }
+    // cleanup unicode-spaces
+    text = text.replace('\xa0', ' ');
+    // First do a greedy-split..
+    let splits = simpleSplit(text);
+    // Filter-out the crap ones
+    let chunks = simpleMerge(splits);
+    //detection of non-sentence chunks:
+    let sentences = smartMerge$1(chunks, world);
     //if we never got a sentence, return the given text
     if (sentences.length === 0) {
       return [text]
@@ -6070,7 +6142,7 @@
     }
     return sentences
   };
-  var sentence = splitSentences;
+  var splitSentences$1 = splitSentences;
 
   const hasHyphen = function (str, model) {
     let parts = str.split(/[-–—]/);
@@ -6218,12 +6290,12 @@
     result = result.filter(s => s);
     return result
   };
-  var term = splitWords;
+  var splitTerms = splitWords;
 
   //all punctuation marks, from https://en.wikipedia.org/wiki/Punctuation
   //we have slightly different rules for start/end - like #hashtags.
   const startings =
-    /^[ \n\t.[\](){}⟨⟩:,،、‒–—―…!‹›«»‐\-?‘’;/⁄·&*•^†‡°¡¿※№÷׺ª%‰+−=‱¶′″‴§~|‖¦©℗®℠™¤₳฿\u0022\uFF02\u0027\u201C\u201F\u201B\u201E\u2E42\u201A\u2035\u2036\u2037\u301D\u0060\u301F]+/;
+    /^[ \n\t.[\](){}⟨⟩:,،、‒–—―…!‹›«»‐\-?‘’;/⁄·&*•^†‡¡¿※№÷׺ª%‰+−=‱¶′″‴§~|‖¦©℗®℠™¤₳฿\u0022\uFF02\u0027\u201C\u201F\u201B\u201E\u2E42\u201A\u2035\u2036\u2037\u301D\u0060\u301F]+/;
   const endings =
     /[ \n\t.'[\](){}⟨⟩:,،、‒–—―…!‹›«»‐\-?‘’;/⁄·&*@•^†‡°¡¿※#№÷׺ª‰+−=‱¶′″‴§~|‖¦©℗®℠™¤₳฿\u0022\uFF02\u201D\u00B4\u301E]+$/;
   const hasApostrophe$1 = /['’]/;
@@ -6289,7 +6361,21 @@
     };
     return parsed
   };
-  var whitespace = parseTerm;
+  var splitWhitespace = parseTerm;
+
+  // 'Björk' to 'Bjork'.
+  const killUnicode = function (str, world) {
+    const unicode = world.model.one.unicode || {};
+    str = str || '';
+    let chars = str.split('');
+    chars.forEach((s, i) => {
+      if (unicode[s]) {
+        chars[i] = unicode[s];
+      }
+    });
+    return chars.join('')
+  };
+  var killUnicode$1 = killUnicode;
 
   /** some basic operations on a string to reduce noise */
   const clean = function (str) {
@@ -6329,7 +6415,7 @@
   const noPeriodAcronym$1 = /[A-Z]{2,}('s|,)?$/;
   const lowerCaseAcronym$1 = /([a-z]\.)+[a-z]\.?$/;
 
-  const isAcronym$1 = function (str) {
+  const isAcronym$3 = function (str) {
     //like N.D.A
     if (periodAcronym$1.test(str) === true) {
       return true
@@ -6350,7 +6436,7 @@
   };
 
   const doAcronym = function (str) {
-    if (isAcronym$1(str)) {
+    if (isAcronym$3(str)) {
       str = str.replace(/\./g, '');
     }
     return str
@@ -6369,27 +6455,13 @@
   };
   var normal = normalize;
 
-  // 'Björk' to 'Bjork'.
-  const killUnicode = function (str, world) {
-    const unicode = world.model.one.unicode || {};
-    str = str || '';
-    let chars = str.split('');
-    chars.forEach((s, i) => {
-      if (unicode[s]) {
-        chars[i] = unicode[s];
-      }
-    });
-    return chars.join('')
-  };
-  var killUnicode$1 = killUnicode;
-
   // turn a string input into a 'document' json format
-  const fromString = function (input, world) {
+  const parse$1 = function (input, world) {
     const { methods, model } = world;
     const { splitSentences, splitTerms, splitWhitespace } = methods.one.tokenize;
     input = input || '';
     // split into sentences
-    let sentences = splitSentences(input, model);
+    let sentences = splitSentences(input, world);
     // split into word objects
     input = sentences.map((txt) => {
       let terms = splitTerms(txt, model);
@@ -6403,14 +6475,49 @@
     });
     return input
   };
+  var fromString = parse$1;
+
+  const isAcronym$2 = /[ .][A-Z]\.? *$/i;
+  const hasEllipse$1 = /(?:\u2026|\.{2,}) *$/;
+  const hasLetter$1 = /\p{L}/u;
+
+  /** does this look like a sentence? */
+  const isSentence$2 = function (str, abbrevs) {
+    // must have a letter
+    if (hasLetter$1.test(str) === false) {
+      return false
+    }
+    // check for 'F.B.I.'
+    if (isAcronym$2.test(str) === true) {
+      return false
+    }
+    //check for '...'
+    if (hasEllipse$1.test(str) === true) {
+      return false
+    }
+    let txt = str.replace(/[.!?\u203D\u2E18\u203C\u2047-\u2049] *$/, '');
+    let words = txt.split(' ');
+    let lastWord = words[words.length - 1].toLowerCase();
+    // check for 'Mr.'
+    if (abbrevs.hasOwnProperty(lastWord) === true) {
+      return false
+    }
+    // //check for jeopardy!
+    // if (blacklist.hasOwnProperty(lastWord)) {
+    //   return false
+    // }
+    return true
+  };
+  var isSentence$3 = isSentence$2;
 
   var methods$1 = {
     one: {
       killUnicode: killUnicode$1,
       tokenize: {
-        splitSentences: sentence,
-        splitTerms: term,
-        splitWhitespace: whitespace,
+        splitSentences: splitSentences$1,
+        isSentence: isSentence$3,
+        splitTerms,
+        splitWhitespace,
         fromString,
       },
     },
@@ -6606,7 +6713,7 @@
     'dl',
     'ml',
     'gal',
-    'ft', //ambiguous
+    // 'ft', //ambiguous
     'qt',
     'pt',
     'tbl',
@@ -6635,13 +6742,12 @@
     'kmph', //kilometers per hour
     'kb', //kilobyte
     'mb', //megabyte
-    'gb', //ambig
+    // 'gb', //ambig
     'tb', //terabyte
     'lx', //lux
     'lm', //lumen
-    'pa', //ambig
+    // 'pa', //ambig
     'fl oz', //
-
     'yb',
   ];
 
@@ -6761,7 +6867,7 @@
     r: 'ŔŕŖŗŘřƦȐȑȒȓɌɍЃГЯгяѓҐґ',
     s: 'ŚśŜŝŞşŠšƧƨȘșȿЅѕ',
     t: 'ŢţŤťŦŧƫƬƭƮȚțȶȾΓΤτϮТт',
-    u: 'µÙÚÛÜùúûüŨũŪūŬŭŮůŰűŲųƯưƱƲǓǔǕǖǗǘǙǚǛǜȔȕȖȗɄΰμυϋύ',
+    u: 'ÙÚÛÜùúûüŨũŪūŬŭŮůŰűŲųƯưƱƲǓǔǕǖǗǘǙǚǛǜȔȕȖȗɄΰυϋύ',
     v: 'νѴѵѶѷ',
     w: 'ŴŵƜωώϖϢϣШЩшщѡѿ',
     x: '×ΧχϗϰХхҲҳӼӽӾӿ',
@@ -6795,7 +6901,7 @@
   // const hasApostrophe = /['’]s$/
 
   const addAliases = function (term, world) {
-    let str = term.normal || term.text;
+    let str = term.normal || term.text || term.machine;
     const aliases = world.model.one.aliases;
     // lookup known aliases like '&'
     if (aliases.hasOwnProperty(str)) {
@@ -7940,10 +8046,49 @@
     { word: 'zur', out: ['zu', 'der'] },  //(to the / to)
   ];
 
+  const isAcronym$1 = /[ .][A-Z]\.? *$/i;
+  const hasEllipse = /(?:\u2026|\.{2,}) *$/;
+  const hasLetter = /\p{L}/u;
+  const isOrdinal = /[0-9]\. *$/;
+
+  /** does this look like a sentence? */
+  const isSentence = function (str, abbrevs) {
+    // must have a letter
+    if (hasLetter.test(str) === false) {
+      return false
+    }
+    // check for 'F.B.I.'
+    if (isAcronym$1.test(str) === true) {
+      return false
+    }
+    // german ordinals like '4.'
+    if (isOrdinal.test(str) === true) {
+      return false
+    }
+    //check for '...'
+    if (hasEllipse.test(str) === true) {
+      return false
+    }
+    let txt = str.replace(/[.!?\u203D\u2E18\u203C\u2047-\u2049] *$/, '');
+    let words = txt.split(' ');
+    let lastWord = words[words.length - 1].toLowerCase();
+    // check for 'Mr.'
+    if (abbrevs.hasOwnProperty(lastWord) === true) {
+      return false
+    }
+    // //check for jeopardy!
+    // if (blacklist.hasOwnProperty(lastWord)) {
+    //   return false
+    // }
+    return true
+  };
+  var isSentence$1 = isSentence;
+
   var tokenizer = {
     mutate: (world) => {
       world.model.one.unicode = unicode$1;
       world.model.one.contractions = contractions;
+      world.methods.one.tokenize.isSentence = isSentence$1;
     },
   };
 
@@ -8079,6 +8224,8 @@
   };
 
   const hasApostrophe = /['‘’‛‵′`´]/;
+  const hasPeriod = /\./;
+  const isNum = /^[0-9+-,]+$/;
 
   // normal regexes
   const doRegs = function (str, regs) {
@@ -8100,6 +8247,12 @@
     if (hasApostrophe.test(term.post) && !hasApostrophe.test(term.pre)) {
       text += term.post.trim();
     }
+    // keep period in number ordinals?
+    if (hasPeriod.test(term.post) && isNum.test(text)) {
+      setTag([term], ['Ordinal', 'NumericValue'], world, false, `1-regex-ordinal`);
+      term.confidence = 0.6;
+      return true
+    }
     let arr = doRegs(text, regexText) || doRegs(normal, regexNormal);
     // hide a bunch of number regexes behind this one
     if (!arr && /[0-9]/.test(normal)) {
@@ -8512,6 +8665,9 @@
     // ending-apostrophes
     [/.{3}[lkmnp]in['‘’‛‵′`´]$/, 'Gerund', "chillin'"],
     [/.{4}s['‘’‛‵′`´]$/, 'Possessive', "flanders'"],
+
+    // german ordinals '4.'
+    // [/^[0-9]+\.$/, 'Ordinal'],
   ];
 
   const rb = 'Adverb';
@@ -9333,7 +9489,7 @@
     api: api$1
   };
 
-  var version = '0.0.4';
+  var version = '0.0.5';
 
   nlp$1.plugin(tokenizer);
   nlp$1.plugin(tagset);