@pie-lib/text-select 1.12.8-next.1 → 1.13.1-beta.0

package/lib/tokenizer/builder.js

@@ -1,311 +0,0 @@
-"use strict";
-
-var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.words = exports.sort = exports.sentences = exports.paragraphs = exports.normalize = exports.intersection = exports.handleSentence = void 0;
-
-var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
-
-var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
-
-var _compact = _interopRequireDefault(require("lodash/compact"));
-
-var _parseEnglish = _interopRequireDefault(require("@pie-framework/parse-english"));
-
-var _clone = _interopRequireDefault(require("lodash/clone"));
-
-var g = function g(str, node) {
-  if (node.children) {
-    return node.children.reduce(g, str);
-  } else if (node.value) {
-    return str + node.value;
-  } else {
-    return str;
-  }
-};
-
-var getParagraph = function getParagraph(p) {
-  return g('', p);
-};
-
-var getSentence = function getSentence(s) {
-  return g('', s);
-};
-
-var getWord = function getWord(w) {
-  return g('', w);
-};
-
-var paragraphs = function paragraphs(text) {
-  var tree = new _parseEnglish["default"]().parse(text);
-  var out = tree.children.reduce(function (acc, child) {
-    if (child.type === 'ParagraphNode') {
-      var paragraph = {
-        text: getParagraph(child),
-        start: child.position.start.offset,
-        end: child.position.end.offset
-      };
-      return acc.concat([paragraph]);
-    } else {
-      return acc;
-    }
-  }, []);
-  return out;
-};
-
-exports.paragraphs = paragraphs;
-
-var handleSentence = function handleSentence(child, acc) {
-  var sentenceChilds = []; // we parse the children of the sentence
-
-  var newAcc = child.children.reduce(function (acc, child) {
-    // if we find a whitespace node that's \n, we end the sentence
-    if (child.type === 'WhiteSpaceNode' && child.value === '\n') {
-      if (sentenceChilds.length) {
-        var firstWord = sentenceChilds[0]; // we create a sentence starting from the first word until the new line
-
-        var sentence = {
-          text: sentenceChilds.map(function (d) {
-            return getSentence(d);
-          }).join(''),
-          start: firstWord.position.start.offset,
-          end: child.position.start.offset
-        }; // we remove all the elements from the array
-
-        sentenceChilds.splice(0, sentenceChilds.length);
-        return acc.concat([sentence]);
-      }
-    } else {
-      // otherwise we add it to the array that contains the child forming a sentence
-      sentenceChilds.push(child);
-    }
-
-    return acc;
-  }, acc); // we treat the case when no \n character is found at the end
-  // so we create a sentence from the last words or white spaces found
-
-  if (sentenceChilds.length) {
-    var firstWord = sentenceChilds[0];
-    var lastWord = sentenceChilds[sentenceChilds.length - 1];
-    var sentence = {
-      text: sentenceChilds.map(function (d) {
-        return getSentence(d);
-      }).join(''),
-      start: firstWord.position.start.offset,
-      end: lastWord.position.end.offset
-    };
-    newAcc = newAcc.concat([sentence]);
-    sentenceChilds.splice(0, sentenceChilds.length);
-  }
-
-  return newAcc;
-};
-
-exports.handleSentence = handleSentence;
-
-var sentences = function sentences(text) {
-  var tree = new _parseEnglish["default"]().parse(text);
-  var out = tree.children.reduce(function (acc, child) {
-    if (child.type === 'ParagraphNode') {
-      return child.children.reduce(function (acc, child) {
-        if (child.type === 'SentenceNode') {
-          var newAcc = handleSentence(child, acc);
-          return newAcc || acc;
-        } else {
-          return acc;
-        }
-      }, acc);
-    } else {
-      return acc;
-    }
-  }, []);
-  return out;
-};
-
-exports.sentences = sentences;
-
-var words = function words(text) {
-  var tree = new _parseEnglish["default"]().parse(text);
-  var out = tree.children.reduce(function (acc, child) {
-    if (child.type === 'ParagraphNode') {
-      return child.children.reduce(function (acc, child) {
-        if (child.type === 'SentenceNode') {
-          return child.children.reduce(function (acc, child) {
-            if (child.type === 'WordNode') {
-              var node = {
-                text: getWord(child),
-                start: child.position.start.offset,
-                end: child.position.end.offset
-              };
-              return acc.concat([node]);
-            } else {
-              return acc;
-            }
-          }, acc);
-        } else {
-          return acc;
-        }
-      }, acc);
-    } else {
-      return acc;
-    }
-  }, []);
-  return out;
-};
-
-exports.words = words;
-
-var Intersection = /*#__PURE__*/function () {
-  function Intersection(results) {
-    (0, _classCallCheck2["default"])(this, Intersection);
-    this.results = results;
-  }
-
-  (0, _createClass2["default"])(Intersection, [{
-    key: "hasOverlap",
-    get: function get() {
-      return this.results.filter(function (r) {
-        return r.type === 'overlap';
-      }).length > 0;
-    }
-  }, {
-    key: "surroundedTokens",
-    get: function get() {
-      return this.results.filter(function (r) {
-        return r.type === 'within-selection';
-      }).map(function (t) {
-        return t.token;
-      });
-    }
-  }]);
-  return Intersection;
-}();
-/**
- * get intersection info for the selection in relation to tokens.
- * @param {{start: number, end: number}} selection
- * @param {{start: number, end: number}[]} tokens
- * @return {tokens: [], type: 'overlap|no-overlap|contains'}
- */
-
-
-var intersection = function intersection(selection, tokens) {
-  var start = selection.start,
-      end = selection.end;
-
-  var startsWithin = function startsWithin(t) {
-    return start >= t.start && start < t.end;
-  };
-
-  var endsWithin = function endsWithin(t) {
-    return end > t.start && end <= t.end;
-  };
-
-  var mapped = tokens.map(function (t) {
-    if (start === t.start && end === t.end) {
-      return {
-        token: t,
-        type: 'exact-fit'
-      };
-    } else if (start <= t.start && end >= t.end) {
-      return {
-        token: t,
-        type: 'within-selection'
-      };
-    } else if (startsWithin(t) || endsWithin(t)) {
-      return {
-        token: t,
-        type: 'overlap'
-      };
-    }
-  });
-  return new Intersection((0, _compact["default"])(mapped));
-};
-
-exports.intersection = intersection;
-
-var sort = function sort(tokens) {
-  if (!Array.isArray(tokens)) {
-    return tokens;
-  } else {
-    var out = (0, _clone["default"])(tokens);
-    out.sort(function (a, b) {
-      var s = a.start < b.start ? -1 : a.start > b.start ? 1 : 0;
-      var e = a.end < b.end ? -1 : a.end > b.end ? 1 : 0;
-
-      if (s === -1 && e !== -1) {
-        throw new Error("sort does not support intersecting tokens. a: ".concat(a.start, "-").concat(a.end, ", b: ").concat(b.start, "-").concat(b.end));
-      }
-
-      return s;
-    });
-    return out;
-  }
-};
-
-exports.sort = sort;
-
-var normalize = function normalize(textToNormalize, tokens) {
-  // making sure text provided is a string
-  var text = textToNormalize || '';
-
-  if (!Array.isArray(tokens) || tokens.length === 0) {
-    return [{
-      text: text,
-      start: 0,
-      end: text.length
-    }];
-  }
-
-  var out = sort(tokens).reduce(function (acc, t, index, outer) {
-    var tokens = [];
-    var lastIndex = acc.lastIndex;
-
-    if (t.start === lastIndex) {
-      tokens = [{
-        text: text.substring(lastIndex, t.end),
-        start: lastIndex,
-        end: t.end,
-        predefined: true,
-        correct: t.correct,
-        isMissing: t.isMissing
-      }];
-    } else if (lastIndex < t.start) {
-      tokens = [{
-        text: text.substring(lastIndex, t.start),
-        start: lastIndex,
-        end: t.start
-      }, {
-        text: text.substring(t.start, t.end),
-        start: t.start,
-        end: t.end,
-        predefined: true,
-        correct: t.correct,
-        isMissing: t.isMissing
-      }];
-    }
-
-    if (index === outer.length - 1 && t.end < text.length) {
-      var last = {
-        text: text.substring(t.end),
-        start: t.end,
-        end: text.length
-      };
-      tokens.push(last);
-    }
-
-    return {
-      lastIndex: tokens.length ? tokens[tokens.length - 1].end : lastIndex,
-      result: acc.result.concat(tokens)
-    };
-  }, {
-    result: [],
-    lastIndex: 0
-  });
-  return out.result;
-};
-
-exports.normalize = normalize;
-//# sourceMappingURL=builder.js.map

package/lib/tokenizer/builder.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../src/tokenizer/builder.js"],"names":["g","str","node","children","reduce","value","getParagraph","p","getSentence","s","getWord","w","paragraphs","text","tree","English","parse","out","acc","child","type","paragraph","start","position","offset","end","concat","handleSentence","sentenceChilds","newAcc","length","firstWord","sentence","map","d","join","splice","push","lastWord","sentences","words","Intersection","results","filter","r","t","token","intersection","selection","tokens","startsWithin","endsWithin","mapped","sort","Array","isArray","a","b","e","Error","normalize","textToNormalize","index","outer","lastIndex","substring","predefined","correct","isMissing","last","result"],"mappings":";;;;;;;;;;;;;AAAA;;AACA;;AACA;;AAEA,IAAMA,CAAC,GAAG,SAAJA,CAAI,CAACC,GAAD,EAAMC,IAAN,EAAe;AACvB,MAAIA,IAAI,CAACC,QAAT,EAAmB;AACjB,WAAOD,IAAI,CAACC,QAAL,CAAcC,MAAd,CAAqBJ,CAArB,EAAwBC,GAAxB,CAAP;AACD,GAFD,MAEO,IAAIC,IAAI,CAACG,KAAT,EAAgB;AACrB,WAAOJ,GAAG,GAAGC,IAAI,CAACG,KAAlB;AACD,GAFM,MAEA;AACL,WAAOJ,GAAP;AACD;AACF,CARD;;AAUA,IAAMK,YAAY,GAAG,SAAfA,YAAe,CAACC,CAAD;AAAA,SAAOP,CAAC,CAAC,EAAD,EAAKO,CAAL,CAAR;AAAA,CAArB;;AAEA,IAAMC,WAAW,GAAG,SAAdA,WAAc,CAACC,CAAD;AAAA,SAAOT,CAAC,CAAC,EAAD,EAAKS,CAAL,CAAR;AAAA,CAApB;;AAEA,IAAMC,OAAO,GAAG,SAAVA,OAAU,CAACC,CAAD;AAAA,SAAOX,CAAC,CAAC,EAAD,EAAKW,CAAL,CAAR;AAAA,CAAhB;;AAEO,IAAMC,UAAU,GAAG,SAAbA,UAAa,CAACC,IAAD,EAAU;AAClC,MAAMC,IAAI,GAAG,IAAIC,wBAAJ,GAAcC,KAAd,CAAoBH,IAApB,CAAb;AAEA,MAAMI,GAAG,GAAGH,IAAI,CAACX,QAAL,CAAcC,MAAd,CAAqB,UAACc,GAAD,EAAMC,KAAN,EAAgB;AAC/C,QAAIA,KAAK,CAACC,IAAN,KAAe,eAAnB,EAAoC;AAClC,UAAMC,SAAS,GAAG;AAChBR,QAAAA,IAAI,EAAEP,YAAY,CAACa,KAAD,CADF;AAEhBG,QAAAA,KAAK,EAAEH,KAAK,CAACI,QAAN,CAAeD,KAAf,CAAqBE,MAFZ;AAGhBC,QAAAA,GAAG,EAAEN,KAAK,CAACI,QAAN,CAAeE,GAAf,CAAmBD;AAHR,OAAlB;AAMA,aAAON,GAAG,CAACQ,MAAJ,CAAW,CAACL,SAAD,CAAX,CAAP;AACD,KARD,MAQO;AACL,aAAOH,GAAP;AACD;AACF,GAZW,EAYT,EAZS,CAAZ;AAcA,SAAOD,GAAP;AACD,CAlBM;;;;AAoBA,IAAMU,cAAc,GAAG,SAAjBA,cAAiB,CAACR,KAAD,EAAQD,GAAR,EAAgB;AAC5C,MAAMU,cAAc,GAAG,EAAvB,CAD4C,CAE5C;;AACA,MAAIC,MAAM,GAAGV,KAAK,CAAChB,QAAN,CAAeC,MAAf,CAAsB,UAASc,GAAT,EAAcC,KAAd,EAAqB;AACtD;AACA,QAAIA,KAAK,CAACC,IAAN,KAAe,gBAAf,IAAmCD,KAAK,CAACd,KAAN,KAAgB,IAAvD,EAA6D;AAC3D,UAAIuB,cAAc,CAACE,MAAnB,EAA2B;AACzB,YAAMC,SAAS,GAAGH,cAAc,CAAC,CAAD,CAAhC,CADyB,CAEzB;;AACA,YAAMI,QAAQ,GAAG;AACfnB,UAAAA,IAAI,EAAEe,cAAc,CAACK,GAAf,CAAmB,UAACC,CAAD;AAAA,mBAAO1B,WAAW,CAAC0B,CAAD,CAAlB;AAAA,WAAnB,EAA0CC,IAA1C,CAA+C,EAA/C,CADS;AAEfb,UAAAA,KAAK,EAAES,SAAS,CAACR,QAAV,CAAmBD,KAAnB,CAAyBE,MAFjB;AAGfC,UAAAA,GAAG,EAAEN,KAAK,CAACI,QAAN,CAAeD,KAAf,CAAqBE;AAHX,SAAjB,CAHyB,CASzB;;AACAI,QAAAA,cAAc,CAACQ,MAAf,CAAsB,CAAtB,EAAyBR,cAAc,CAACE,MAAxC;AACA,eAAOZ,GAAG,CAACQ,MAAJ,CAAW,CAACM,QAAD,CAAX,CAAP;AACD;AACF,KAdD,MAcO;AACL;AACAJ,MAAAA,cAAc,CAACS,IAAf,CAAoBlB,KAApB;AACD;;AAED,WAAOD,GAAP;AACD,GAtBY,EAsBVA,GAtBU,CAAb,CAH4C,CA2B5C;AACA;;AACA,MAAIU,cAAc,CAACE,MAAnB,EAA2B;AACzB,QAAMC,SAAS,GAAGH,cAAc,CAAC,CAAD,CAAhC;AACA,QAAMU,QAAQ,GAAGV,cAAc,CAACA,cAAc,CAACE,MAAf,GAAwB,CAAzB,CAA/B;AACA,QAAME,QAAQ,GAAG;AACfnB,MAAAA,IAAI,EAAEe,cAAc,CAACK,GAAf,CAAmB,UAACC,CAAD;AAAA,eAAO1B,WAAW,CAAC0B,CAAD,CAAlB;AAAA,OAAnB,EAA0CC,IAA1C,CAA+C,EAA/C,CADS;AAEfb,MAAAA,KAAK,EAAES,SAAS,CAACR,QAAV,CAAmBD,KAAnB,CAAyBE,MAFjB;AAGfC,MAAAA,GAAG,EAAEa,QAAQ,CAACf,QAAT,CAAkBE,GAAlB,CAAsBD;AAHZ,KAAjB;AAMAK,IAAAA,MAAM,GAAGA,MAAM,CAACH,MAAP,CAAc,CAACM,QAAD,CAAd,CAAT;AAEAJ,IAAAA,cAAc,CAACQ,MAAf,CAAsB,CAAtB,EAAyBR,cAAc,CAACE,MAAxC;AACD;;AAED,SAAOD,MAAP;AACD,CA5CM;;;;AA8CA,IAAMU,SAAS,GAAG,SAAZA,SAAY,CAAC1B,IAAD,EAAU;AACjC,MAAMC,IAAI,GAAG,IAAIC,wBAAJ,GAAcC,KAAd,CAAoBH,IAApB,CAAb;AAEA,MAAMI,GAAG,GAAGH,IAAI,CAACX,QAAL,CAAcC,MAAd,CAAqB,UAACc,GAAD,EAAMC,KAAN,EAAgB;AAC/C,QAAIA,KAAK,CAACC,IAAN,KAAe,eAAnB,EAAoC;AAClC,aAAOD,KAAK,CAAChB,QAAN,CAAeC,MAAf,CAAsB,UAACc,GAAD,EAAMC,KAAN,EAAgB;AAC3C,YAAIA,KAAK,CAACC,IAAN,KAAe,cAAnB,EAAmC;AACjC,cAAMS,MAAM,GAAGF,cAAc,CAACR,KAAD,EAAQD,GAAR,CAA7B;AAEA,iBAAOW,MAAM,IAAIX,GAAjB;AACD,SAJD,MAIO;AACL,iBAAOA,GAAP;AACD;AACF,OARM,EAQJA,GARI,CAAP;AASD,KAVD,MAUO;AACL,aAAOA,GAAP;AACD;AACF,GAdW,EAcT,EAdS,CAAZ;AAgBA,SAAOD,GAAP;AACD,CApBM;;;;AAqBA,IAAMuB,KAAK,GAAG,SAARA,KAAQ,CAAC3B,IAAD,EAAU;AAC7B,MAAMC,IAAI,GAAG,IAAIC,wBAAJ,GAAcC,KAAd,CAAoBH,IAApB,CAAb;AAEA,MAAMI,GAAG,GAAGH,IAAI,CAACX,QAAL,CAAcC,MAAd,CAAqB,UAACc,GAAD,EAAMC,KAAN,EAAgB;AAC/C,QAAIA,KAAK,CAACC,IAAN,KAAe,eAAnB,EAAoC;AAClC,aAAOD,KAAK,CAAChB,QAAN,CAAeC,MAAf,CAAsB,UAACc,GAAD,EAAMC,KAAN,EAAgB;AAC3C,YAAIA,KAAK,CAACC,IAAN,KAAe,cAAnB,EAAmC;AACjC,iBAAOD,KAAK,CAAChB,QAAN,CAAeC,MAAf,CAAsB,UAACc,GAAD,EAAMC,KAAN,EAAgB;AAC3C,gBAAIA,KAAK,CAACC,IAAN,KAAe,UAAnB,EAA+B;AAC7B,kBAAMlB,IAAI,GAAG;AACXW,gBAAAA,IAAI,EAAEH,OAAO,CAACS,KAAD,CADF;AAEXG,gBAAAA,KAAK,EAAEH,KAAK,CAACI,QAAN,CAAeD,KAAf,CAAqBE,MAFjB;AAGXC,gBAAAA,GAAG,EAAEN,KAAK,CAACI,QAAN,CAAeE,GAAf,CAAmBD;AAHb,eAAb;AAKA,qBAAON,GAAG,CAACQ,MAAJ,CAAW,CAACxB,IAAD,CAAX,CAAP;AACD,aAPD,MAOO;AACL,qBAAOgB,GAAP;AACD;AACF,WAXM,EAWJA,GAXI,CAAP;AAYD,SAbD,MAaO;AACL,iBAAOA,GAAP;AACD;AACF,OAjBM,EAiBJA,GAjBI,CAAP;AAkBD,KAnBD,MAmBO;AACL,aAAOA,GAAP;AACD;AACF,GAvBW,EAuBT,EAvBS,CAAZ;AAyBA,SAAOD,GAAP;AACD,CA7BM;;;;IA+BDwB,Y;AACJ,wBAAYC,OAAZ,EAAqB;AAAA;AACnB,SAAKA,OAAL,GAAeA,OAAf;AACD;;;;SAED,eAAiB;AACf,aAAO,KAAKA,OAAL,CAAaC,MAAb,CAAoB,UAACC,CAAD;AAAA,eAAOA,CAAC,CAACxB,IAAF,KAAW,SAAlB;AAAA,OAApB,EAAiDU,MAAjD,GAA0D,CAAjE;AACD;;;SAED,eAAuB;AACrB,aAAO,KAAKY,OAAL,CAAaC,MAAb,CAAoB,UAACC,CAAD;AAAA,eAAOA,CAAC,CAACxB,IAAF,KAAW,kBAAlB;AAAA,OAApB,EAA0Da,GAA1D,CAA8D,UAACY,CAAD;AAAA,eAAOA,CAAC,CAACC,KAAT;AAAA,OAA9D,CAAP;AACD;;;;AAEH;AACA;AACA;AACA;AACA;AACA;;;AACO,IAAMC,YAAY,GAAG,SAAfA,YAAe,CAACC,SAAD,EAAYC,MAAZ,EAAuB;AACjD,MAAQ3B,KAAR,GAAuB0B,SAAvB,CAAQ1B,KAAR;AAAA,MAAeG,GAAf,GAAuBuB,SAAvB,CAAevB,GAAf;;AAEA,MAAMyB,YAAY,GAAG,SAAfA,YAAe,CAACL,CAAD;AAAA,WAAOvB,KAAK,IAAIuB,CAAC,CAACvB,KAAX,IAAoBA,KAAK,GAAGuB,CAAC,CAACpB,GAArC;AAAA,GAArB;;AACA,MAAM0B,UAAU,GAAG,SAAbA,UAAa,CAACN,CAAD;AAAA,WAAOpB,GAAG,GAAGoB,CAAC,CAACvB,KAAR,IAAiBG,GAAG,IAAIoB,CAAC,CAACpB,GAAjC;AAAA,GAAnB;;AAEA,MAAM2B,MAAM,GAAGH,MAAM,CAAChB,GAAP,CAAW,UAACY,CAAD,EAAO;AAC/B,QAAIvB,KAAK,KAAKuB,CAAC,CAACvB,KAAZ,IAAqBG,GAAG,KAAKoB,CAAC,CAACpB,GAAnC,EAAwC;AACtC,aAAO;AAAEqB,QAAAA,KAAK,EAAED,CAAT;AAAYzB,QAAAA,IAAI,EAAE;AAAlB,OAAP;AACD,KAFD,MAEO,IAAIE,KAAK,IAAIuB,CAAC,CAACvB,KAAX,IAAoBG,GAAG,IAAIoB,CAAC,CAACpB,GAAjC,EAAsC;AAC3C,aAAO;AAAEqB,QAAAA,KAAK,EAAED,CAAT;AAAYzB,QAAAA,IAAI,EAAE;AAAlB,OAAP;AACD,KAFM,MAEA,IAAI8B,YAAY,CAACL,CAAD,CAAZ,IAAmBM,UAAU,CAACN,CAAD,CAAjC,EAAsC;AAC3C,aAAO;AAAEC,QAAAA,KAAK,EAAED,CAAT;AAAYzB,QAAAA,IAAI,EAAE;AAAlB,OAAP;AACD;AACF,GARc,CAAf;AASA,SAAO,IAAIqB,YAAJ,CAAiB,yBAAQW,MAAR,CAAjB,CAAP;AACD,CAhBM;;;;AAkBA,IAAMC,IAAI,GAAG,SAAPA,IAAO,CAACJ,MAAD,EAAY;AAC9B,MAAI,CAACK,KAAK,CAACC,OAAN,CAAcN,MAAd,CAAL,EAA4B;AAC1B,WAAOA,MAAP;AACD,GAFD,MAEO;AACL,QAAMhC,GAAG,GAAG,uBAAMgC,MAAN,CAAZ;AACAhC,IAAAA,GAAG,CAACoC,IAAJ,CAAS,UAACG,CAAD,EAAIC,CAAJ,EAAU;AACjB,UAAMhD,CAAC,GAAG+C,CAAC,CAAClC,KAAF,GAAUmC,CAAC,CAACnC,KAAZ,GAAoB,CAAC,CAArB,GAAyBkC,CAAC,CAAClC,KAAF,GAAUmC,CAAC,CAACnC,KAAZ,GAAoB,CAApB,GAAwB,CAA3D;AACA,UAAMoC,CAAC,GAAGF,CAAC,CAAC/B,GAAF,GAAQgC,CAAC,CAAChC,GAAV,GAAgB,CAAC,CAAjB,GAAqB+B,CAAC,CAAC/B,GAAF,GAAQgC,CAAC,CAAChC,GAAV,GAAgB,CAAhB,GAAoB,CAAnD;;AACA,UAAIhB,CAAC,KAAK,CAAC,CAAP,IAAYiD,CAAC,KAAK,CAAC,CAAvB,EAA0B;AACxB,cAAM,IAAIC,KAAJ,yDAA2DH,CAAC,CAAClC,KAA7D,cAAsEkC,CAAC,CAAC/B,GAAxE,kBAAmFgC,CAAC,CAACnC,KAArF,cAA8FmC,CAAC,CAAChC,GAAhG,EAAN;AACD;;AACD,aAAOhB,CAAP;AACD,KAPD;AAQA,WAAOQ,GAAP;AACD;AACF,CAfM;;;;AAiBA,IAAM2C,SAAS,GAAG,SAAZA,SAAY,CAACC,eAAD,EAAkBZ,MAAlB,EAA6B;AACpD;AACA,MAAMpC,IAAI,GAAGgD,eAAe,IAAI,EAAhC;;AAEA,MAAI,CAACP,KAAK,CAACC,OAAN,CAAcN,MAAd,CAAD,IAA0BA,MAAM,CAACnB,MAAP,KAAkB,CAAhD,EAAmD;AACjD,WAAO,CACL;AACEjB,MAAAA,IAAI,EAAJA,IADF;AAEES,MAAAA,KAAK,EAAE,CAFT;AAGEG,MAAAA,GAAG,EAAEZ,IAAI,CAACiB;AAHZ,KADK,CAAP;AAOD;;AAED,MAAMb,GAAG,GAAGoC,IAAI,CAACJ,MAAD,CAAJ,CAAa7C,MAAb,CACV,UAACc,GAAD,EAAM2B,CAAN,EAASiB,KAAT,EAAgBC,KAAhB,EAA0B;AACxB,QAAId,MAAM,GAAG,EAAb;AACA,QAAMe,SAAS,GAAG9C,GAAG,CAAC8C,SAAtB;;AAEA,QAAInB,CAAC,CAACvB,KAAF,KAAY0C,SAAhB,EAA2B;AACzBf,MAAAA,MAAM,GAAG,CACP;AACEpC,QAAAA,IAAI,EAAEA,IAAI,CAACoD,SAAL,CAAeD,SAAf,EAA0BnB,CAAC,CAACpB,GAA5B,CADR;AAEEH,QAAAA,KAAK,EAAE0C,SAFT;AAGEvC,QAAAA,GAAG,EAAEoB,CAAC,CAACpB,GAHT;AAIEyC,QAAAA,UAAU,EAAE,IAJd;AAKEC,QAAAA,OAAO,EAAEtB,CAAC,CAACsB,OALb;AAMEC,QAAAA,SAAS,EAAEvB,CAAC,CAACuB;AANf,OADO,CAAT;AAUD,KAXD,MAWO,IAAIJ,SAAS,GAAGnB,CAAC,CAACvB,KAAlB,EAAyB;AAC9B2B,MAAAA,MAAM,GAAG,CACP;AACEpC,QAAAA,IAAI,EAAEA,IAAI,CAACoD,SAAL,CAAeD,SAAf,EAA0BnB,CAAC,CAACvB,KAA5B,CADR;AAEEA,QAAAA,KAAK,EAAE0C,SAFT;AAGEvC,QAAAA,GAAG,EAAEoB,CAAC,CAACvB;AAHT,OADO,EAMP;AACET,QAAAA,IAAI,EAAEA,IAAI,CAACoD,SAAL,CAAepB,CAAC,CAACvB,KAAjB,EAAwBuB,CAAC,CAACpB,GAA1B,CADR;AAEEH,QAAAA,KAAK,EAAEuB,CAAC,CAACvB,KAFX;AAGEG,QAAAA,GAAG,EAAEoB,CAAC,CAACpB,GAHT;AAIEyC,QAAAA,UAAU,EAAE,IAJd;AAKEC,QAAAA,OAAO,EAAEtB,CAAC,CAACsB,OALb;AAMEC,QAAAA,SAAS,EAAEvB,CAAC,CAACuB;AANf,OANO,CAAT;AAeD;;AAED,QAAIN,KAAK,KAAKC,KAAK,CAACjC,MAAN,GAAe,CAAzB,IAA8Be,CAAC,CAACpB,GAAF,GAAQZ,IAAI,CAACiB,MAA/C,EAAuD;AACrD,UAAMuC,IAAI,GAAG;AACXxD,QAAAA,IAAI,EAAEA,IAAI,CAACoD,SAAL,CAAepB,CAAC,CAACpB,GAAjB,CADK;AAEXH,QAAAA,KAAK,EAAEuB,CAAC,CAACpB,GAFE;AAGXA,QAAAA,GAAG,EAAEZ,IAAI,CAACiB;AAHC,OAAb;AAKAmB,MAAAA,MAAM,CAACZ,IAAP,CAAYgC,IAAZ;AACD;;AAED,WAAO;AACLL,MAAAA,SAAS,EAAEf,MAAM,CAACnB,MAAP,GAAgBmB,MAAM,CAACA,MAAM,CAACnB,MAAP,GAAgB,CAAjB,CAAN,CAA0BL,GAA1C,GAAgDuC,SADtD;AAELM,MAAAA,MAAM,EAAEpD,GAAG,CAACoD,MAAJ,CAAW5C,MAAX,CAAkBuB,MAAlB;AAFH,KAAP;AAID,GA/CS,EAgDV;AAAEqB,IAAAA,MAAM,EAAE,EAAV;AAAcN,IAAAA,SAAS,EAAE;AAAzB,GAhDU,CAAZ;AAmDA,SAAO/C,GAAG,CAACqD,MAAX;AACD,CAlEM","sourcesContent":["import compact from 'lodash/compact';\nimport English from '@pie-framework/parse-english';\nimport clone from 'lodash/clone';\n\nconst g = (str, node) => {\n  if (node.children) {\n    return node.children.reduce(g, str);\n  } else if (node.value) {\n    return str + node.value;\n  } else {\n    return str;\n  }\n};\n\nconst getParagraph = (p) => g('', p);\n\nconst getSentence = (s) => g('', s);\n\nconst getWord = (w) => g('', w);\n\nexport const paragraphs = (text) => {\n  const tree = new English().parse(text);\n\n  const out = tree.children.reduce((acc, child) => {\n    if (child.type === 'ParagraphNode') {\n      const paragraph = {\n        text: getParagraph(child),\n        start: child.position.start.offset,\n        end: child.position.end.offset,\n      };\n\n      return acc.concat([paragraph]);\n    } else {\n      return acc;\n    }\n  }, []);\n\n  return out;\n};\n\nexport const handleSentence = (child, acc) => {\n  const sentenceChilds = [];\n  // we parse the children of the sentence\n  let newAcc = child.children.reduce(function(acc, child) {\n    // if we find a whitespace node that's \\n, we end the sentence\n    if (child.type === 'WhiteSpaceNode' && child.value === '\\n') {\n      if (sentenceChilds.length) {\n        const firstWord = sentenceChilds[0];\n        // we create a sentence starting from the first word until the new line\n        const sentence = {\n          text: sentenceChilds.map((d) => getSentence(d)).join(''),\n          start: firstWord.position.start.offset,\n          end: child.position.start.offset,\n        };\n\n        // we remove all the elements from the array\n        sentenceChilds.splice(0, sentenceChilds.length);\n        return acc.concat([sentence]);\n      }\n    } else {\n      // otherwise we add it to the array that contains the child forming a sentence\n      sentenceChilds.push(child);\n    }\n\n    return acc;\n  }, acc);\n\n  // we treat the case when no \\n character is found at the end\n  // so we create a sentence from the last words or white spaces found\n  if (sentenceChilds.length) {\n    const firstWord = sentenceChilds[0];\n    const lastWord = sentenceChilds[sentenceChilds.length - 1];\n    const sentence = {\n      text: sentenceChilds.map((d) => getSentence(d)).join(''),\n      start: firstWord.position.start.offset,\n      end: lastWord.position.end.offset,\n    };\n\n    newAcc = newAcc.concat([sentence]);\n\n    sentenceChilds.splice(0, sentenceChilds.length);\n  }\n\n  return newAcc;\n};\n\nexport const sentences = (text) => {\n  const tree = new English().parse(text);\n\n  const out = tree.children.reduce((acc, child) => {\n    if (child.type === 'ParagraphNode') {\n      return child.children.reduce((acc, child) => {\n        if (child.type === 'SentenceNode') {\n          const newAcc = handleSentence(child, acc);\n\n          return newAcc || acc;\n        } else {\n          return acc;\n        }\n      }, acc);\n    } else {\n      return acc;\n    }\n  }, []);\n\n  return out;\n};\nexport const words = (text) => {\n  const tree = new English().parse(text);\n\n  const out = tree.children.reduce((acc, child) => {\n    if (child.type === 'ParagraphNode') {\n      return child.children.reduce((acc, child) => {\n        if (child.type === 'SentenceNode') {\n          return child.children.reduce((acc, child) => {\n            if (child.type === 'WordNode') {\n              const node = {\n                text: getWord(child),\n                start: child.position.start.offset,\n                end: child.position.end.offset,\n              };\n              return acc.concat([node]);\n            } else {\n              return acc;\n            }\n          }, acc);\n        } else {\n          return acc;\n        }\n      }, acc);\n    } else {\n      return acc;\n    }\n  }, []);\n\n  return out;\n};\n\nclass Intersection {\n  constructor(results) {\n    this.results = results;\n  }\n\n  get hasOverlap() {\n    return this.results.filter((r) => r.type === 'overlap').length > 0;\n  }\n\n  get surroundedTokens() {\n    return this.results.filter((r) => r.type === 'within-selection').map((t) => t.token);\n  }\n}\n/**\n * get intersection info for the selection in relation to tokens.\n * @param {{start: number, end: number}} selection\n * @param {{start: number, end: number}[]} tokens\n * @return {tokens: [], type: 'overlap|no-overlap|contains'}\n */\nexport const intersection = (selection, tokens) => {\n  const { start, end } = selection;\n\n  const startsWithin = (t) => start >= t.start && start < t.end;\n  const endsWithin = (t) => end > t.start && end <= t.end;\n\n  const mapped = tokens.map((t) => {\n    if (start === t.start && end === t.end) {\n      return { token: t, type: 'exact-fit' };\n    } else if (start <= t.start && end >= t.end) {\n      return { token: t, type: 'within-selection' };\n    } else if (startsWithin(t) || endsWithin(t)) {\n      return { token: t, type: 'overlap' };\n    }\n  });\n  return new Intersection(compact(mapped));\n};\n\nexport const sort = (tokens) => {\n  if (!Array.isArray(tokens)) {\n    return tokens;\n  } else {\n    const out = clone(tokens);\n    out.sort((a, b) => {\n      const s = a.start < b.start ? -1 : a.start > b.start ? 1 : 0;\n      const e = a.end < b.end ? -1 : a.end > b.end ? 1 : 0;\n      if (s === -1 && e !== -1) {\n        throw new Error(`sort does not support intersecting tokens. a: ${a.start}-${a.end}, b: ${b.start}-${b.end}`);\n      }\n      return s;\n    });\n    return out;\n  }\n};\n\nexport const normalize = (textToNormalize, tokens) => {\n  // making sure text provided is a string\n  const text = textToNormalize || '';\n\n  if (!Array.isArray(tokens) || tokens.length === 0) {\n    return [\n      {\n        text,\n        start: 0,\n        end: text.length,\n      },\n    ];\n  }\n\n  const out = sort(tokens).reduce(\n    (acc, t, index, outer) => {\n      let tokens = [];\n      const lastIndex = acc.lastIndex;\n\n      if (t.start === lastIndex) {\n        tokens = [\n          {\n            text: text.substring(lastIndex, t.end),\n            start: lastIndex,\n            end: t.end,\n            predefined: true,\n            correct: t.correct,\n            isMissing: t.isMissing,\n          },\n        ];\n      } else if (lastIndex < t.start) {\n        tokens = [\n          {\n            text: text.substring(lastIndex, t.start),\n            start: lastIndex,\n            end: t.start,\n          },\n          {\n            text: text.substring(t.start, t.end),\n            start: t.start,\n            end: t.end,\n            predefined: true,\n            correct: t.correct,\n            isMissing: t.isMissing,\n          },\n        ];\n      }\n\n      if (index === outer.length - 1 && t.end < text.length) {\n        const last = {\n          text: text.substring(t.end),\n          start: t.end,\n          end: text.length,\n        };\n        tokens.push(last);\n      }\n\n      return {\n        lastIndex: tokens.length ? tokens[tokens.length - 1].end : lastIndex,\n        result: acc.result.concat(tokens),\n      };\n    },\n    { result: [], lastIndex: 0 },\n  );\n\n  return out.result;\n};\n"],"file":"builder.js"}

package/lib/tokenizer/controls.js

@@ -1,123 +0,0 @@
-"use strict";
-
-var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports["default"] = exports.Controls = void 0;
-
-var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
-
-var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
-
-var _inherits2 = _interopRequireDefault(require("@babel/runtime/helpers/inherits"));
-
-var _possibleConstructorReturn2 = _interopRequireDefault(require("@babel/runtime/helpers/possibleConstructorReturn"));
-
-var _getPrototypeOf2 = _interopRequireDefault(require("@babel/runtime/helpers/getPrototypeOf"));
-
-var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
-
-var _react = _interopRequireDefault(require("react"));
-
-var _propTypes = _interopRequireDefault(require("prop-types"));
-
-var _Button = _interopRequireDefault(require("@material-ui/core/Button"));
-
-var _styles = require("@material-ui/core/styles");
-
-var _Switch = _interopRequireDefault(require("@material-ui/core/Switch"));
-
-var _FormControlLabel = _interopRequireDefault(require("@material-ui/core/FormControlLabel"));
-
-function _createSuper(Derived) { var hasNativeReflectConstruct = _isNativeReflectConstruct(); return function _createSuperInternal() { var Super = (0, _getPrototypeOf2["default"])(Derived), result; if (hasNativeReflectConstruct) { var NewTarget = (0, _getPrototypeOf2["default"])(this).constructor; result = Reflect.construct(Super, arguments, NewTarget); } else { result = Super.apply(this, arguments); } return (0, _possibleConstructorReturn2["default"])(this, result); }; }
-
-function _isNativeReflectConstruct() { if (typeof Reflect === "undefined" || !Reflect.construct) return false; if (Reflect.construct.sham) return false; if (typeof Proxy === "function") return true; try { Boolean.prototype.valueOf.call(Reflect.construct(Boolean, [], function () {})); return true; } catch (e) { return false; } }
-
-var Controls = /*#__PURE__*/function (_React$Component) {
-  (0, _inherits2["default"])(Controls, _React$Component);
-
-  var _super = _createSuper(Controls);
-
-  function Controls() {
-    (0, _classCallCheck2["default"])(this, Controls);
-    return _super.apply(this, arguments);
-  }
-
-  (0, _createClass2["default"])(Controls, [{
-    key: "render",
-    value: function render() {
-      var _this$props = this.props,
-          classes = _this$props.classes,
-          onClear = _this$props.onClear,
-          onWords = _this$props.onWords,
-          onSentences = _this$props.onSentences,
-          onParagraphs = _this$props.onParagraphs,
-          setCorrectMode = _this$props.setCorrectMode,
-          onToggleCorrectMode = _this$props.onToggleCorrectMode;
-      return /*#__PURE__*/_react["default"].createElement("div", {
-        className: classes.controls
-      }, /*#__PURE__*/_react["default"].createElement("div", null, /*#__PURE__*/_react["default"].createElement(_Button["default"], {
-        onClick: onWords,
-        className: classes.button,
-        size: "small",
-        color: "primary",
-        disabled: setCorrectMode
-      }, "Words"), /*#__PURE__*/_react["default"].createElement(_Button["default"], {
-        onClick: onSentences,
-        className: classes.button,
-        size: "small",
-        color: "primary",
-        disabled: setCorrectMode
-      }, "Sentences"), /*#__PURE__*/_react["default"].createElement(_Button["default"], {
-        onClick: onParagraphs,
-        className: classes.button,
-        size: "small",
-        color: "primary",
-        disabled: setCorrectMode
-      }, "Paragraphs"), /*#__PURE__*/_react["default"].createElement(_Button["default"], {
-        className: classes.button,
-        size: "small",
-        color: "secondary",
-        onClick: onClear,
-        disabled: setCorrectMode
-      }, "Clear")), /*#__PURE__*/_react["default"].createElement(_FormControlLabel["default"], {
-        control: /*#__PURE__*/_react["default"].createElement(_Switch["default"], {
-          checked: setCorrectMode,
-          onChange: onToggleCorrectMode
-        }),
-        label: "Set correct answers"
-      }));
-    }
-  }]);
-  return Controls;
-}(_react["default"].Component);
-
-exports.Controls = Controls;
-(0, _defineProperty2["default"])(Controls, "propTypes", {
-  classes: _propTypes["default"].object.isRequired,
-  onClear: _propTypes["default"].func.isRequired,
-  onWords: _propTypes["default"].func.isRequired,
-  onSentences: _propTypes["default"].func.isRequired,
-  onParagraphs: _propTypes["default"].func.isRequired,
-  setCorrectMode: _propTypes["default"].bool.isRequired,
-  onToggleCorrectMode: _propTypes["default"].func.isRequired
-});
-(0, _defineProperty2["default"])(Controls, "defaultProps", {});
-
-var _default = (0, _styles.withStyles)(function (theme) {
-  return {
-    button: {
-      marginRight: theme.spacing.unit
-    },
-    controls: {
-      display: 'flex',
-      alignItems: 'center',
-      justifyContent: 'space-between'
-    }
-  };
-})(Controls);
-
-exports["default"] = _default;
-//# sourceMappingURL=controls.js.map

package/lib/tokenizer/controls.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../src/tokenizer/controls.jsx"],"names":["Controls","props","classes","onClear","onWords","onSentences","onParagraphs","setCorrectMode","onToggleCorrectMode","controls","button","React","Component","PropTypes","object","isRequired","func","bool","theme","marginRight","spacing","unit","display","alignItems","justifyContent"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;;;IAEaA,Q;;;;;;;;;;;;WAaX,kBAAS;AACP,wBAAsG,KAAKC,KAA3G;AAAA,UAAQC,OAAR,eAAQA,OAAR;AAAA,UAAiBC,OAAjB,eAAiBA,OAAjB;AAAA,UAA0BC,OAA1B,eAA0BA,OAA1B;AAAA,UAAmCC,WAAnC,eAAmCA,WAAnC;AAAA,UAAgDC,YAAhD,eAAgDA,YAAhD;AAAA,UAA8DC,cAA9D,eAA8DA,cAA9D;AAAA,UAA8EC,mBAA9E,eAA8EA,mBAA9E;AAEA,0BACE;AAAK,QAAA,SAAS,EAAEN,OAAO,CAACO;AAAxB,sBACE,0DACE,gCAAC,kBAAD;AAAQ,QAAA,OAAO,EAAEL,OAAjB;AAA0B,QAAA,SAAS,EAAEF,OAAO,CAACQ,MAA7C;AAAqD,QAAA,IAAI,EAAC,OAA1D;AAAkE,QAAA,KAAK,EAAC,SAAxE;AAAkF,QAAA,QAAQ,EAAEH;AAA5F,iBADF,eAIE,gCAAC,kBAAD;AACE,QAAA,OAAO,EAAEF,WADX;AAEE,QAAA,SAAS,EAAEH,OAAO,CAACQ,MAFrB;AAGE,QAAA,IAAI,EAAC,OAHP;AAIE,QAAA,KAAK,EAAC,SAJR;AAKE,QAAA,QAAQ,EAAEH;AALZ,qBAJF,eAaE,gCAAC,kBAAD;AACE,QAAA,OAAO,EAAED,YADX;AAEE,QAAA,SAAS,EAAEJ,OAAO,CAACQ,MAFrB;AAGE,QAAA,IAAI,EAAC,OAHP;AAIE,QAAA,KAAK,EAAC,SAJR;AAKE,QAAA,QAAQ,EAAEH;AALZ,sBAbF,eAsBE,gCAAC,kBAAD;AAAQ,QAAA,SAAS,EAAEL,OAAO,CAACQ,MAA3B;AAAmC,QAAA,IAAI,EAAC,OAAxC;AAAgD,QAAA,KAAK,EAAC,WAAtD;AAAkE,QAAA,OAAO,EAAEP,OAA3E;AAAoF,QAAA,QAAQ,EAAEI;AAA9F,iBAtBF,CADF,eA2BE,gCAAC,4BAAD;AACE,QAAA,OAAO,eAAE,gCAAC,kBAAD;AAAQ,UAAA,OAAO,EAAEA,cAAjB;AAAiC,UAAA,QAAQ,EAAEC;AAA3C,UADX;AAEE,QAAA,KAAK,EAAC;AAFR,QA3BF,CADF;AAkCD;;;EAlD2BG,kBAAMC,S;;;iCAAvBZ,Q,eACQ;AACjBE,EAAAA,OAAO,EAAEW,sBAAUC,MAAV,CAAiBC,UADT;AAEjBZ,EAAAA,OAAO,EAAEU,sBAAUG,IAAV,CAAeD,UAFP;AAGjBX,EAAAA,OAAO,EAAES,sBAAUG,IAAV,CAAeD,UAHP;AAIjBV,EAAAA,WAAW,EAAEQ,sBAAUG,IAAV,CAAeD,UAJX;AAKjBT,EAAAA,YAAY,EAAEO,sBAAUG,IAAV,CAAeD,UALZ;AAMjBR,EAAAA,cAAc,EAAEM,sBAAUI,IAAV,CAAeF,UANd;AAOjBP,EAAAA,mBAAmB,EAAEK,sBAAUG,IAAV,CAAeD;AAPnB,C;iCADRf,Q,kBAWW,E;;eAyCT,wBAAW,UAACkB,KAAD;AAAA,SAAY;AACpCR,IAAAA,MAAM,EAAE;AACNS,MAAAA,WAAW,EAAED,KAAK,CAACE,OAAN,CAAcC;AADrB,KAD4B;AAIpCZ,IAAAA,QAAQ,EAAE;AACRa,MAAAA,OAAO,EAAE,MADD;AAERC,MAAAA,UAAU,EAAE,QAFJ;AAGRC,MAAAA,cAAc,EAAE;AAHR;AAJ0B,GAAZ;AAAA,CAAX,EASXxB,QATW,C","sourcesContent":["import React from 'react';\nimport PropTypes from 'prop-types';\nimport Button from '@material-ui/core/Button';\nimport { withStyles } from '@material-ui/core/styles';\nimport Switch from '@material-ui/core/Switch';\nimport FormControlLabel from '@material-ui/core/FormControlLabel';\n\nexport class Controls extends React.Component {\n  static propTypes = {\n    classes: PropTypes.object.isRequired,\n    onClear: PropTypes.func.isRequired,\n    onWords: PropTypes.func.isRequired,\n    onSentences: PropTypes.func.isRequired,\n    onParagraphs: PropTypes.func.isRequired,\n    setCorrectMode: PropTypes.bool.isRequired,\n    onToggleCorrectMode: PropTypes.func.isRequired,\n  };\n\n  static defaultProps = {};\n\n  render() {\n    const { classes, onClear, onWords, onSentences, onParagraphs, setCorrectMode, onToggleCorrectMode } = this.props;\n\n    return (\n      <div className={classes.controls}>\n        <div>\n          <Button onClick={onWords} className={classes.button} size=\"small\" color=\"primary\" disabled={setCorrectMode}>\n            Words\n          </Button>\n          <Button\n            onClick={onSentences}\n            className={classes.button}\n            size=\"small\"\n            color=\"primary\"\n            disabled={setCorrectMode}\n          >\n            Sentences\n          </Button>\n          <Button\n            onClick={onParagraphs}\n            className={classes.button}\n            size=\"small\"\n            color=\"primary\"\n            disabled={setCorrectMode}\n          >\n            Paragraphs\n          </Button>\n          <Button className={classes.button} size=\"small\" color=\"secondary\" onClick={onClear} disabled={setCorrectMode}>\n            Clear\n          </Button>\n        </div>\n        <FormControlLabel\n          control={<Switch checked={setCorrectMode} onChange={onToggleCorrectMode} />}\n          label=\"Set correct answers\"\n        />\n      </div>\n    );\n  }\n}\nexport default withStyles((theme) => ({\n  button: {\n    marginRight: theme.spacing.unit,\n  },\n  controls: {\n    display: 'flex',\n    alignItems: 'center',\n    justifyContent: 'space-between',\n  },\n}))(Controls);\n"],"file":"controls.js"}