@echogarden/text-segmentation 0.3.3 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (14) hide show
  1. package/dist/Patterns.js +1 -1
  2. package/dist/Patterns.js.map +1 -1
  3. package/dist/Test.js +25 -2
  4. package/dist/Test.js.map +1 -1
  5. package/dist/TextSegmentation.d.ts +1 -0
  6. package/dist/TextSegmentation.js +69 -37
  7. package/dist/TextSegmentation.js.map +1 -1
  8. package/dist/WordSequence.js +1 -1
  9. package/dist/WordSequence.js.map +1 -1
  10. package/package.json +2 -2
  11. package/src/Patterns.ts +1 -1
  12. package/src/Test.ts +31 -2
  13. package/src/TextSegmentation.ts +86 -41
  14. package/src/WordSequence.ts +1 -1
package/dist/Patterns.js CHANGED
@@ -184,7 +184,7 @@ const sentenceSeparatorCharacterPattern = [
184
184
  ];
185
185
  const sentenceSeparatorTrailingPunctuationPattern = [
186
186
  inputStart,
187
- anyOf('"', '”', '’', ')', ']', '}', '»', ...sentenceSeparatorCharacters, ...phraseSeparatorCharacters, oneOrMore(whitespace)),
187
+ anyOf('"', '”', '’', ')', ']', '}', '»', '」', ...sentenceSeparatorCharacters, ...phraseSeparatorCharacters, whitespace),
188
188
  inputEnd
189
189
  ];
190
190
  ////////////////////////////////////////////////////////////////////////////////////////////////
package/dist/Patterns.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Patterns.js","sourceRoot":"","sources":["../src/Patterns.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,eAAe,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAEtL,gGAAgG;AAChG,0BAA0B;AAC1B,gGAAgG;AAChG,MAAM,UAAU,wBAAwB,CAAC,YAAsB;IAC9D,OAAO,KAAK,CACX,uBAAuB,CAAC,YAAY,CAAC,EACrC,kBAAkB,CAClB,CAAA;AACF,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,YAAsB;IAC7D,MAAM,mBAAmB,GACxB,OAAO,CACN,KAAK,CAAC,GAAG,YAAY,CAAC,EACtB,EAAE,eAAe,EAAE,oBAAoB,EAAE,CACzC,CAAA;IAEF,OAAO,mBAAmB,CAAA;AAC3B,CAAC;AAED,gGAAgG;AAChG,4BAA4B;AAC5B,gGAAgG;AAChG,MAAM,aAAa,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAA;AAC/C,MAAM,WAAW,GAAG,eAAe,CAAC,MAAM,CAAC,CAAA;AAC3C,MAAM,YAAY,GAAG,eAAe,CAAC,gBAAgB,CAAC,CAAA;AACtD,MAAM,kBAAkB,GAAG,eAAe,CAAC,aAAa,CAAC,CAAA;AAEzD,MAAM,gBAAgB,GAAG,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;AAC7C,MAAM,oBAAoB,GAAG,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;AAEhD,MAAM,mBAAmB,GAAG,KAAK,CAChC,aAAa,EACb,WAAW,CACX,CAAA;AAED,MAAM,0BAA0B,GAAG,KAAK,CACvC,aAAa,EACb,WAAW,EACX,YAAY,CACZ,CAAA;AAED,kEAAkE;AAClE,MAAM,YAAY,GAAG,KAAK,CACzB,CAAC,eAAe,CAAC,qBAAqB,CAAC,EAAE,eAAe,CAAC,gBAAgB,CAAC,CAAC,EAC3E,eAAe,CAAC,oBAAoB,CAAC,EACrC,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAC7C,CAAA;AAED,MAAM,oBAAoB,GAAG,CAAC,GAAG,CAAC,CAAA;AAClC,MAAM,kBAAkB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;AAE3G,MAAM,oCAAoC,GAAG,KAAK,CAAC,GAAG,oBAAoB,EAAE,GAAG,kBAAkB,CAAC,CAAA;AAElG,gGAAgG;AAChG,mBAAmB;AACnB,gGAAgG;AAChG,MAAM,uBAAuB,GAC5B,OAAO,CACN,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IAC3B,YAAY,EAAE,oBAAoB;IAClC,YAAY,EAAE,oBAAoB;CAClC,CAAC,CAAA;AAEH,MAAM,iBAAiB,GAAG,OAAO,CAAC;IACjC,SAAS,CAAC,oBAAoB,CAAC;IAE/B,SAAS,CAAC;QACT,GAAG;QACH,SAAS,CAAC,oBAAoB,CAAC;KAC/B,CAAC;CACF,EAAE;IACF,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,UAAU,CAAC;IAC/D,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,QAAQ,CAAC;CAC7D,CAAC,CAAA;AAEF,MAAM,+BAA+B,GACpC,OAAO,CACN,GAAG,EAAE;IACL,YAAY,EAAE,oBAAoB;IAClC,YAAY,EAAE;QACb,QAAQ,CAAC,CAAC,EAAE,oBAAoB,CAAC;QACjC,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,QAAQ,CAAC;KAC/C;CACD,CAAC,CAAA;AAEH,MAAM,kBAAkB,GACvB,OAAO,CACN,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE;IACjB,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,UAAU,CAAC;IAC/D,YAAY,EAAE,oBAAoB;CAClC,CAAC,CAAA;AAEH,MAAM,aAAa,GAAG;IACrB,QAAQ,CAAC,kBAAkB,CAAC;IAC5B,YAAY;IAEZ,UAAU,CAAC,KAAK,CACf,YAAY,EACZ,uBAAuB,EACvB,+BAA+B,CAC/B,CAAC;CACF,CAAA;AAED,MAAM,eAAe,GAAG;IACvB,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC;IACf,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACzB,SAAS,CAAC,oBAAoB,CAAC;CAC/B,CAAA;AAED,MAAM,gDAAgD,GAAG,OAAO,CAAC;IAChE,aAAa;IAEb,QAAQ,CAAC,KAAK,CACb,eAAe,EAEf,OAAO,CACN,UAAU,CAAC,KAAK,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE;QACnD,eAAe,EAAE,YAAY;KAC7B,CAAC,CACF,CAAC;CAAC,EAAE;IAEL,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,UAAU,CAAC;IAC/D,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,QAAQ,CAAC;CAC7D,CAAC,CAAA;AAGF,MAAM,oCAAoC,GACzC,OAAO,CAAC;IACP,oCAAoC;IACpC,aAAa;CACb,EAAE;IACF,eAAe,EAAE,YAAY;IAC7B,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,QAAQ,CAAC;CAC7D,CAAC,CAAA;AAEH,MAAM,oCAAoC,GACzC,OAAO,CAAC;IACP,aAAa;IACb,oCAAoC;CACpC,EAAE;IACF,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,UAAU,CAAC;IAC/D,eAAe,EAAE,KAAK,CAAC,YAAY,CAAC;CACpC,CAAC,CAAA;AAEH,MAAM,2BAA2B,GAAG,KAAK,CACxC,oCAAoC,EACpC,oCAAoC,CACpC,CAAA;AAED,gGAAgG;AAChG,gBAAgB;AAChB,gGAAgG;AAChG,MAAM,WAAW,GAAG,OAAO,CAAC;IAC3B,KAAK,CAAC,oBAAoB,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,oBAAoB,CAAC,EAAE,CAAC,GAAG,EAAE,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;IAEpG,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;QAChB,GAAG;QACH,KAAK,CAAC,oBAAoB,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,oBAAoB,CAAC,CAAC;KACxE,CAAC;CACF,EAAE;IACF,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,UAAU,CAAC;IAC/D,eAAe,EAAE,GAAG;IACpB,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;IACrF,eAAe,EAAE,GAAG;CACpB,CAAC,CAAA;AAEF,gGAAgG;AAChG,kBAAkB;AAClB,gGAAgG;AAChG,MAAM,iCAAiC,GACtC,OAAO,CACN,KAAK,CACJ;IACC,aAAa;IAEb,SAAS,CAAC;QACT,IAAI;QACJ,aAAa;KACb,CAAC;IAEF,QAAQ,CAAC,GAAG,CAAC;CACb,EACD;IACC,aAAa;IAEb,SAAS,CAAC;QACT,GAAG;QACH,QAAQ,CAAC,GAAG,CAAC;QACb,aAAa;KACb,CAAC;IAEF,QAAQ,CAAC,GAAG,CAAC;CACb,CACD,EAAE;IACH,eAAe,EAAE,KAAK,CAAC,mBAAmB,EAAE,YAAY,CAAC;IACzD,eAAe,EAAE,KAAK,CAAC,mBAAmB,EAAE,YAAY,CAAC;CACzD,CAAC,CAAA;AAEH,MAAM,oBAAoB,GACzB,KAAK,CACJ,aAAa,EACb,WAAW,EACX,YAAY,CACZ,CAAA;AAEF,MAAM,yBAAyB,GAC9B,OAAO,CACN,gBAAgB,EAAE;IAElB,YAAY,EAAE,0BAA0B;IACxC,YAAY,EAAE,mBAAmB;CACjC,CAAC,CAAA;AAEH,MAAM,yBAAyB,GAC9B,OAAO,CACN,gBAAgB,EAAE;IAElB,YAAY,EAAE,UAAU;IACxB,YAAY,EAAE,CAAC,mBAAmB,EAAE,mBAAmB,EAAE,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;CACrF,CAAC,CAAA;AAEH,MAAM,gBAAgB,GACrB,SAAS,CACR,KAAK,CACJ,oBAAoB,EACpB,yBAAyB,EACzB,GAAG,CACH,CACD,CAAA;AAEF,MAAM,qBAAqB,GAAG;IAC7B,gBAAgB;IAEhB,SAAS,CAAC;QACT,GAAG;QACH,gBAAgB;KAChB,CAAC;CACF,CAAA;AAED,MAAM,uBAAuB,GAAG,OAAO,CAAC;IACvC,gBAAgB;IAEhB,SAAS,CAAC;QACT,GAAG;QACH,gBAAgB;KAChB,CAAC;CACF,EAAE;AACF,8CAA8C;AAC9C,2CAA2C;CAC3C,CAAC,CAAA;AAEF,MAAM,8BAA8B,GAAG;IACtC,gBAAgB;IAEhB,SAAS,CAAC;QACT,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC;QACf,gBAAgB;KAChB,CAAC;CACF,CAAA;AAED,MAAM,kBAAkB,GAAG,KAAK,CAC/B,YAAY,EAEZ,WAAW,EACX,2BAA2B,EAE3B,uBAAuB,EACvB,gDAAgD,EAEhD,8BAA8B,EAC9B,iCAAiC,EACjC,gBAAgB,CAChB,CAAA;AAED,gGAAgG;AAChG,6BAA6B;AAC7B,gGAAgG;AAChG,MAAM,yBAAyB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;AAE/E,MAAM,sBAAsB,GAAG;IAC9B,UAAU;IACV,KAAK,CAAC,GAAG,yBAAyB,CAAC;IACnC,QAAQ;CACR,CAAA;AAED,MAAM,yCAAyC,GAAG;IACjD,UAAU;IACV,KAAK,CAAC,GAAG,yBAAyB,EAAE,GAAG,EAAE,GAAG,CAAC;IAC7C,QAAQ;CACR,CAAA;AAED,gGAAgG;AAChG,+BAA+B;AAC/B,gGAAgG;AAChG,MAAM,2BAA2B,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,CAAA;AAExE,MAAM,iCAAiC,GAAG;IACzC,UAAU;IACV,KAAK,CAAC,GAAG,2BAA2B,CAAC;IACrC,QAAQ;CACR,CAAA;AAED,MAAM,2CAA2C,GAAG;IACnD,UAAU;IACV,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,2BAA2B,EAAE,GAAG,yBAAyB,EAAE,SAAS,CAAC,UAAU,CAAC,CAAC;IAC7H,QAAQ;CACR,CAAA;AAED,gGAAgG;AAChG,iBAAiB;AACjB,gGAAgG;AAEhG,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,UAAU,EAAE,SAAS,CAAC,UAAU,CAAC,EAAE,QAAQ,CAAC,CAAA;AAC9E,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,UAAU,EAAE,UAAU,CAAC,CAAA;AACnE,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;AAE/D,gGAAgG;AAChG,+BAA+B;AAC/B,gGAAgG;AAChG,MAAM,CAAC,MAAM,mBAAmB,GAAG,WAAW,CAAC,oBAAoB,CAAC,CAAA;AACpE,MAAM,CAAC,MAAM,uBAAuB,GAAG,WAAW,CAAC,iBAAiB,CAAC,CAAA;AAErE,MAAM,CAAC,MAAM,yBAAyB,GAAG,WAAW,CAAC,aAAa,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;AAErF,MAAM,CAAC,MAAM,qBAAqB,GAAG,WAAW,CAAC,sBAAsB,CAAC,CAAA;AACxE,MAAM,CAAC,MAAM,wCAAwC,GAAG,WAAW,CAAC,yCAAyC,CAAC,CAAA;AAE9G,MAAM,CAAC,MAAM,gCAAgC,GAAG,WAAW,CAAC,iCAAiC,CAAC,CAAA;AAC9F,MAAM,CAAC,MAAM,mDAAmD,GAAG,WAAW,CAAC,2CAA2C,CAAC,CAAA;AAE3H,MAAM,CAAC,MAAM,0BAA0B,GAAG,WAAW,CAAC,2BAA2B,CAAC,CAAA;AAClF,MAAM,CAAC,MAAM,wBAAwB,GAAG,WAAW,CAAC,yBAAyB,CAAC,CAAA"}
1
+ {"version":3,"file":"Patterns.js","sourceRoot":"","sources":["../src/Patterns.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,eAAe,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAEtL,gGAAgG;AAChG,0BAA0B;AAC1B,gGAAgG;AAChG,MAAM,UAAU,wBAAwB,CAAC,YAAsB;IAC9D,OAAO,KAAK,CACX,uBAAuB,CAAC,YAAY,CAAC,EACrC,kBAAkB,CAClB,CAAA;AACF,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,YAAsB;IAC7D,MAAM,mBAAmB,GACxB,OAAO,CACN,KAAK,CAAC,GAAG,YAAY,CAAC,EACtB,EAAE,eAAe,EAAE,oBAAoB,EAAE,CACzC,CAAA;IAEF,OAAO,mBAAmB,CAAA;AAC3B,CAAC;AAED,gGAAgG;AAChG,4BAA4B;AAC5B,gGAAgG;AAChG,MAAM,aAAa,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAA;AAC/C,MAAM,WAAW,GAAG,eAAe,CAAC,MAAM,CAAC,CAAA;AAC3C,MAAM,YAAY,GAAG,eAAe,CAAC,gBAAgB,CAAC,CAAA;AACtD,MAAM,kBAAkB,GAAG,eAAe,CAAC,aAAa,CAAC,CAAA;AAEzD,MAAM,gBAAgB,GAAG,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;AAC7C,MAAM,oBAAoB,GAAG,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;AAEhD,MAAM,mBAAmB,GAAG,KAAK,CAChC,aAAa,EACb,WAAW,CACX,CAAA;AAED,MAAM,0BAA0B,GAAG,KAAK,CACvC,aAAa,EACb,WAAW,EACX,YAAY,CACZ,CAAA;AAED,kEAAkE;AAClE,MAAM,YAAY,GAAG,KAAK,CACzB,CAAC,eAAe,CAAC,qBAAqB,CAAC,EAAE,eAAe,CAAC,gBAAgB,CAAC,CAAC,EAC3E,eAAe,CAAC,oBAAoB,CAAC,EACrC,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAC7C,CAAA;AAED,MAAM,oBAAoB,GAAG,CAAC,GAAG,CAAC,CAAA;AAClC,MAAM,kBAAkB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;AAE3G,MAAM,oCAAoC,GAAG,KAAK,CAAC,GAAG,oBAAoB,EAAE,GAAG,kBAAkB,CAAC,CAAA;AAElG,gGAAgG;AAChG,mBAAmB;AACnB,gGAAgG;AAChG,MAAM,uBAAuB,GAC5B,OAAO,CACN,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IAC3B,YAAY,EAAE,oBAAoB;IAClC,YAAY,EAAE,oBAAoB;CAClC,CAAC,CAAA;AAEH,MAAM,iBAAiB,GAAG,OAAO,CAAC;IACjC,SAAS,CAAC,oBAAoB,CAAC;IAE/B,SAAS,CAAC;QACT,GAAG;QACH,SAAS,CAAC,oBAAoB,CAAC;KAC/B,CAAC;CACF,EAAE;IACF,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,UAAU,CAAC;IAC/D,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,QAAQ,CAAC;CAC7D,CAAC,CAAA;AAEF,MAAM,+BAA+B,GACpC,OAAO,CACN,GAAG,EAAE;IACL,YAAY,EAAE,oBAAoB;IAClC,YAAY,EAAE;QACb,QAAQ,CAAC,CAAC,EAAE,oBAAoB,CAAC;QACjC,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,QAAQ,CAAC;KAC/C;CACD,CAAC,CAAA;AAEH,MAAM,kBAAkB,GACvB,OAAO,CACN,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE;IACjB,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,UAAU,CAAC;IAC/D,YAAY,EAAE,oBAAoB;CAClC,CAAC,CAAA;AAEH,MAAM,aAAa,GAAG;IACrB,QAAQ,CAAC,kBAAkB,CAAC;IAC5B,YAAY;IAEZ,UAAU,CAAC,KAAK,CACf,YAAY,EACZ,uBAAuB,EACvB,+BAA+B,CAC/B,CAAC;CACF,CAAA;AAED,MAAM,eAAe,GAAG;IACvB,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC;IACf,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACzB,SAAS,CAAC,oBAAoB,CAAC;CAC/B,CAAA;AAED,MAAM,gDAAgD,GAAG,OAAO,CAAC;IAChE,aAAa;IAEb,QAAQ,CAAC,KAAK,CACb,eAAe,EAEf,OAAO,CACN,UAAU,CAAC,KAAK,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE;QACnD,eAAe,EAAE,YAAY;KAC7B,CAAC,CACF,CAAC;CAAC,EAAE;IAEL,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,UAAU,CAAC;IAC/D,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,QAAQ,CAAC;CAC7D,CAAC,CAAA;AAGF,MAAM,oCAAoC,GACzC,OAAO,CAAC;IACP,oCAAoC;IACpC,aAAa;CACb,EAAE;IACF,eAAe,EAAE,YAAY;IAC7B,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,QAAQ,CAAC;CAC7D,CAAC,CAAA;AAEH,MAAM,oCAAoC,GACzC,OAAO,CAAC;IACP,aAAa;IACb,oCAAoC;CACpC,EAAE;IACF,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,UAAU,CAAC;IAC/D,eAAe,EAAE,KAAK,CAAC,YAAY,CAAC;CACpC,CAAC,CAAA;AAEH,MAAM,2BAA2B,GAAG,KAAK,CACxC,oCAAoC,EACpC,oCAAoC,CACpC,CAAA;AAED,gGAAgG;AAChG,gBAAgB;AAChB,gGAAgG;AAChG,MAAM,WAAW,GAAG,OAAO,CAAC;IAC3B,KAAK,CAAC,oBAAoB,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,oBAAoB,CAAC,EAAE,CAAC,GAAG,EAAE,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;IAEpG,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;QAChB,GAAG;QACH,KAAK,CAAC,oBAAoB,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,oBAAoB,CAAC,CAAC;KACxE,CAAC;CACF,EAAE;IACF,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,UAAU,CAAC;IAC/D,eAAe,EAAE,GAAG;IACpB,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;IACrF,eAAe,EAAE,GAAG;CACpB,CAAC,CAAA;AAEF,gGAAgG;AAChG,kBAAkB;AAClB,gGAAgG;AAChG,MAAM,iCAAiC,GACtC,OAAO,CACN,KAAK,CACJ;IACC,aAAa;IAEb,SAAS,CAAC;QACT,IAAI;QACJ,aAAa;KACb,CAAC;IAEF,QAAQ,CAAC,GAAG,CAAC;CACb,EACD;IACC,aAAa;IAEb,SAAS,CAAC;QACT,GAAG;QACH,QAAQ,CAAC,GAAG,CAAC;QACb,aAAa;KACb,CAAC;IAEF,QAAQ,CAAC,GAAG,CAAC;CACb,CACD,EAAE;IACH,eAAe,EAAE,KAAK,CAAC,mBAAmB,EAAE,YAAY,CAAC;IACzD,eAAe,EAAE,KAAK,CAAC,mBAAmB,EAAE,YAAY,CAAC;CACzD,CAAC,CAAA;AAEH,MAAM,oBAAoB,GACzB,KAAK,CACJ,aAAa,EACb,WAAW,EACX,YAAY,CACZ,CAAA;AAEF,MAAM,yBAAyB,GAC9B,OAAO,CACN,gBAAgB,EAAE;IAElB,YAAY,EAAE,0BAA0B;IACxC,YAAY,EAAE,mBAAmB;CACjC,CAAC,CAAA;AAEH,MAAM,yBAAyB,GAC9B,OAAO,CACN,gBAAgB,EAAE;IAElB,YAAY,EAAE,UAAU;IACxB,YAAY,EAAE,CAAC,mBAAmB,EAAE,mBAAmB,EAAE,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;CACrF,CAAC,CAAA;AAEH,MAAM,gBAAgB,GACrB,SAAS,CACR,KAAK,CACJ,oBAAoB,EACpB,yBAAyB,EACzB,GAAG,CACH,CACD,CAAA;AAEF,MAAM,qBAAqB,GAAG;IAC7B,gBAAgB;IAEhB,SAAS,CAAC;QACT,GAAG;QACH,gBAAgB;KAChB,CAAC;CACF,CAAA;AAED,MAAM,uBAAuB,GAAG,OAAO,CAAC;IACvC,gBAAgB;IAEhB,SAAS,CAAC;QACT,GAAG;QACH,gBAAgB;KAChB,CAAC;CACF,EAAE;AACF,8CAA8C;AAC9C,2CAA2C;CAC3C,CAAC,CAAA;AAEF,MAAM,8BAA8B,GAAG;IACtC,gBAAgB;IAEhB,SAAS,CAAC;QACT,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC;QACf,gBAAgB;KAChB,CAAC;CACF,CAAA;AAED,MAAM,kBAAkB,GAAG,KAAK,CAC/B,YAAY,EAEZ,WAAW,EACX,2BAA2B,EAE3B,uBAAuB,EACvB,gDAAgD,EAEhD,8BAA8B,EAC9B,iCAAiC,EACjC,gBAAgB,CAChB,CAAA;AAED,gGAAgG;AAChG,6BAA6B;AAC7B,gGAAgG;AAChG,MAAM,yBAAyB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;AAE/E,MAAM,sBAAsB,GAAG;IAC9B,UAAU;IACV,KAAK,CAAC,GAAG,yBAAyB,CAAC;IACnC,QAAQ;CACR,CAAA;AAED,MAAM,yCAAyC,GAAG;IACjD,UAAU;IACV,KAAK,CAAC,GAAG,yBAAyB,EAAE,GAAG,EAAE,GAAG,CAAC;IAC7C,QAAQ;CACR,CAAA;AAED,gGAAgG;AAChG,+BAA+B;AAC/B,gGAAgG;AAChG,MAAM,2BAA2B,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,CAAA;AAExE,MAAM,iCAAiC,GAAG;IACzC,UAAU;IACV,KAAK,CAAC,GAAG,2BAA2B,CAAC;IACrC,QAAQ;CACR,CAAA;AAED,MAAM,2CAA2C,GAAG;IACnD,UAAU;IACV,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,2BAA2B,EAAE,GAAG,yBAAyB,EAAE,UAAU,CAAC;IACvH,QAAQ;CACR,CAAA;AAED,gGAAgG;AAChG,iBAAiB;AACjB,gGAAgG;AAEhG,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,UAAU,EAAE,SAAS,CAAC,UAAU,CAAC,EAAE,QAAQ,CAAC,CAAA;AAC9E,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,UAAU,EAAE,UAAU,CAAC,CAAA;AACnE,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;AAE/D,gGAAgG;AAChG,+BAA+B;AAC/B,gGAAgG;AAChG,MAAM,CAAC,MAAM,mBAAmB,GAAG,WAAW,CAAC,oBAAoB,CAAC,CAAA;AACpE,MAAM,CAAC,MAAM,uBAAuB,GAAG,WAAW,CAAC,iBAAiB,CAAC,CAAA;AAErE,MAAM,CAAC,MAAM,yBAAyB,GAAG,WAAW,CAAC,aAAa,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;AAErF,MAAM,CAAC,MAAM,qBAAqB,GAAG,WAAW,CAAC,sBAAsB,CAAC,CAAA;AACxE,MAAM,CAAC,MAAM,wCAAwC,GAAG,WAAW,CAAC,yCAAyC,CAAC,CAAA;AAE9G,MAAM,CAAC,MAAM,gCAAgC,GAAG,WAAW,CAAC,iCAAiC,CAAC,CAAA;AAC9F,MAAM,CAAC,MAAM,mDAAmD,GAAG,WAAW,CAAC,2CAA2C,CAAC,CAAA;AAE3H,MAAM,CAAC,MAAM,0BAA0B,GAAG,WAAW,CAAC,2BAA2B,CAAC,CAAA;AAClF,MAAM,CAAC,MAAM,wBAAwB,GAAG,WAAW,CAAC,yBAAyB,CAAC,CAAA"}
package/dist/Test.js CHANGED
@@ -15,7 +15,7 @@ async function test1() {
15
15
  }
16
16
  async function test2() {
17
17
  const { readFileSync, writeFileSync } = await import('fs');
18
- const text = readFileSync('test-data/Test2.txt', 'utf-8');
18
+ const text = readFileSync('test-data/Japanese3.txt', 'utf-8');
19
19
  const timer = new Timer();
20
20
  let result;
21
21
  for (let i = 0; i < 5; i++) {
@@ -87,6 +87,29 @@ async function test4() {
87
87
  const result = (await splitToWords(text)).wordArray;
88
88
  log(result.join(' | '));
89
89
  }
90
+ async function test5() {
91
+ const text1 = `
92
+ ただ
93
+
94
+ ◆
95
+
96
+ 街の
97
+ `;
98
+ const text2 = `
99
+ Hello World. Yo!
100
+
101
+
102
+
103
+ gooooogoo
104
+
105
+ , Hi. BOBO.
106
+ `;
107
+ const wordSequence = await splitToWords(text1);
108
+ const result = await segmentWordSequence(wordSequence);
109
+ const sentencesText = result.sentences.map(x => x.text);
110
+ log(sentencesText.join(' \n '));
111
+ }
90
112
  //test1()
91
- test2();
113
+ //test2()
114
+ test5();
92
115
  //# sourceMappingURL=Test.js.map
package/dist/Test.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Test.js","sourceRoot":"","sources":["../src/Test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wCAAwC,EAA2C,WAAW,EAAE,mBAAmB,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AACvL,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAE5C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAA;AAEvB,KAAK,UAAU,KAAK;IACnB,+GAA+G;IAC/G,MAAM,IAAI,GAAG,8CAA8C,CAAA;IAC3D,iEAAiE;IAEjE,MAAM,OAAO,GAAwB;QACpC,QAAQ,EAAE,IAAI;KACd,CAAA;IAED,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;IAEtD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAA;IAEnD,MAAM,aAAa,GAAG,MAAM,mBAAmB,CAAC,YAAY,CAAC,CAAA;IAE7D,MAAM,CAAC,GAAG,CAAC,CAAA;AACZ,CAAC;AAED,KAAK,UAAU,KAAK;IACnB,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAA;IAC1D,MAAM,IAAI,GAAG,YAAY,CAAC,qBAAqB,EAAE,OAAO,CAAC,CAAA;IAEzD,MAAM,KAAK,GAAG,IAAI,KAAK,EAAE,CAAA;IAEzB,IAAI,MAA0B,CAAA;IAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,MAAM,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE;YAChC,QAAQ,EAAE,IAAI;YACd,kBAAkB,EAAE,EAAE;YACtB,6BAA6B,EAAE,IAAI;SACnC,CAAC,CAAA;QAEF,KAAK,CAAC,aAAa,CAAC,sBAAsB,CAAC,CAAA;IAC5C,CAAC;IAED,GAAG,CAAC,EAAE,CAAC,CAAA;IAEP,EAAE;IAEF,IAAI,aAAa,GAAG,EAAE,CAAA;IAEtB,KAAK,IAAI,aAAa,GAAG,CAAC,EAAE,aAAa,GAAG,MAAO,CAAC,SAAS,CAAC,MAAM,EAAE,aAAa,EAAE,EAAE,CAAC;QACvF,MAAM,QAAQ,GAAG,MAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAA;QACjD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAA;QAEhC,KAAK,IAAI,WAAW,GAAG,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC;YACvE,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAA;YAEnC,aAAa,IAAI,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAEnD,IAAI,WAAW,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtC,aAAa,IAAI,KAAK,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAA;YAC1C,CAAC;QACF,CAAC;QAED,IAAI,aAAa,GAAG,MAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClD,aAAa,IAAI,KAAK,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAA;QAC3C,CAAC;IACF,CAAC;IAED,aAAa,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAA;AAClD,CAAC;AAED,KAAK,UAAU,KAAK;IACnB,MAAM,IAAI,GAAG,mDAAmD,CAAA;IAEhE,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAA;IAEtC,MAAM,yBAAyB,GAAG,KAAK,CAAC,qBAAqB,CAAA;IAC7D,MAAM,0BAA0B,GAAG,IAAI,YAAY,EAAE,CAAA;IACrD,0BAA0B,CAAC,OAAO,GAAG,yBAAyB,CAAA;IAE9D,MAAM,aAAa,GAAG,wCAAwC,CAAC,0BAA0B,EAAE,IAAI,CAAC,CAAA;IAEhG,MAAM,CAAC,GAAG,CAAC,CAAA;AACZ,CAAC;AAED,KAAK,UAAU,KAAK;IACnB,MAAM,IAAI,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA4Bb,CAAA;IAEA,MAAM,MAAM,GAAG,CAAC,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAA;IACnD,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAA;AAExB,CAAC;AAED,SAAS;AACT,KAAK,EAAE,CAAA"}
1
+ {"version":3,"file":"Test.js","sourceRoot":"","sources":["../src/Test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wCAAwC,EAA2C,WAAW,EAAE,mBAAmB,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AACvL,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAE5C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAA;AAEvB,KAAK,UAAU,KAAK;IACnB,+GAA+G;IAC/G,MAAM,IAAI,GAAG,8CAA8C,CAAA;IAC3D,iEAAiE;IAEjE,MAAM,OAAO,GAAwB;QACpC,QAAQ,EAAE,IAAI;KACd,CAAA;IAED,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;IAEtD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAA;IAEnD,MAAM,aAAa,GAAG,MAAM,mBAAmB,CAAC,YAAY,CAAC,CAAA;IAE7D,MAAM,CAAC,GAAG,CAAC,CAAA;AACZ,CAAC;AAED,KAAK,UAAU,KAAK;IACnB,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAA;IAC1D,MAAM,IAAI,GAAG,YAAY,CAAC,yBAAyB,EAAE,OAAO,CAAC,CAAA;IAE7D,MAAM,KAAK,GAAG,IAAI,KAAK,EAAE,CAAA;IAEzB,IAAI,MAA0B,CAAA;IAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,MAAM,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE;YAChC,QAAQ,EAAE,IAAI;YACd,kBAAkB,EAAE,EAAE;YACtB,6BAA6B,EAAE,IAAI;SACnC,CAAC,CAAA;QAEF,KAAK,CAAC,aAAa,CAAC,sBAAsB,CAAC,CAAA;IAC5C,CAAC;IAED,GAAG,CAAC,EAAE,CAAC,CAAA;IAEP,EAAE;IAEF,IAAI,aAAa,GAAG,EAAE,CAAA;IAEtB,KAAK,IAAI,aAAa,GAAG,CAAC,EAAE,aAAa,GAAG,MAAO,CAAC,SAAS,CAAC,MAAM,EAAE,aAAa,EAAE,EAAE,CAAC;QACvF,MAAM,QAAQ,GAAG,MAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAA;QACjD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAA;QAEhC,KAAK,IAAI,WAAW,GAAG,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC;YACvE,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAA;YAEnC,aAAa,IAAI,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAEnD,IAAI,WAAW,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtC,aAAa,IAAI,KAAK,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAA;YAC1C,CAAC;QACF,CAAC;QAED,IAAI,aAAa,GAAG,MAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClD,aAAa,IAAI,KAAK,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAA;QAC3C,CAAC;IACF,CAAC;IAED,aAAa,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAA;AAClD,CAAC;AAED,KAAK,UAAU,KAAK;IACnB,MAAM,IAAI,GAAG,mDAAmD,CAAA;IAEhE,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAA;IAEtC,MAAM,yBAAyB,GAAG,KAAK,CAAC,qBAAqB,CAAA;IAC7D,MAAM,0BAA0B,GAAG,IAAI,YAAY,EAAE,CAAA;IACrD,0BAA0B,CAAC,OAAO,GAAG,yBAAyB,CAAA;IAE9D,MAAM,aAAa,GAAG,wCAAwC,CAAC,0BAA0B,EAAE,IAAI,CAAC,CAAA;IAEhG,MAAM,CAAC,GAAG,CAAC,CAAA;AACZ,CAAC;AAED,KAAK,UAAU,KAAK;IACnB,MAAM,IAAI,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA4Bb,CAAA;IAEA,MAAM,MAAM,GAAG,CAAC,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAA;IACnD,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAA;AACxB,CAAC;AAGD,KAAK,UAAU,KAAK;IACnB,MAAM,KAAK,GAAG;;;;;;CAMd,CAAA;IAEA,MAAM,KAAK,GAAG;;;;;;;;CAQd,CAAA;IAEA,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,KAAK,CAAC,CAAA;IAE9C,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,YAAY,CAAC,CAAA;IAEtD,MAAM,aAAa,GAAG,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;IAEvD,GAAG,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA;AAChC,CAAC;AAED,SAAS;AACT,SAAS;AACT,KAAK,EAAE,CAAA"}
package/dist/TextSegmentation.d.ts CHANGED
@@ -10,6 +10,7 @@ export declare function addMissingPunctuationWordsToWordSequence(wordSequence: W
10
10
  export interface SegmentationResult {
11
11
  words: WordSequence;
12
12
  sentences: Sentence[];
13
+ segmentSentenceRanges: Range[];
13
14
  }
14
15
  export declare class TextFragment {
15
16
  wordRange: Range;
package/dist/TextSegmentation.js CHANGED
@@ -13,52 +13,80 @@ export async function segmentText(text, options) {
13
13
  return segmentWordSequence(wordSequence);
14
14
  }
15
15
  export async function segmentWordSequence(wordSequence) {
16
- const sentenceWordRanges = [];
17
- const minimumSentenceLetterCount = 2;
18
- let sentenceStartWordOffset = 0;
19
- let currentSentenceLetterCount = 0;
20
- for (let wordIndex = 0; wordIndex < wordSequence.length; wordIndex++) {
21
- const word = wordSequence.getWordAt(wordIndex);
22
- if (currentSentenceLetterCount < minimumSentenceLetterCount) {
23
- const matches = word.matchAll(letterPatternGlobalRegExp);
24
- for (const _ of matches) {
25
- currentSentenceLetterCount += 1;
26
- if (currentSentenceLetterCount >= minimumSentenceLetterCount) {
27
- break;
16
+ const segmentWordRanges = [];
17
+ {
18
+ let segmentStartWordOffset = 0;
19
+ let nonWhitespaceWordSeen = false;
20
+ for (let wordIndex = 0; wordIndex < wordSequence.length; wordIndex++) {
21
+ const word = wordSequence.getWordAt(wordIndex);
22
+ if (word.endsWith('\n') && nonWhitespaceWordSeen) {
23
+ const nextWord = wordIndex < wordSequence.length - 1 ? wordSequence.getWordAt(wordIndex + 1) : '';
24
+ if (nextWord !== '\r' && nextWord !== '\n') {
25
+ segmentWordRanges.push({ start: segmentStartWordOffset, end: wordIndex + 1 });
26
+ segmentStartWordOffset = wordIndex + 1;
28
27
  }
29
28
  }
29
+ if (nonWhitespaceWordSeen === false) {
30
+ nonWhitespaceWordSeen = !whitespacePatternRegExp.test(word);
31
+ }
32
+ }
33
+ if (segmentStartWordOffset < wordSequence.length) {
34
+ segmentWordRanges.push({ start: segmentStartWordOffset, end: wordSequence.length });
30
35
  }
31
- if (currentSentenceLetterCount >= minimumSentenceLetterCount && sentenceSeparatorCharacterRegExp.test(word)) {
32
- let trailingSequenceEndIndex = wordIndex;
33
- let trailingSequenceContainedWhitespace = false;
34
- while (trailingSequenceEndIndex < wordSequence.length) {
35
- const trailingWord = wordSequence.getWordAt(trailingSequenceEndIndex);
36
- if (sentenceSeparatorTrailingPunctuationCharacterRegExp.test(trailingWord)) {
37
- if (!trailingSequenceContainedWhitespace && whitespacePatternRegExp.test(trailingWord)) {
38
- trailingSequenceContainedWhitespace = true;
36
+ }
37
+ const sentenceWordRanges = [];
38
+ const segmentSentenceRanges = [];
39
+ {
40
+ const minimumSentenceLetterCount = 2;
41
+ for (let segmentIndex = 0; segmentIndex < segmentWordRanges.length; segmentIndex++) {
42
+ const segmentWordRange = segmentWordRanges[segmentIndex];
43
+ const segmentStartWordIndex = segmentWordRange.start;
44
+ const segmentEndWordIndex = segmentWordRange.end;
45
+ segmentSentenceRanges.push({ start: sentenceWordRanges.length, end: sentenceWordRanges.length });
46
+ let sentenceStartWordOffset = segmentStartWordIndex;
47
+ let currentSentenceLetterCount = 0;
48
+ for (let wordIndex = segmentStartWordIndex; wordIndex < segmentEndWordIndex; wordIndex++) {
49
+ const word = wordSequence.getWordAt(wordIndex);
50
+ if (currentSentenceLetterCount < minimumSentenceLetterCount) {
51
+ const matches = word.matchAll(letterPatternGlobalRegExp);
52
+ for (const _ of matches) {
53
+ currentSentenceLetterCount += 1;
54
+ if (currentSentenceLetterCount >= minimumSentenceLetterCount) {
55
+ break;
56
+ }
39
57
  }
40
- trailingSequenceEndIndex++;
41
58
  }
42
- else {
43
- break;
59
+ if (currentSentenceLetterCount >= minimumSentenceLetterCount && sentenceSeparatorCharacterRegExp.test(word)) {
60
+ let trailingSequenceEndIndex = wordIndex;
61
+ let trailingSequenceContainedWhitespace = false;
62
+ while (trailingSequenceEndIndex < segmentEndWordIndex) {
63
+ const trailingWord = wordSequence.getWordAt(trailingSequenceEndIndex);
64
+ if (sentenceSeparatorTrailingPunctuationCharacterRegExp.test(trailingWord)) {
65
+ if (!trailingSequenceContainedWhitespace && whitespacePatternRegExp.test(trailingWord)) {
66
+ trailingSequenceContainedWhitespace = true;
67
+ }
68
+ trailingSequenceEndIndex++;
69
+ }
70
+ else {
71
+ break;
72
+ }
73
+ }
74
+ sentenceWordRanges.push({
75
+ start: sentenceStartWordOffset,
76
+ end: trailingSequenceEndIndex
77
+ });
78
+ segmentSentenceRanges[segmentSentenceRanges.length - 1].end += 1;
79
+ sentenceStartWordOffset = trailingSequenceEndIndex;
80
+ currentSentenceLetterCount = 0;
81
+ wordIndex = trailingSequenceEndIndex - 1;
44
82
  }
45
83
  }
46
- if (trailingSequenceEndIndex === wordSequence.length ||
47
- trailingSequenceContainedWhitespace ||
48
- ['。', '？', '！'].includes(word)) {
49
- sentenceWordRanges.push({
50
- start: sentenceStartWordOffset,
51
- end: trailingSequenceEndIndex
52
- });
53
- sentenceStartWordOffset = trailingSequenceEndIndex;
54
- currentSentenceLetterCount = 0;
55
- wordIndex = trailingSequenceEndIndex - 1;
84
+ if (sentenceStartWordOffset < segmentEndWordIndex) {
85
+ sentenceWordRanges.push({ start: sentenceStartWordOffset, end: segmentEndWordIndex });
86
+ segmentSentenceRanges[segmentSentenceRanges.length - 1].end += 1;
56
87
  }
57
88
  }
58
89
  }
59
- if (sentenceStartWordOffset < wordSequence.length) {
60
- sentenceWordRanges.push({ start: sentenceStartWordOffset, end: wordSequence.length });
61
- }
62
90
  const sentences = [];
63
91
  for (const wordRange of sentenceWordRanges) {
64
92
  const sentenceWordSequence = wordSequence.slice(wordRange.start, wordRange.end);
@@ -70,7 +98,10 @@ export async function segmentWordSequence(wordSequence) {
70
98
  let phraseStartWordOffset = sentence.wordRange.start;
71
99
  for (let wordIndex = phraseStartWordOffset; wordIndex < sentenceEndWordOffset; wordIndex++) {
72
100
  const currentWord = wordSequence.getWordAt(wordIndex);
73
- if (phraseSeparatorRegExp.test(currentWord)) {
101
+ const isCurrentWordPhraseSeparator = phraseSeparatorRegExp.test(currentWord) &&
102
+ ((currentWord !== ':' && currentWord !== ';') ||
103
+ whitespacePatternRegExp.test(wordSequence.getWordAt(wordIndex + 1)));
104
+ if (isCurrentWordPhraseSeparator) {
74
105
  let whitespaceSeenOnce = false;
75
106
  while (wordIndex < sentenceEndWordOffset - 1) {
76
107
  const nextWord = wordSequence.getWordAt(wordIndex + 1);
@@ -103,6 +134,7 @@ export async function segmentWordSequence(wordSequence) {
103
134
  const result = {
104
135
  words: wordSequence,
105
136
  sentences,
137
+ segmentSentenceRanges,
106
138
  };
107
139
  return result;
108
140
  }
package/dist/TextSegmentation.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"TextSegmentation.js","sourceRoot":"","sources":["../src/TextSegmentation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wBAAwB,IAAI,wBAAwB,EAAE,qBAAqB,EAAE,mDAAmD,EAAE,gCAAgC,EAAE,uBAAuB,EAAE,yBAAyB,EAA0D,MAAM,eAAe,CAAA;AAC9S,OAAO,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,wCAAwC,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AACzJ,OAAO,EAAE,yBAAyB,EAAE,MAAM,iCAAiC,CAAA;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAChD,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAA;AAC/D,OAAO,EAAE,YAAY,EAAkB,MAAM,mBAAmB,CAAA;AAEhE,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAE7C,gGAAgG;AAChG,mBAAmB;AACnB,gGAAgG;AAChG,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAY,EAAE,OAA6B;IAC5E,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;IAEtD,OAAO,mBAAmB,CAAC,YAAY,CAAC,CAAA;AACzC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,YAA0B;IACnE,MAAM,kBAAkB,GAAY,EAAE,CAAA;IAEtC,MAAM,0BAA0B,GAAG,CAAC,CAAA;IAEpC,IAAI,uBAAuB,GAAG,CAAC,CAAA;IAC/B,IAAI,0BAA0B,GAAG,CAAC,CAAA;IAElC,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;QAE9C,IAAI,0BAA0B,GAAG,0BAA0B,EAAE,CAAC;YAC7D,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,yBAAyB,CAAC,CAAA;YAExD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;gBACzB,0BAA0B,IAAI,CAAC,CAAA;gBAE/B,IAAI,0BAA0B,IAAI,0BAA0B,EAAE,CAAC;oBAC9D,MAAK;gBACN,CAAC;YACF,CAAC;QACF,CAAC;QAED,IAAI,0BAA0B,IAAI,0BAA0B,IAAI,gCAAgC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7G,IAAI,wBAAwB,GAAG,SAAS,CAAA;YACxC,IAAI,mCAAmC,GAAG,KAAK,CAAA;YAE/C,OAAO,wBAAwB,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC;gBACvD,MAAM,YAAY,GAAG,YAAY,CAAC,SAAS,CAAC,wBAAwB,CAAC,CAAA;gBAErE,IAAI,mDAAmD,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;oBAC5E,IAAI,CAAC,mCAAmC,IAAI,uBAAuB,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;wBACxF,mCAAmC,GAAG,IAAI,CAAA;oBAC3C,CAAC;oBAED,wBAAwB,EAAE,CAAA;gBAC3B,CAAC;qBAAM,CAAC;oBACP,MAAK;gBACN,CAAC;YACF,CAAC;YAED,IAAI,wBAAwB,KAAK,YAAY,CAAC,MAAM;gBACnD,mCAAmC;gBACnC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBAEjC,kBAAkB,CAAC,IAAI,CAAC;oBACvB,KAAK,EAAE,uBAAuB;oBAC9B,GAAG,EAAE,wBAAwB;iBAC7B,CAAC,CAAA;gBAEF,uBAAuB,GAAG,wBAAwB,CAAA;gBAClD,0BAA0B,GAAG,CAAC,CAAA;gBAE9B,SAAS,GAAG,wBAAwB,GAAG,CAAC,CAAA;YACzC,CAAC;QACF,CAAC;IACF,CAAC;IAED,IAAI,uBAAuB,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC;QACnD,kBAAkB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,uBAAuB,EAAE,GAAG,EAAE,YAAY,CAAC,MAAM,EAAE,CAAC,CAAA;IACtF,CAAC;IAED,MAAM,SAAS,GAAe,EAAE,CAAA;IAEhC,KAAK,MAAM,SAAS,IAAI,kBAAkB,EAAE,CAAC;QAC5C,MAAM,oBAAoB,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,CAAA;QAE/E,SAAS,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC,CAAA;IAC9D,CAAC;IAED,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QAClC,MAAM,gBAAgB,GAAY,EAAE,CAAA;QAEpC,IAAI,qBAAqB,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAA;QAClD,IAAI,qBAAqB,GAAG,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAA;QAEpD,KAAK,IAAI,SAAS,GAAG,qBAAqB,EAAE,SAAS,GAAG,qBAAqB,EAAE,SAAS,EAAE,EAAE,CAAC;YAC5F,MAAM,WAAW,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;YAErD,IAAI,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC7C,IAAI,kBAAkB,GAAG,KAAK,CAAA;gBAE9B,OAAO,SAAS,GAAG,qBAAqB,GAAG,CAAC,EAAE,CAAC;oBAC9C,MAAM,QAAQ,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC,CAAA;oBAEtD,IAAI,CAAC,mDAAmD,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;wBACzE,MAAK;oBACN,CAAC;oBAED,IAAI,uBAAuB,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAC5C,kBAAkB,GAAG,IAAI,CAAA;oBAC1B,CAAC;oBAED,IAAI,QAAQ,KAAK,GAAG,IAAI,kBAAkB,EAAE,CAAC;wBAC5C,MAAK;oBACN,CAAC;oBAED,SAAS,IAAI,CAAC,CAAA;gBACf,CAAC;gBAED,gBAAgB,CAAC,IAAI,CAAC;oBACrB,KAAK,EAAE,qBAAqB;oBAC5B,GAAG,EAAE,SAAS,GAAG,CAAC;iBAClB,CAAC,CAAA;gBAEF,qBAAqB,GAAG,SAAS,GAAG,CAAC,CAAA;YACtC,CAAC;QACF,CAAC;QAED,IAAI,qBAAqB,GAAG,qBAAqB,EAAE,CAAC;YACnD,gBAAgB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,qBAAqB,EAAE,GAAG,EAAE,qBAAqB,EAAE,CAAC,CAAA;QACpF,CAAC;QAED,KAAK,MAAM,SAAS,IAAI,gBAAgB,EAAE,CAAC;YAC1C,MAAM,kBAAkB,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,CAAA;YAE7E,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC,CAAA;QACjE,CAAC;IACF,CAAC;IAED,MAAM,MAAM,GAAuB;QAClC,KAAK,EAAE,YAAY;QACnB,SAAS;KACT,CAAA;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED,MAAM,yBAAyB,GAAG,IAAI,GAAG,EAAkB,CAAA;AAE3D,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,OAA6B;IAC7E,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,GAAG,EAAE,CAAA;IACb,CAAC;IAED,OAAO,GAAG,EAAE,GAAG,0BAA0B,EAAE,GAAG,OAAO,EAAE,CAAA;IAEvD,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACtB,OAAO,CAAC,QAAQ,GAAG,oBAAoB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;IAC1D,CAAC;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA;IAE7C,IAAI,kBAAkB,GAAG,yBAAyB,CAAC,GAAG,CAAC,aAAa,CAAC,CAAA;IAErE,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACzB,kBAAkB,GAAG,iCAAiC,CAAC,OAAO,CAAC,CAAA;QAE/D,yBAAyB,CAAC,GAAG,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAAA;IACjE,CAAC;IAED,IAAI,YAAY,GAAG,IAAI,YAAY,EAAE,CAAA;IAErC,SAAS,0BAA0B,CAAC,WAAmB,EAAE,SAAiB;QACzE,MAAM,wBAAwB,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,CAAA;QAEvE,IAAI,UAAU,GAAG,WAAW,CAAA;QAC5B,IAAI,0BAA0B,GAAG,WAAW,CAAA;QAE5C,SAAS,0BAA0B;YAClC,IAAI,UAAU,GAAG,0BAA0B,EAAE,CAAC;gBAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,0BAA0B,EAAE,UAAU,CAAC,CAAA;gBACvE,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,0BAA0B,EAAE,IAAI,CAAC,CAAA;gBAEhE,0BAA0B,GAAG,UAAU,CAAA;YACxC,CAAC;QACF,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,wBAAwB,EAAE,CAAC;YAC7C,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;gBAClB,UAAU,IAAI,CAAC,CAAA;gBAEf,SAAQ;YACT,CAAC;YAED,0BAA0B,EAAE,CAAA;YAE5B,UAAU,IAAI,IAAI,CAAC,MAAM,CAAA;YAEzB,0BAA0B,EAAE,CAAA;QAC7B,CAAC;QAED,0BAA0B,EAAE,CAAA;IAC7B,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAA;IAErD,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAE1B,IAAI,WAAW,EAAE,CAAC;QACjB,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAQ,CAAC,CAAC,CAAC,CAAA;YACjC,MAAM,gBAAgB,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YACnC,MAAM,cAAc,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YAEjC,IAAI,gBAAgB,GAAG,kBAAkB,EAAE,CAAC;gBAC3C,0BAA0B,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAA;YACjE,CAAC;YAED,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,gBAAgB,EAAE,cAAc,CAAC,CAAA;YACjE,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAA;YAEvD,kBAAkB,GAAG,cAAc,CAAA;QACpC,CAAC;QAED,0BAA0B,CAAC,kBAAkB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IAC5D,CAAC;IAED,IAAI,OAAO,CAAC,6BAA6B,EAAE,CAAC;QAC3C,YAAY,GAAG,MAAM,yBAAyB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;IACnE,CAAC;IAED,OAAO,YAAY,CAAA;AACpB,CAAC;AAED,KAAK,UAAU,yBAAyB,CAAC,cAAsB,EAAE,YAA0B;IAC1F,MAAM,eAAe,GAAG,MAAM,kBAAkB,EAAE,CAAA;IAElD,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;QACnC,OAAO,YAAY,CAAA;IACpB,CAAC;IAED,IAAI,cAAc,GAAG,KAAK,CAAA;IAE1B,MAAM,eAAe,GAAG,IAAI,YAAY,EAAE,CAAA;IAE1C,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;QACjD,MAAM,eAAe,GAAG,SAAS,CAAC,WAAW,CAAA;QAE7C,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;QAE9C,IAAI,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1C,IAAI,CAAC,cAAc,EAAE,CAAC;gBACrB,MAAM,eAAe,CAAC,UAAU,EAAE,CAAA;gBAElC,cAAc,GAAG,IAAI,CAAA;YACtB,CAAC;YAED,MAAM,UAAU,GAAG,CAAC,GAAG,eAAe,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAA;YAErE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChD,MAAM,kBAAkB,GAAG,eAAe,GAAG,UAAU,CAAC,CAAC,CAAC,CAAA;gBAC1D,MAAM,gBAAgB,GAAG,eAAe,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;gBAE5D,MAAM,WAAW,GAAG,cAAc,CAAC,SAAS,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAA;gBAElF,eAAe,CAAC,OAAO,CACtB,WAAW,EACX,kBAAkB,EAClB,KAAK,CACL,CAAA;YACF,CAAC;QACF,CAAC;aAAM,CAAC;YACP,MAAM,QAAQ,GAAG,cAAc,CAAC,SAAS,CAAC,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,SAAS,CAAC,CAAA;YAErF,eAAe,CAAC,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,aAAa,CAAC,CAAA;QAClF,CAAC;IACF,CAAC;IAED,OAAO,eAAe,CAAA;AACvB,CAAC;AAED,uDAAuD;AACvD,MAAM,UAAU,wCAAwC,CAAC,YAA0B,EAAE,UAAkB;IACtG,MAAM,2BAA2B,GAAG,IAAI,GAAG,EAAkB,CAAA;IAE7D,MAAM,2BAA2B,GAAG,IAAI,YAAY,EAAE,CAAA;IAEtD,SAAS,0BAA0B,CAAC,SAAiB,EAAE,iBAAyB;QAC/E,IAAI,UAAU,GAAG,iBAAiB,CAAA;QAElC,kFAAkF;QAClF,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC9B,MAAM,aAAa,GAAG,UAAU,GAAG,IAAI,CAAC,MAAM,CAAA;YAE9C,MAAM,SAAS,GAAG,2BAA2B,CAAC,SAAS,CAAA;YAEvD,IAAI,IAAI,KAAK,GAAG,IAAI,SAAS,IAAI,SAAS,CAAC,aAAa,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;gBACvF,2BAA2B,CAAC,SAAS,CAAC,IAAI,IAAI,GAAG,CAAA;gBACjD,2BAA2B,CAAC,SAAS,CAAC,SAAS,GAAG,aAAa,CAAA;YAChE,CAAC;iBAAM,CAAC;gBACP,MAAM,QAAQ,GAAG,UAAU,CAAC,SAAS,CAAC,UAAU,EAAE,aAAa,CAAC,CAAA;gBAEhE,2BAA2B,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,EAAE,IAAI,CAAC,CAAA;YAChE,CAAC;YAED,UAAU,GAAG,aAAa,CAAA;QAC3B,CAAC;IACF,CAAC;IAED,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,MAAM,SAAS,GAAG,YAAY,CAAC,UAAU,CAAC,SAAS,CAAC,CAAA;QACpD,MAAM,eAAe,GAAG,SAAS,CAAC,WAAW,CAAA;QAE7C,MAAM,qBAAqB,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAA;QAE/F,8FAA8F;QAC9F,IAAI,qBAAqB,KAAK,eAAe,EAAE,CAAC;YAC/C,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,qBAAqB,EAAE,eAAe,CAAC,CAAA;YAE9E,0BAA0B,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAA;QAC7D,CAAC;QAED,2BAA2B,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QACnD,2BAA2B,CAAC,GAAG,CAAC,2BAA2B,CAAC,MAAM,GAAG,CAAC,EAAE,SAAS,CAAC,CAAA;QAElF,oEAAoE;QACpE,IAAI,SAAS,KAAK,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3C,IAAI,UAAU,CAAC,MAAM,KAAK,SAAS,CAAC,SAAS,EAAE,CAAC;gBAC/C,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,SAAS,CAAC,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC,CAAA;gBAE9E,0BAA0B,CAAC,SAAS,EAAE,SAAS,CAAC,SAAS,CAAC,CAAA;YAC3D,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,EAAE,2BAA2B,EAAE,2BAA2B,EAAE,CAAA;AACpE,CAAC;AAED,SAAS,oBAAoB,CAAC,YAA0B,EAAE,IAAY;IACrE,MAAM,iBAAiB,GAAY,EAAE,CAAA;IACrC,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAA;IAExC,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;QACpC,iBAAiB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAA;IACtE,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,MAAM,KAAK,GAAG,WAAW,CAAC,CAAC,CAAC,CAAA;QAE5B,MAAM,iBAAiB,GAAG,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,IAAI,CAAC,CAAA;QAE5D,IAAI,KAAK,CAAC,WAAW,GAAG,iBAAiB,EAAE,CAAC;YAC3C,iBAAiB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,iBAAiB,EAAE,GAAG,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAA;QAC7E,CAAC;QAED,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;YACzB,iBAAiB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE,GAAG,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAA;QAC3E,CAAC;IACF,CAAC;IAED,CAAC;QACA,MAAM,aAAa,GAAG,WAAW,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,SAAS,CAAA;QAEpE,IAAI,aAAa,IAAI,aAAa,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAClD,iBAAiB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,aAAa,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAA;QACnE,CAAC;IACF,CAAC;IAED,OAAO,iBAAiB,CAAA;AACzB,CAAC;AAED,gGAAgG;AAChG,iBAAiB;AACjB,gGAAgG;AAChG,SAAS,iCAAiC,CAAC,OAA4B;IACtE,MAAM,uBAAuB,GAAG,gBAAgB,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IAC9E,MAAM,2BAA2B,GAAG,sBAAsB,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IACxF,MAAM,8BAA8B,GAAG,wCAAwC,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IAC7G,MAAM,6CAA6C,GAAG,8BAA8B,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAA;IACzH,MAAM,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,EAAE,CAAA;IAE3D,IAAI,YAAY,GAAG;QAClB,GAAG,kBAAkB;QACrB,GAAG,uBAAuB;QAC1B,GAAG,2BAA2B;QAC9B,GAAG,8BAA8B;QACjC,GAAG,6CAA6C;QAChD,GAAG,gBAAgB;QACnB,GAAG,eAAe;KAClB,CAAA;IAED,MAAM,WAAW,GAAG,wBAAwB,CAAC;QAC5C,GAAG,YAAY;QACf,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACrD,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;KACrD,CAAC,CAAA;IAEF,MAAM,kBAAkB,GAAG,WAAW,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;IAErE,OAAO,kBAAkB,CAAA;AAC1B,CAAC;AAED,KAAK,UAAU,kBAAkB;IAChC,IAAI,CAAC;QACJ,MAAM,eAAe,GAAG,MAAM,MAAM,CAAC,mCAAmC,CAAC,CAAA;QAEzE,OAAO,eAAe,CAAA;IACvB,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAA;IACjB,CAAC;AACF,CAAC;AAUD,MAAM,OAAO,YAAY;IACxB,SAAS,CAAO;IAChB,KAAK,CAAc;IAEnB,YAAY,SAAgB,EAAE,KAAmB;QAChD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;QAC1B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAA;IACnB,CAAC;IAED,IAAI,IAAI;QACP,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAA;IACvB,CAAC;IAED,IAAI,SAAS;QACZ,OAAO;YACN,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,WAAW;YACxC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,SAAS;SACnC,CAAA;IACF,CAAC;CACD;AAED,MAAM,OAAO,QAAS,SAAQ,YAAY;IACzC,OAAO,GAAa,EAAE,CAAA;CACtB;AAED,MAAM,OAAO,MAAO,SAAQ,YAAY;CACvC;AAaD,MAAM,CAAC,MAAM,0BAA0B,GAAwB;IAC9D,QAAQ,EAAE,EAAE;IACZ,kBAAkB,EAAE,EAAE;IACtB,6BAA6B,EAAE,IAAI;CACnC,CAAA"}
1
+ {"version":3,"file":"TextSegmentation.js","sourceRoot":"","sources":["../src/TextSegmentation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wBAAwB,IAAI,wBAAwB,EAAE,qBAAqB,EAAE,mDAAmD,EAAE,gCAAgC,EAAE,uBAAuB,EAAE,yBAAyB,EAA0D,MAAM,eAAe,CAAA;AAC9S,OAAO,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,wCAAwC,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AACzJ,OAAO,EAAE,yBAAyB,EAAE,MAAM,iCAAiC,CAAA;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAChD,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAA;AAC/D,OAAO,EAAE,YAAY,EAAkB,MAAM,mBAAmB,CAAA;AAEhE,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAE7C,gGAAgG;AAChG,mBAAmB;AACnB,gGAAgG;AAChG,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAY,EAAE,OAA6B;IAC5E,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;IAEtD,OAAO,mBAAmB,CAAC,YAAY,CAAC,CAAA;AACzC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,YAA0B;IACnE,MAAM,iBAAiB,GAAY,EAAE,CAAA;IAErC,CAAC;QACA,IAAI,sBAAsB,GAAG,CAAC,CAAA;QAC9B,IAAI,qBAAqB,GAAG,KAAK,CAAA;QAEjC,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;YACtE,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;YAE9C,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,qBAAqB,EAAE,CAAC;gBAClD,MAAM,QAAQ,GAAG,SAAS,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;gBAEjG,IAAI,QAAQ,KAAK,IAAI,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;oBAC5C,iBAAiB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,sBAAsB,EAAE,GAAG,EAAE,SAAS,GAAG,CAAC,EAAE,CAAC,CAAA;oBAC7E,sBAAsB,GAAG,SAAS,GAAG,CAAC,CAAA;gBACvC,CAAC;YACF,CAAC;YAED,IAAI,qBAAqB,KAAK,KAAK,EAAE,CAAC;gBACrC,qBAAqB,GAAG,CAAC,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAC5D,CAAC;QACF,CAAC;QAED,IAAI,sBAAsB,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC;YAClD,iBAAiB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,sBAAsB,EAAE,GAAG,EAAE,YAAY,CAAC,MAAM,EAAE,CAAC,CAAA;QACpF,CAAC;IACF,CAAC;IAED,MAAM,kBAAkB,GAAY,EAAE,CAAA;IACtC,MAAM,qBAAqB,GAAY,EAAE,CAAA;IAEzC,CAAC;QACA,MAAM,0BAA0B,GAAG,CAAC,CAAA;QAEpC,KAAK,IAAI,YAAY,GAAG,CAAC,EAAE,YAAY,GAAG,iBAAiB,CAAC,MAAM,EAAE,YAAY,EAAE,EAAE,CAAC;YACpF,MAAM,gBAAgB,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAA;YAExD,MAAM,qBAAqB,GAAG,gBAAgB,CAAC,KAAK,CAAA;YACpD,MAAM,mBAAmB,GAAG,gBAAgB,CAAC,GAAG,CAAA;YAEhD,qBAAqB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,kBAAkB,CAAC,MAAM,EAAE,GAAG,EAAE,kBAAkB,CAAC,MAAM,EAAE,CAAC,CAAA;YAEhG,IAAI,uBAAuB,GAAG,qBAAqB,CAAA;YACnD,IAAI,0BAA0B,GAAG,CAAC,CAAA;YAElC,KAAK,IAAI,SAAS,GAAG,qBAAqB,EAAE,SAAS,GAAG,mBAAmB,EAAE,SAAS,EAAE,EAAE,CAAC;gBAC1F,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;gBAE9C,IAAI,0BAA0B,GAAG,0BAA0B,EAAE,CAAC;oBAC7D,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,yBAAyB,CAAC,CAAA;oBAExD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;wBACzB,0BAA0B,IAAI,CAAC,CAAA;wBAE/B,IAAI,0BAA0B,IAAI,0BAA0B,EAAE,CAAC;4BAC9D,MAAK;wBACN,CAAC;oBACF,CAAC;gBACF,CAAC;gBAED,IAAI,0BAA0B,IAAI,0BAA0B,IAAI,gCAAgC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC7G,IAAI,wBAAwB,GAAG,SAAS,CAAA;oBACxC,IAAI,mCAAmC,GAAG,KAAK,CAAA;oBAE/C,OAAO,wBAAwB,GAAG,mBAAmB,EAAE,CAAC;wBACvD,MAAM,YAAY,GAAG,YAAY,CAAC,SAAS,CAAC,wBAAwB,CAAC,CAAA;wBAErE,IAAI,mDAAmD,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;4BAC5E,IAAI,CAAC,mCAAmC,IAAI,uBAAuB,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;gCACxF,mCAAmC,GAAG,IAAI,CAAA;4BAC3C,CAAC;4BAED,wBAAwB,EAAE,CAAA;wBAC3B,CAAC;6BAAM,CAAC;4BACP,MAAK;wBACN,CAAC;oBACF,CAAC;oBAED,kBAAkB,CAAC,IAAI,CAAC;wBACvB,KAAK,EAAE,uBAAuB;wBAC9B,GAAG,EAAE,wBAAwB;qBAC7B,CAAC,CAAA;oBAEF,qBAAqB,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAA;oBAEhE,uBAAuB,GAAG,wBAAwB,CAAA;oBAClD,0BAA0B,GAAG,CAAC,CAAA;oBAE9B,SAAS,GAAG,wBAAwB,GAAG,CAAC,CAAA;gBACzC,CAAC;YACF,CAAC;YAED,IAAI,uBAAuB,GAAG,mBAAmB,EAAE,CAAC;gBACnD,kBAAkB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,uBAAuB,EAAE,GAAG,EAAE,mBAAmB,EAAE,CAAC,CAAA;gBACrF,qBAAqB,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAA;YACjE,CAAC;QACF,CAAC;IACF,CAAC;IAED,MAAM,SAAS,GAAe,EAAE,CAAA;IAEhC,KAAK,MAAM,SAAS,IAAI,kBAAkB,EAAE,CAAC;QAC5C,MAAM,oBAAoB,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,CAAA;QAE/E,SAAS,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC,CAAA;IAC9D,CAAC;IAED,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QAClC,MAAM,gBAAgB,GAAY,EAAE,CAAA;QAEpC,IAAI,qBAAqB,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAA;QAClD,IAAI,qBAAqB,GAAG,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAA;QAEpD,KAAK,IAAI,SAAS,GAAG,qBAAqB,EAAE,SAAS,GAAG,qBAAqB,EAAE,SAAS,EAAE,EAAE,CAAC;YAC5F,MAAM,WAAW,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;YAErD,MAAM,4BAA4B,GACjC,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC;gBACvC,CAAC,CAAC,WAAW,KAAK,GAAG,IAAI,WAAW,KAAK,GAAG,CAAC;oBAC7C,uBAAuB,CAAC,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;YAErE,IAAI,4BAA4B,EAAE,CAAC;gBAClC,IAAI,kBAAkB,GAAG,KAAK,CAAA;gBAE9B,OAAO,SAAS,GAAG,qBAAqB,GAAG,CAAC,EAAE,CAAC;oBAC9C,MAAM,QAAQ,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC,CAAA;oBAEtD,IAAI,CAAC,mDAAmD,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;wBACzE,MAAK;oBACN,CAAC;oBAED,IAAI,uBAAuB,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAC5C,kBAAkB,GAAG,IAAI,CAAA;oBAC1B,CAAC;oBAED,IAAI,QAAQ,KAAK,GAAG,IAAI,kBAAkB,EAAE,CAAC;wBAC5C,MAAK;oBACN,CAAC;oBAED,SAAS,IAAI,CAAC,CAAA;gBACf,CAAC;gBAED,gBAAgB,CAAC,IAAI,CAAC;oBACrB,KAAK,EAAE,qBAAqB;oBAC5B,GAAG,EAAE,SAAS,GAAG,CAAC;iBAClB,CAAC,CAAA;gBAEF,qBAAqB,GAAG,SAAS,GAAG,CAAC,CAAA;YACtC,CAAC;QACF,CAAC;QAED,IAAI,qBAAqB,GAAG,qBAAqB,EAAE,CAAC;YACnD,gBAAgB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,qBAAqB,EAAE,GAAG,EAAE,qBAAqB,EAAE,CAAC,CAAA;QACpF,CAAC;QAED,KAAK,MAAM,SAAS,IAAI,gBAAgB,EAAE,CAAC;YAC1C,MAAM,kBAAkB,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,CAAA;YAE7E,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC,CAAA;QACjE,CAAC;IACF,CAAC;IAED,MAAM,MAAM,GAAuB;QAClC,KAAK,EAAE,YAAY;QACnB,SAAS;QACT,qBAAqB;KACrB,CAAA;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED,MAAM,yBAAyB,GAAG,IAAI,GAAG,EAAkB,CAAA;AAE3D,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,OAA6B;IAC7E,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,GAAG,EAAE,CAAA;IACb,CAAC;IAED,OAAO,GAAG,EAAE,GAAG,0BAA0B,EAAE,GAAG,OAAO,EAAE,CAAA;IAEvD,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACtB,OAAO,CAAC,QAAQ,GAAG,oBAAoB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;IAC1D,CAAC;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA;IAE7C,IAAI,kBAAkB,GAAG,yBAAyB,CAAC,GAAG,CAAC,aAAa,CAAC,CAAA;IAErE,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACzB,kBAAkB,GAAG,iCAAiC,CAAC,OAAO,CAAC,CAAA;QAE/D,yBAAyB,CAAC,GAAG,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAAA;IACjE,CAAC;IAED,IAAI,YAAY,GAAG,IAAI,YAAY,EAAE,CAAA;IAErC,SAAS,0BAA0B,CAAC,WAAmB,EAAE,SAAiB;QACzE,MAAM,wBAAwB,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,CAAA;QAEvE,IAAI,UAAU,GAAG,WAAW,CAAA;QAC5B,IAAI,0BAA0B,GAAG,WAAW,CAAA;QAE5C,SAAS,0BAA0B;YAClC,IAAI,UAAU,GAAG,0BAA0B,EAAE,CAAC;gBAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,0BAA0B,EAAE,UAAU,CAAC,CAAA;gBACvE,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,0BAA0B,EAAE,IAAI,CAAC,CAAA;gBAEhE,0BAA0B,GAAG,UAAU,CAAA;YACxC,CAAC;QACF,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,wBAAwB,EAAE,CAAC;YAC7C,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;gBAClB,UAAU,IAAI,CAAC,CAAA;gBAEf,SAAQ;YACT,CAAC;YAED,0BAA0B,EAAE,CAAA;YAE5B,UAAU,IAAI,IAAI,CAAC,MAAM,CAAA;YAEzB,0BAA0B,EAAE,CAAA;QAC7B,CAAC;QAED,0BAA0B,EAAE,CAAA;IAC7B,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAA;IAErD,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAE1B,IAAI,WAAW,EAAE,CAAC;QACjB,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAQ,CAAC,CAAC,CAAC,CAAA;YACjC,MAAM,gBAAgB,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YACnC,MAAM,cAAc,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YAEjC,IAAI,gBAAgB,GAAG,kBAAkB,EAAE,CAAC;gBAC3C,0BAA0B,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAA;YACjE,CAAC;YAED,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,gBAAgB,EAAE,cAAc,CAAC,CAAA;YACjE,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAA;YAEvD,kBAAkB,GAAG,cAAc,CAAA;QACpC,CAAC;QAED,0BAA0B,CAAC,kBAAkB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IAC5D,CAAC;IAED,IAAI,OAAO,CAAC,6BAA6B,EAAE,CAAC;QAC3C,YAAY,GAAG,MAAM,yBAAyB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;IACnE,CAAC;IAED,OAAO,YAAY,CAAA;AACpB,CAAC;AAED,KAAK,UAAU,yBAAyB,CAAC,cAAsB,EAAE,YAA0B;IAC1F,MAAM,eAAe,GAAG,MAAM,kBAAkB,EAAE,CAAA;IAElD,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;QACnC,OAAO,YAAY,CAAA;IACpB,CAAC;IAED,IAAI,cAAc,GAAG,KAAK,CAAA;IAE1B,MAAM,eAAe,GAAG,IAAI,YAAY,EAAE,CAAA;IAE1C,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;QACjD,MAAM,eAAe,GAAG,SAAS,CAAC,WAAW,CAAA;QAE7C,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;QAE9C,IAAI,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1C,IAAI,CAAC,cAAc,EAAE,CAAC;gBACrB,MAAM,eAAe,CAAC,UAAU,EAAE,CAAA;gBAElC,cAAc,GAAG,IAAI,CAAA;YACtB,CAAC;YAED,MAAM,UAAU,GAAG,CAAC,GAAG,eAAe,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAA;YAErE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChD,MAAM,kBAAkB,GAAG,eAAe,GAAG,UAAU,CAAC,CAAC,CAAC,CAAA;gBAC1D,MAAM,gBAAgB,GAAG,eAAe,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;gBAE5D,MAAM,WAAW,GAAG,cAAc,CAAC,SAAS,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAA;gBAElF,eAAe,CAAC,OAAO,CACtB,WAAW,EACX,kBAAkB,EAClB,KAAK,CACL,CAAA;YACF,CAAC;QACF,CAAC;aAAM,CAAC;YACP,MAAM,QAAQ,GAAG,cAAc,CAAC,SAAS,CAAC,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,SAAS,CAAC,CAAA;YAErF,eAAe,CAAC,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,aAAa,CAAC,CAAA;QAClF,CAAC;IACF,CAAC;IAED,OAAO,eAAe,CAAA;AACvB,CAAC;AAED,uDAAuD;AACvD,MAAM,UAAU,wCAAwC,CAAC,YAA0B,EAAE,UAAkB;IACtG,MAAM,2BAA2B,GAAG,IAAI,GAAG,EAAkB,CAAA;IAE7D,MAAM,2BAA2B,GAAG,IAAI,YAAY,EAAE,CAAA;IAEtD,SAAS,0BAA0B,CAAC,SAAiB,EAAE,iBAAyB;QAC/E,IAAI,UAAU,GAAG,iBAAiB,CAAA;QAElC,kFAAkF;QAClF,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC9B,MAAM,aAAa,GAAG,UAAU,GAAG,IAAI,CAAC,MAAM,CAAA;YAE9C,MAAM,SAAS,GAAG,2BAA2B,CAAC,SAAS,CAAA;YAEvD,IAAI,IAAI,KAAK,GAAG,IAAI,SAAS,IAAI,SAAS,CAAC,aAAa,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;gBACvF,2BAA2B,CAAC,SAAS,CAAC,IAAI,IAAI,GAAG,CAAA;gBACjD,2BAA2B,CAAC,SAAS,CAAC,SAAS,GAAG,aAAa,CAAA;YAChE,CAAC;iBAAM,CAAC;gBACP,MAAM,QAAQ,GAAG,UAAU,CAAC,SAAS,CAAC,UAAU,EAAE,aAAa,CAAC,CAAA;gBAEhE,2BAA2B,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,EAAE,IAAI,CAAC,CAAA;YAChE,CAAC;YAED,UAAU,GAAG,aAAa,CAAA;QAC3B,CAAC;IACF,CAAC;IAED,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,MAAM,SAAS,GAAG,YAAY,CAAC,UAAU,CAAC,SAAS,CAAC,CAAA;QACpD,MAAM,eAAe,GAAG,SAAS,CAAC,WAAW,CAAA;QAE7C,MAAM,qBAAqB,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAA;QAE/F,8FAA8F;QAC9F,IAAI,qBAAqB,KAAK,eAAe,EAAE,CAAC;YAC/C,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,qBAAqB,EAAE,eAAe,CAAC,CAAA;YAE9E,0BAA0B,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAA;QAC7D,CAAC;QAED,2BAA2B,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QACnD,2BAA2B,CAAC,GAAG,CAAC,2BAA2B,CAAC,MAAM,GAAG,CAAC,EAAE,SAAS,CAAC,CAAA;QAElF,oEAAoE;QACpE,IAAI,SAAS,KAAK,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3C,IAAI,UAAU,CAAC,MAAM,KAAK,SAAS,CAAC,SAAS,EAAE,CAAC;gBAC/C,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,SAAS,CAAC,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC,CAAA;gBAE9E,0BAA0B,CAAC,SAAS,EAAE,SAAS,CAAC,SAAS,CAAC,CAAA;YAC3D,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,EAAE,2BAA2B,EAAE,2BAA2B,EAAE,CAAA;AACpE,CAAC;AAED,SAAS,oBAAoB,CAAC,YAA0B,EAAE,IAAY;IACrE,MAAM,iBAAiB,GAAY,EAAE,CAAA;IACrC,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAA;IAExC,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;QACpC,iBAAiB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAA;IACtE,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,MAAM,KAAK,GAAG,WAAW,CAAC,CAAC,CAAC,CAAA;QAE5B,MAAM,iBAAiB,GAAG,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,IAAI,CAAC,CAAA;QAE5D,IAAI,KAAK,CAAC,WAAW,GAAG,iBAAiB,EAAE,CAAC;YAC3C,iBAAiB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,iBAAiB,EAAE,GAAG,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAA;QAC7E,CAAC;QAED,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;YACzB,iBAAiB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE,GAAG,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAA;QAC3E,CAAC;IACF,CAAC;IAED,CAAC;QACA,MAAM,aAAa,GAAG,WAAW,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,SAAS,CAAA;QAEpE,IAAI,aAAa,IAAI,aAAa,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAClD,iBAAiB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,aAAa,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAA;QACnE,CAAC;IACF,CAAC;IAED,OAAO,iBAAiB,CAAA;AACzB,CAAC;AAED,gGAAgG;AAChG,iBAAiB;AACjB,gGAAgG;AAChG,SAAS,iCAAiC,CAAC,OAA4B;IACtE,MAAM,uBAAuB,GAAG,gBAAgB,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IAC9E,MAAM,2BAA2B,GAAG,sBAAsB,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IACxF,MAAM,8BAA8B,GAAG,wCAAwC,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IAC7G,MAAM,6CAA6C,GAAG,8BAA8B,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAA;IACzH,MAAM,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,EAAE,CAAA;IAE3D,IAAI,YAAY,GAAG;QAClB,GAAG,kBAAkB;QACrB,GAAG,uBAAuB;QAC1B,GAAG,2BAA2B;QAC9B,GAAG,8BAA8B;QACjC,GAAG,6CAA6C;QAChD,GAAG,gBAAgB;QACnB,GAAG,eAAe;KAClB,CAAA;IAED,MAAM,WAAW,GAAG,wBAAwB,CAAC;QAC5C,GAAG,YAAY;QACf,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACrD,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;KACrD,CAAC,CAAA;IAEF,MAAM,kBAAkB,GAAG,WAAW,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;IAErE,OAAO,kBAAkB,CAAA;AAC1B,CAAC;AAED,KAAK,UAAU,kBAAkB;IAChC,IAAI,CAAC;QACJ,MAAM,eAAe,GAAG,MAAM,MAAM,CAAC,mCAAmC,CAAC,CAAA;QAEzE,OAAO,eAAe,CAAA;IACvB,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAA;IACjB,CAAC;AACF,CAAC;AAWD,MAAM,OAAO,YAAY;IACxB,SAAS,CAAO;IAChB,KAAK,CAAc;IAEnB,YAAY,SAAgB,EAAE,KAAmB;QAChD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;QAC1B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAA;IACnB,CAAC;IAED,IAAI,IAAI;QACP,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAA;IACvB,CAAC;IAED,IAAI,SAAS;QACZ,OAAO;YACN,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,WAAW;YACxC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,SAAS;SACnC,CAAA;IACF,CAAC;CACD;AAED,MAAM,OAAO,QAAS,SAAQ,YAAY;IACzC,OAAO,GAAa,EAAE,CAAA;CACtB;AAED,MAAM,OAAO,MAAO,SAAQ,YAAY;CACvC;AAaD,MAAM,CAAC,MAAM,0BAA0B,GAAwB;IAC9D,QAAQ,EAAE,EAAE;IACZ,kBAAkB,EAAE,EAAE;IACtB,6BAA6B,EAAE,IAAI;CACnC,CAAA"}
package/dist/WordSequence.js CHANGED
@@ -19,7 +19,7 @@ export class WordSequence {
19
19
  }
20
20
  }
21
21
  getWordAt(index) {
22
- return this.entries[index].text;
22
+ return this.entries[index]?.text;
23
23
  }
24
24
  getEntryRange(startIndex, endIndex) {
25
25
  return this.entries.slice(startIndex, endIndex);
package/dist/WordSequence.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"WordSequence.js","sourceRoot":"","sources":["../src/WordSequence.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,YAAY;IACxB,OAAO,GAAgB,EAAE,CAAA;IAEzB,OAAO,CAAC,IAAY,EAAE,eAAuB,EAAE,aAAsB;QACpE,MAAM,WAAW,GAAG,eAAe,CAAA;QACnC,MAAM,SAAS,GAAG,WAAW,GAAG,IAAI,CAAC,MAAM,CAAA;QAE3C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YACjB,IAAI;YACJ,WAAW;YACX,SAAS;YACT,aAAa;SACb,CAAC,CAAA;IACH,CAAC;IAED,YAAY,CAAC,UAAkB,EAAE,QAAgB;QAChD,OAAO,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IACzE,CAAC;IAED,CAAC,gBAAgB,CAAC,UAAkB,EAAE,QAAgB;QACrD,KAAK,IAAI,CAAC,GAAG,UAAU,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA;QACxB,CAAC;IACF,CAAC;IAED,SAAS,CAAC,KAAa;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAA;IAChC,CAAC;IAED,aAAa,CAAC,UAAkB,EAAE,QAAgB;QACjD,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;IAChD,CAAC;IAED,CAAC,iBAAiB,CAAC,UAAkB,EAAE,QAAgB;QACtD,KAAK,IAAI,CAAC,GAAG,UAAU,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;QACzB,CAAC;IACF,CAAC;IAED,UAAU,CAAC,KAAa;QACvB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IAC3B,CAAC;IAED,KAAK;QACJ,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IAClC,CAAC;IAED,KAAK,CAAC,UAAkB,EAAE,QAAgB;QACzC,MAAM,cAAc,GAAG,IAAI,YAAY,EAAE,CAAA;QAEzC,cAAc,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;QAEjE,OAAO,cAAc,CAAA;IACtB,CAAC;IAED,IAAI,SAAS;QACZ,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC7C,CAAC;IAED,IAAI,SAAS;QACZ,OAAO,IAAI,CAAC,UAAU,EAAE,IAAI,CAAA;IAC7B,CAAC;IAED,IAAI,QAAQ;QACX,OAAO,IAAI,CAAC,SAAS,EAAE,IAAI,CAAA;IAC5B,CAAC;IAED,IAAI,UAAU;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;IACvB,CAAC;IAED,IAAI,SAAS;QACZ,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACrC,CAAC;IAED,IAAI,kBAAkB;QACrB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,aAAa,KAAK,IAAI,CAAC,CAAA;IAClE,CAAC;IAED,IAAI,gBAAgB;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,aAAa,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC3F,CAAC;IAED,IAAI,qBAAqB;QACxB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,aAAa,KAAK,KAAK,CAAC,CAAA;IACnE,CAAC;IAED,IAAI,mBAAmB;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,aAAa,KAAK,KAAK,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC5F,CAAC;IAED,IAAI,IAAI;QACP,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC/B,CAAC;IAED,IAAI,MAAM;QACT,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;IAC3B,CAAC;CACD"}
1
+ {"version":3,"file":"WordSequence.js","sourceRoot":"","sources":["../src/WordSequence.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,YAAY;IACxB,OAAO,GAAgB,EAAE,CAAA;IAEzB,OAAO,CAAC,IAAY,EAAE,eAAuB,EAAE,aAAsB;QACpE,MAAM,WAAW,GAAG,eAAe,CAAA;QACnC,MAAM,SAAS,GAAG,WAAW,GAAG,IAAI,CAAC,MAAM,CAAA;QAE3C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YACjB,IAAI;YACJ,WAAW;YACX,SAAS;YACT,aAAa;SACb,CAAC,CAAA;IACH,CAAC;IAED,YAAY,CAAC,UAAkB,EAAE,QAAgB;QAChD,OAAO,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IACzE,CAAC;IAED,CAAC,gBAAgB,CAAC,UAAkB,EAAE,QAAgB;QACrD,KAAK,IAAI,CAAC,GAAG,UAAU,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA;QACxB,CAAC;IACF,CAAC;IAED,SAAS,CAAC,KAAa;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,IAAI,CAAA;IACjC,CAAC;IAED,aAAa,CAAC,UAAkB,EAAE,QAAgB;QACjD,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;IAChD,CAAC;IAED,CAAC,iBAAiB,CAAC,UAAkB,EAAE,QAAgB;QACtD,KAAK,IAAI,CAAC,GAAG,UAAU,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;QACzB,CAAC;IACF,CAAC;IAED,UAAU,CAAC,KAAa;QACvB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IAC3B,CAAC;IAED,KAAK;QACJ,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IAClC,CAAC;IAED,KAAK,CAAC,UAAkB,EAAE,QAAgB;QACzC,MAAM,cAAc,GAAG,IAAI,YAAY,EAAE,CAAA;QAEzC,cAAc,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;QAEjE,OAAO,cAAc,CAAA;IACtB,CAAC;IAED,IAAI,SAAS;QACZ,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC7C,CAAC;IAED,IAAI,SAAS;QACZ,OAAO,IAAI,CAAC,UAAU,EAAE,IAAI,CAAA;IAC7B,CAAC;IAED,IAAI,QAAQ;QACX,OAAO,IAAI,CAAC,SAAS,EAAE,IAAI,CAAA;IAC5B,CAAC;IAED,IAAI,UAAU;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;IACvB,CAAC;IAED,IAAI,SAAS;QACZ,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACrC,CAAC;IAED,IAAI,kBAAkB;QACrB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,aAAa,KAAK,IAAI,CAAC,CAAA;IAClE,CAAC;IAED,IAAI,gBAAgB;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,aAAa,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC3F,CAAC;IAED,IAAI,qBAAqB;QACxB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,aAAa,KAAK,KAAK,CAAC,CAAA;IACnE,CAAC;IAED,IAAI,mBAAmB;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,aAAa,KAAK,KAAK,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC5F,CAAC;IAED,IAAI,IAAI;QACP,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC/B,CAAC;IAED,IAAI,MAAM;QACT,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;IAC3B,CAAC;CACD"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@echogarden/text-segmentation",
3
- "version": "0.3.3",
3
+ "version": "0.4.1",
4
4
  "description": "A library for multilingual word, phrase and sentence segmentation.",
5
5
  "author": "Rotem Dan",
6
6
  "license": "MIT",
@@ -50,6 +50,6 @@
50
50
  }
51
51
  },
52
52
  "devDependencies": {
53
- "@types/node": "~22.15.19"
53
+ "@types/node": "~24.3.0"
54
54
  }
55
55
  }
package/src/Patterns.ts CHANGED
@@ -305,7 +305,7 @@ const sentenceSeparatorCharacterPattern = [
305
305
 
306
306
  const sentenceSeparatorTrailingPunctuationPattern = [
307
307
  inputStart,
308
- anyOf('"', '”', '’', ')', ']', '}', '»', ...sentenceSeparatorCharacters, ...phraseSeparatorCharacters, oneOrMore(whitespace)),
308
+ anyOf('"', '”', '’', ')', ']', '}', '»', '」', ...sentenceSeparatorCharacters, ...phraseSeparatorCharacters, whitespace),
309
309
  inputEnd
310
310
  ]
311
311
 
package/src/Test.ts CHANGED
@@ -23,7 +23,7 @@ async function test1() {
23
23
 
24
24
  async function test2() {
25
25
  const { readFileSync, writeFileSync } = await import('fs')
26
- const text = readFileSync('test-data/Test2.txt', 'utf-8')
26
+ const text = readFileSync('test-data/Japanese3.txt', 'utf-8')
27
27
 
28
28
  const timer = new Timer()
29
29
 
@@ -114,8 +114,37 @@ async function test4() {
114
114
 
115
115
  const result = (await splitToWords(text)).wordArray
116
116
  log(result.join(' | '))
117
+ }
118
+
119
+
120
+ async function test5() {
121
+ const text1 = `
122
+ ただ
123
+
124
+ ◆
125
+
126
+ 街の
127
+ `
128
+
129
+ const text2 = `
130
+ Hello World. Yo!
131
+
132
+
133
+
134
+ gooooogoo
135
+
136
+ , Hi. BOBO.
137
+ `
138
+
139
+ const wordSequence = await splitToWords(text1)
140
+
141
+ const result = await segmentWordSequence(wordSequence)
142
+
143
+ const sentencesText = result.sentences.map(x => x.text)
117
144
 
145
+ log(sentencesText.join(' \n '))
118
146
  }
119
147
 
120
148
  //test1()
121
- test2()
149
+ //test2()
150
+ test5()
package/src/TextSegmentation.ts CHANGED
@@ -17,67 +17,105 @@ export async function segmentText(text: string, options?: SegmentationOptions) {
17
17
  }
18
18
 
19
19
  export async function segmentWordSequence(wordSequence: WordSequence) {
20
+ const segmentWordRanges: Range[] = []
21
+
22
+ {
23
+ let segmentStartWordOffset = 0
24
+ let nonWhitespaceWordSeen = false
25
+
26
+ for (let wordIndex = 0; wordIndex < wordSequence.length; wordIndex++) {
27
+ const word = wordSequence.getWordAt(wordIndex)
28
+
29
+ if (word.endsWith('\n') && nonWhitespaceWordSeen) {
30
+ const nextWord = wordIndex < wordSequence.length - 1 ? wordSequence.getWordAt(wordIndex + 1) : ''
31
+
32
+ if (nextWord !== '\r' && nextWord !== '\n') {
33
+ segmentWordRanges.push({ start: segmentStartWordOffset, end: wordIndex + 1 })
34
+ segmentStartWordOffset = wordIndex + 1
35
+ }
36
+ }
37
+
38
+ if (nonWhitespaceWordSeen === false) {
39
+ nonWhitespaceWordSeen = !whitespacePatternRegExp.test(word)
40
+ }
41
+ }
42
+
43
+ if (segmentStartWordOffset < wordSequence.length) {
44
+ segmentWordRanges.push({ start: segmentStartWordOffset, end: wordSequence.length })
45
+ }
46
+ }
47
+
20
48
  const sentenceWordRanges: Range[] = []
49
+ const segmentSentenceRanges: Range[] = []
21
50
 
22
- const minimumSentenceLetterCount = 2
51
+ {
52
+ const minimumSentenceLetterCount = 2
23
53
 
24
- let sentenceStartWordOffset = 0
25
- let currentSentenceLetterCount = 0
54
+ for (let segmentIndex = 0; segmentIndex < segmentWordRanges.length; segmentIndex++) {
55
+ const segmentWordRange = segmentWordRanges[segmentIndex]
26
56
 
27
- for (let wordIndex = 0; wordIndex < wordSequence.length; wordIndex++) {
28
- const word = wordSequence.getWordAt(wordIndex)
57
+ const segmentStartWordIndex = segmentWordRange.start
58
+ const segmentEndWordIndex = segmentWordRange.end
59
+
60
+ segmentSentenceRanges.push({ start: sentenceWordRanges.length, end: sentenceWordRanges.length })
61
+
62
+ let sentenceStartWordOffset = segmentStartWordIndex
63
+ let currentSentenceLetterCount = 0
64
+
65
+ for (let wordIndex = segmentStartWordIndex; wordIndex < segmentEndWordIndex; wordIndex++) {
66
+ const word = wordSequence.getWordAt(wordIndex)
29
67
 
30
- if (currentSentenceLetterCount < minimumSentenceLetterCount) {
31
- const matches = word.matchAll(letterPatternGlobalRegExp)
68
+ if (currentSentenceLetterCount < minimumSentenceLetterCount) {
69
+ const matches = word.matchAll(letterPatternGlobalRegExp)
32
70
 
33
- for (const _ of matches) {
34
- currentSentenceLetterCount += 1
71
+ for (const _ of matches) {
72
+ currentSentenceLetterCount += 1
35
73
 
36
- if (currentSentenceLetterCount >= minimumSentenceLetterCount) {
37
- break
74
+ if (currentSentenceLetterCount >= minimumSentenceLetterCount) {
75
+ break
76
+ }
77
+ }
38
78
  }
39
- }
40
- }
41
79
 
42
- if (currentSentenceLetterCount >= minimumSentenceLetterCount && sentenceSeparatorCharacterRegExp.test(word)) {
43
- let trailingSequenceEndIndex = wordIndex
44
- let trailingSequenceContainedWhitespace = false
80
+ if (currentSentenceLetterCount >= minimumSentenceLetterCount && sentenceSeparatorCharacterRegExp.test(word)) {
81
+ let trailingSequenceEndIndex = wordIndex
82
+ let trailingSequenceContainedWhitespace = false
83
+
84
+ while (trailingSequenceEndIndex < segmentEndWordIndex) {
85
+ const trailingWord = wordSequence.getWordAt(trailingSequenceEndIndex)
45
86
 
46
- while (trailingSequenceEndIndex < wordSequence.length) {
47
- const trailingWord = wordSequence.getWordAt(trailingSequenceEndIndex)
87
+ if (sentenceSeparatorTrailingPunctuationCharacterRegExp.test(trailingWord)) {
88
+ if (!trailingSequenceContainedWhitespace && whitespacePatternRegExp.test(trailingWord)) {
89
+ trailingSequenceContainedWhitespace = true
90
+ }
48
91
 
49
- if (sentenceSeparatorTrailingPunctuationCharacterRegExp.test(trailingWord)) {
50
- if (!trailingSequenceContainedWhitespace && whitespacePatternRegExp.test(trailingWord)) {
51
- trailingSequenceContainedWhitespace = true
92
+ trailingSequenceEndIndex++
93
+ } else {
94
+ break
95
+ }
52
96
  }
53
97
 
54
- trailingSequenceEndIndex++
55
- } else {
56
- break
57
- }
58
- }
98
+ sentenceWordRanges.push({
99
+ start: sentenceStartWordOffset,
100
+ end: trailingSequenceEndIndex
101
+ })
59
102
 
60
- if (trailingSequenceEndIndex === wordSequence.length ||
61
- trailingSequenceContainedWhitespace ||
62
- ['。', '？', '！'].includes(word)) {
103
+ segmentSentenceRanges[segmentSentenceRanges.length - 1].end += 1
63
104
 
64
- sentenceWordRanges.push({
65
- start: sentenceStartWordOffset,
66
- end: trailingSequenceEndIndex
67
- })
105
+ sentenceStartWordOffset = trailingSequenceEndIndex
106
+ currentSentenceLetterCount = 0
68
107
 
69
- sentenceStartWordOffset = trailingSequenceEndIndex
70
- currentSentenceLetterCount = 0
108
+ wordIndex = trailingSequenceEndIndex - 1
109
+ }
110
+ }
71
111
 
72
- wordIndex = trailingSequenceEndIndex - 1
112
+ if (sentenceStartWordOffset < segmentEndWordIndex) {
113
+ sentenceWordRanges.push({ start: sentenceStartWordOffset, end: segmentEndWordIndex })
114
+ segmentSentenceRanges[segmentSentenceRanges.length - 1].end += 1
73
115
  }
74
116
  }
75
117
  }
76
118
 
77
- if (sentenceStartWordOffset < wordSequence.length) {
78
- sentenceWordRanges.push({ start: sentenceStartWordOffset, end: wordSequence.length })
79
- }
80
-
81
119
  const sentences: Sentence[] = []
82
120
 
83
121
  for (const wordRange of sentenceWordRanges) {
@@ -95,7 +133,12 @@ export async function segmentWordSequence(wordSequence: WordSequence) {
95
133
  for (let wordIndex = phraseStartWordOffset; wordIndex < sentenceEndWordOffset; wordIndex++) {
96
134
  const currentWord = wordSequence.getWordAt(wordIndex)
97
135
 
98
- if (phraseSeparatorRegExp.test(currentWord)) {
136
+ const isCurrentWordPhraseSeparator =
137
+ phraseSeparatorRegExp.test(currentWord) &&
138
+ ((currentWord !== ':' && currentWord !== ';') ||
139
+ whitespacePatternRegExp.test(wordSequence.getWordAt(wordIndex + 1)))
140
+
141
+ if (isCurrentWordPhraseSeparator) {
99
142
  let whitespaceSeenOnce = false
100
143
 
101
144
  while (wordIndex < sentenceEndWordOffset - 1) {
@@ -139,6 +182,7 @@ export async function segmentWordSequence(wordSequence: WordSequence) {
139
182
  const result: SegmentationResult = {
140
183
  words: wordSequence,
141
184
  sentences,
185
+ segmentSentenceRanges,
142
186
  }
143
187
 
144
188
  return result
@@ -416,6 +460,7 @@ async function getIcuSegmentation() {
416
460
  export interface SegmentationResult {
417
461
  words: WordSequence
418
462
  sentences: Sentence[]
463
+ segmentSentenceRanges: Range[]
419
464
  }
420
465
 
421
466
  export class TextFragment {
package/src/WordSequence.ts CHANGED
@@ -24,7 +24,7 @@ export class WordSequence {
24
24
  }
25
25
 
26
26
  getWordAt(index: number) {
27
- return this.entries[index].text
27
+ return this.entries[index]?.text
28
28
  }
29
29
 
30
30
  getEntryRange(startIndex: number, endIndex: number) {