@echogarden/text-segmentation 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -25
- package/dist/EastAsianCharacterPatterns.d.ts +0 -6
- package/dist/EastAsianCharacterPatterns.js +6 -6
- package/dist/EastAsianCharacterPatterns.js.map +1 -1
- package/dist/Patterns.d.ts +4 -29
- package/dist/Patterns.js +136 -61
- package/dist/Patterns.js.map +1 -1
- package/dist/Suppressions.d.ts +1 -0
- package/dist/Suppressions.js +19 -9
- package/dist/Suppressions.js.map +1 -1
- package/dist/Test.js +13 -5
- package/dist/Test.js.map +1 -1
- package/dist/TextSegmentation.d.ts +3 -4
- package/dist/TextSegmentation.js +68 -39
- package/dist/TextSegmentation.js.map +1 -1
- package/dist/WordSequence.d.ts +5 -3
- package/dist/WordSequence.js +13 -6
- package/dist/WordSequence.js.map +1 -1
- package/dist/utilities/Utilities.d.ts +1 -0
- package/dist/utilities/Utilities.js +7 -0
- package/dist/utilities/Utilities.js.map +1 -1
- package/package.json +4 -4
- package/src/EastAsianCharacterPatterns.ts +6 -6
- package/src/Patterns.ts +177 -87
- package/src/Suppressions.ts +20 -9
- package/src/Test.ts +18 -5
- package/src/TextSegmentation.ts +91 -51
- package/src/WordSequence.ts +17 -7
- package/src/utilities/Utilities.ts +10 -0
package/dist/Suppressions.js
CHANGED
|
@@ -1,15 +1,25 @@
|
|
|
1
1
|
export const cldrSuppressions = {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
2
|
+
//en: ['L.P.', 'Alt.', 'Approx.', 'E.G.', 'O.', 'Maj.', 'Misc.', 'P.O.', 'J.D.', 'Jam.', 'Card.', 'Dec.', 'Sept.', 'MR.', 'Long.', 'Hat.', 'G.', 'Link.', 'DC.', 'D.C.', 'M.T.', 'Hz.', 'Mrs.', 'By.', 'Act.', 'Var.', 'N.V.', 'Aug.', 'B.', 'S.A.', 'Up.', 'Job.', 'Num.', 'M.I.T.', 'Ok.', 'Org.', 'Ex.', 'Cont.', 'U.', 'Mart.', 'Fn.', 'Abs.', 'Lt.', 'OK.', 'Z.', 'E.', 'Kb.', 'Est.', 'A.M.', 'L.A.', 'Prof.', 'U.S.', 'Nov.', 'Ph.D.', 'Mar.', 'I.T.', 'exec.', 'Jan.', 'N.Y.', 'X.', 'Md.', 'Op.', 'vs.', 'D.A.', 'A.D.', 'R.L.', 'P.M.', 'Or.', 'M.R.', 'Cap.', 'PC.', 'Feb.', 'Exec.', 'I.e.', 'Sep.', 'Gb.', 'K.', 'U.S.C.', 'Mt.', 'S.', 'A.S.', 'C.O.D.', 'Capt.', 'Col.', 'In.', 'C.F.', 'Adj.', 'AD.', 'I.D.', 'Mgr.', 'R.T.', 'B.V.', 'M.', 'Conn.', 'Yr.', 'Rev.', 'Phys.', 'pp.', 'Ms.', 'To.', 'Sgt.', 'J.K.', 'Nr.', 'Jun.', 'Fri.', 'S.A.R.', 'Lev.', 'Lt.Cdr.', 'Def.', 'F.', 'Do.', 'Joe.', 'Id.', 'Mr.', 'Dept.', 'Is.', 'Pvt.', 'Diff.', 'Hon.B.A.', 'Q.', 'Mb.', 'On.', 'Min.', 'J.B.', 'Ed.', 'AB.', 'A.', 'S.p.A.', 'I.', 'a.m.', 'Comm.', 'Go.', 'VS.', 'L.', 'All.', 'PP.', 'P.V.', 'T.', 'K.R.', 'Etc.', 'D.', 'Adv.', 'Lib.', 'E.g.', 'Pro.', 'U.S.A.', 'S.E.', 'AA.', 'Rep.', 'Sq.', 'As.'],
|
|
3
|
+
en: ['L.P.', 'Alt.', 'Approx.', 'E.G.', 'Maj.', 'Misc.', 'P.O.', 'J.D.', 'Dec.', 'Sept.', 'MR.', 'DC.', 'D.C.', 'M.T.', 'Mrs.', 'N.V.', 'Aug.', 'S.A.', 'Num.', 'M.I.T.', 'Org.', 'Ex.', 'Cont.', 'Mart.', 'Fn.', 'Abs.', 'Lt.', 'Est.', 'A.M.', 'L.A.', 'Prof.', 'U.S.', 'Nov.', 'Ph.D.', 'Mar.', 'I.T.', 'exec.', 'Jan.', 'N.Y.', 'Md.', 'Op.', 'vs.', 'D.A.', 'A.D.', 'R.L.', 'P.M.', 'M.R.', 'Feb.', 'Exec.', 'I.e.', 'Sep.', 'U.S.C.', 'Mt.', 'A.S.', 'C.O.D.', 'Capt.', 'Col.', 'C.F.', 'Adj.', 'I.D.', 'Mgr.', 'R.T.', 'B.V.', 'M.', 'Conn.', 'Yr.', 'Rev.', 'Phys.', 'pp.', 'Ms.', 'Sgt.', 'J.K.', 'Nr.', 'Jun.', 'Fri.', 'S.A.R.', 'Lev.', 'Lt.Cdr.', 'Def.', 'Mr.', 'Dept.', 'Pvt.', 'Diff.', 'Hon.B.A.', 'Mb.', 'Min.', 'J.B.', 'Ed.', 'AB.', 'S.p.A.', 'I.', 'a.m.', 'Comm.', 'VS.', 'PP.', 'P.V.', 'K.R.', 'Etc.', 'Adv.', 'Lib.', 'E.g.', 'U.S.A.', 'S.E.', 'AA.', 'Rep.', 'Sq.'],
|
|
4
|
+
de: ['Port.', 'Alt.', 'Di.', 'Ges.', 'frz.', 'entspr.', 'Gebr.', 'erw.', 'Frl.', 'Inh.', 'k.u.k.', 'Ca.', 'J.D.', 'Ausg.', 'evtl.', 'So.', 'i.B.', 's.a.', 'kgl.', 'Sept.', 'o.B.', 'Sa.', 'ev.', 'Dez.', 'am.', 'i.R.', 'eigtl.', 'i.J.', 'u.U.', 'G.', 'z.Hd.', 'u.A.w.g.', 'Kl.', 'Spezif.', 'Obj.', 'Ing.', 'D. h.', 'Folg.', 'Akt.', 'i.A.', 'Msp.', 'U.U.', 'Chr.', 'R.', 'Einh.', 'schwäb.', 'Vgl.', 'Aug.', 'Dipl.-Ing.', 'W.', 'B.', 'U. U.', 'J.', 'Fa.', 'Mo.', 'n.u.Z.', 'Op.', 'Mrd.', 'e.h.', 'Hr.', 'Hrn.', 'Ztr.', 'k. u. k.', 'Bibl.', 'd.Ä.', 'b.', 'M.', 'i.H.', 'v.R.w.', 'o.A.', 'St.', 'Dr.', 'Fn.', 'Abs.', 'Rd.', 'Dtzd.', 'Jahrh.', 'Z.', 'Std.', 'n. Chr.', 'möbl.', 'tägl.', 'gest.', 'gesch.', 'z.B.', 'Hbf.', 'Abt.', 'A.M.', 'e.Wz.', 'v.T.', 'Nov.', 'z.', 'Prot.', 'U.S.', 'Wg.', 'u.v.a.', 'Adr.', 'App.', 'ggf.', 'ggfs.', 'Jan.', 'O.', 'Rel.', 'od.', 'Pfd.', 'a.a.O.', 'p.Adr.', 'P.', 'Gem.', 'v. Chr.', 'Art.', 'z.Z.', 'S.A.', 'i.V.', 'verh.', 'Ausschl.', 'm.W.', 'Dir.', 'Verf.', 'Sek.', 'r.', 'Chin.', 'Feb.', 'Int.', 'Sep.', 'Gesch.', 'schweiz.', 'Bed.', 'a.Rh.', 'jew.', 'vgl.', 'a.M.', 'Str.', 'exkl.', 'gek.', 'Erf.', 'u.Ä.', 'ehem.', 'näml.', 'u. Z.', 'v. u. Z.', 'sog.', 'C.', 'Dipl.-Kfm.', 'mtl.', 'Hrsg.', 'Qu.', 'röm.', 'u.', 'U.', 'Adj.', 'Kap.', 'hpts.', 'a.D.', 'gedr.', 'Best.', 'N.', 'v.u.Z.', 'Phys.', 'Fr.', 'd.J.', 'Reg.-Bez.', 'm.E.', 'schles.', 'Max.', 'Ltd.', 'südd.', 'inkl.', 'geb.', 'Ggf.', 'Inc.', 'kath.', 'kfm.', 'Nr.', 'Proz.', 'Dim.', 'verw.', 'Reg.', 'Dat.', 'Evtl.', 'led.', 'F.', 'Test.', 'Schr.', 'Do.', 'PIN.', 'Z. Zt.', 'v.Chr.', 'Tägl.', 's.', 'amtl.', 'Temp.', 'Mind.', 'e.V.', 'Abw.', 'P.M.', 'F.f.', 'a.a.S.', 'Mod.', 'Co.', 'Min.', 'Allg.', 'Geograph.', 'Jr.', 'Urspr.', 'Apr.', 'Z. B.', 'v.H.', 'A.', 'einschl.', 'Trans.', 'zzgl.', 'StR.', 'Fam.', 'I.', 'jhrl.', 'u.a.', 'Ben.', 'o.g.', 'Kfm.', 'Konv.', 'Mi.', 'L.', 'beil.', 'T.', 'Ursprüngl.', 'röm.-kath.', 'Okt.', 'u.ä.', 'Tel.', 'D.', 'Ber.', 'Kop.', 'Mio.', 'Y.', 'U.S.A.', 'v. H.', 'Forts. f.', 'Rep.', 'Hptst.', 'österr.'],
|
|
5
|
+
es: ['Rdos.', 'JJ.OO.', 'Sres.', 'fig.', 'may.', 'RR.HH.', 'oct.', 'cap.', 'mié.', 'doc.', 'Excmo.', 'Trab.', 'Excmos.', 'Kit.', 'Inc.', 'FF.CC.', 'DC.', 'ago.', 'trad.', 'SA.', 'Rvdos.', 'ed.', 'Exmo.', 'jul.', 'col.', 'RAM.', 'Srtas.', 'ene.', 'Rol.', 'Fabric.', 'Comm.', 'vid.', 'Da.', 'dic.', 'ss.', 'abr.', 'ntra.', 'Sra.', 'dtor.', 'cf.', 'dom.', 'prov.', 'Emm.', 'Sr.', 'licdo.', 'p.ej.', 'bol.', 'figs.', 'Vda.', 'Dr.', 'ntro.', 'Desv.', 'O.M.', 'Ldo.', 'Drs.', 'sáb.', 'feb.', 'Ltda.', 'Lcda.', 'Exma.', 'C.V.', 'SS.MM.', 'Lda.', 'U.S.', 'hnos.', 'R.D.', 'Korn.', 'v.gr.', 'vs.', 'Ilmas.', 'Rdo.', 'ej.', 'vie.', 'jue.', 'a. C.', 'Ilmos.', 'e. c.', 'Excma.', 'afma.', 'licda.', 'Em.', 'K.', 'sras.', 'MM.', 'fund.', 'Mons.', 'Lcdo.', 'afmo.', 'C.', 'A.C.', 'dptos.', 'Col.', 'Srta.', 'Av.', 'Ant.', 'depto.', 'Var.', 'H.P.', 'D.', 'M.', 'C.P.', 'Rev.', 'Rvdmos.', 'Fr.', 'Ilmo.', 'afmos.', 'Ltd.', 'afmas.', 'prof.', 'lun.', 'SS.AA.', 'Sol.', 'nov.', 'mss.', 'Dña.', 'Seg.', 'mar.', 'Rvdmo.', 'Reg.', 'ms.', 'Sras.', 'sres.', 'U.S.A.', 'Sta.', 'Sdad.', 'Dra.', 'srs.', 'R.U.', 'deptos.', 'dpto.', 'jun.', 'bco.', 'Cía.', 'Id.', 'Mr.', 'e.g.', 'C.S.', 'Excmas.', 'Dª.', 'Rvdo.', 'Lic.', 'cfr.', 'Corp.', 'Dto.', 'Ilma.', 'L.', 'All.', 'PP.', 'd. C.', 'Ltdo.', 'mtro.', 'Mrs.', 'Desc.', 'Avda.', 'Exmas.', 'a. e. c.', 'Bien.', 'Exmos.', 'AA.', 'Sto.', 'CA.', 'sept.', 'Exc.', 'c/c.'],
|
|
6
|
+
fr: ['aux.', 'config.', 'collab.', 'M.', 'dim.', 'imprim.', 'oct.', 'syst.', 'bull.', 'MM.', 'doc.', 'P.O.', 'hôp.', 'Mart.', 'juil.', 'broch.', 'adr.', 'symb.', 'C.', 'anc.', 'voit.', 'Jr.', 'graph.', 'dir.', 'éd.', 'fig.', 'édit.', 'niv.', 'quart.', 'cam.', 'éval.', 'anon.', 'réf.', 'Comm.', 'Prof.', 'févr.', 'indus.', 'DC.', 'équiv.', 'illustr.', 'acoust.', 'nov.', 'L.', 'All.', 'U.S.', 'S.M.A.R.T.', 'sept.', 'avr.', 'jeu.', 'dest.', 'P.-D. G.', 'ill.', 'coll.', 'encycl.', 'mer.', 'Desc.', 'ven.', 'P.', 'lun.', 'Inc.', 'sam.', 'D.', 'append.', 'Var.', 'categ.', 'janv.', 'S.A.', 'imm.', 'U.S.A.', 'mar.', 'exempl.', 'déc.', 'ann.', 'U.', 'synth.', 'dict.', 'av. J.-C.', 'W.', 'Op.', 'ap. J.-C.', 'gouv.', 'trav. publ.'],
|
|
7
|
+
it: ['N.B.', 'div.', 'a.C.', 'fig.', 'd.p.R.', 'c.c.p.', 'Cfr.', 'vol.', 'Geom.', 'O.d.G.', 'S.p.A.', 'ver.', 'N.d.A.', 'dott.', 'arch.', 'd.C.', 'N.d.T.', 'rag.', 'Sig.', 'Mod.', 'pag.', 'dr.', 'tav.', 'N.d.E.', 'DC.', 'mitt.', 'Ing.', 'int.', 'on.', 'C.P.', 'ag.', 'L.', 'U.S.', 'S.M.A.R.T.', 'p.i.', 'tab.', 'Ltd.', 'Liv.', 'D.', 'U.S.A.', 'sez.', 'avv.', 'S.A.R.', 'all.', 'p.'],
|
|
8
|
+
pt: ['psicol.', 'fig.', 'compl.', 'rep.', 'cap.', 'doc.', 'fisiol.', 'dipl.', 'astron.', 'port.', 'eletrôn.', 'geom.', 'mov.', 'ago.', 'trad.', 'arquit.', 'dez.', 'ed.', 'apt.', 'Exmo.', 'col.', 'ff.', 'univ.', 'res.', 'R.', 'transp.', 'D.C', 'l.', 'des.', 'fev.', 'abr.', 'liter.', 'lat.', 'Dir.', 'cf.', 'adm.', 'fot.', 'p.m.', 'P.M.', 'créd.', 'jur.', 'com.', 'anat.', 'dir.', 'end.', 'fís.', 'E.', 'Est.', 'cont.', 'matem.', 'Drs.', 'gên.', 'neol.', 'pág.', 'índ.', 'Ltda.', 'Exma.', 'esp.', 'ingl.', 'tecnol.', 'Mar.', 'símb.', 'Pe.', 'pal.', 'filos.', 'V.T.', 'fasc.', 'vs.', 'mai.', 'S.A.', 'profa.', 'N.Sra.', 'r.s.v.p.', 'cel.', 'mat.', 'abrev.', 'out.', 'long.', 'aux.', 'arit.', 'aer.', 'jul.', 'lin.', 'S.', 'méd.', 'odontol.', 'org.', 'A.C.', 'jun.', 'déb.', 'Av.', 'álg.', 'sup.', 'fl.', 'odont.', 'caps.', 'relat.', 'organiz.', 'hist.', 'Fr.', 'Ilmo.', 'fem.', 'ap.', 'Ltd.', 'pol.', 'séc.', 'prof.', 'cx.', 'nov.', 'quím.', 'mús.', 'agric.', 'mar.', 'W.C.', 'fr.', 'cat.', 'jan.', 'pron.', 'rel.', 'autom.', 'Sta.', 'Dra.', 'p.', 'tel.', 'div.', 'p. ex.', 'a.C.', 'bras.', 'Alm.', 'Dr.', 'comp.', 'pq.', 'arqueol.', 'náut.', 'biogr.', 'f.', 'círc.', 'fac.', 'd.C.', 'apart.', 'ex.', 'Jr.', 'set.', 'tec.', 'sociol.', 'gram.', 'ind.', 'Ilma.', 'vol.', 'eng.', 'rod.', 'Ph.D.', 'Dras.', 'pp.', 'elem.', 'máq.', 'cód.', 'eletr.', 'prod.', 'ref.', 'fil.', 'a.m.', 'A.M', 'obs.', 'N.T.', 'contab.', 'Sto.', 'lit.', 'educ.', 'rementente', 'desc.', 'próx.'],
|
|
9
|
+
ru: ['руб.', 'янв.', 'до н. э.', 'сент.', 'тел.', 'дек.', 'февр.', 'нояб.', 'апр.', 'н. э.', 'окт.', 'тыс.', 'авг.', 'проф.', 'н.э.', 'кв.', 'ул.', 'отд.'],
|
|
10
|
+
};
|
|
11
|
+
export const additionalSuppressions = {
|
|
12
|
+
en: [
|
|
13
|
+
'TL;DR'
|
|
14
|
+
],
|
|
9
15
|
};
|
|
10
16
|
export const leadingApostropheContractionSuppressions = {
|
|
11
|
-
'en': [
|
|
12
|
-
|
|
17
|
+
'en': [
|
|
18
|
+
`'cause`, `'til`, `'bout`, `'twas`, `'tis`
|
|
19
|
+
],
|
|
20
|
+
'af': [
|
|
21
|
+
`'n`
|
|
22
|
+
]
|
|
13
23
|
};
|
|
14
24
|
export const nounSuppressions = [
|
|
15
25
|
'C#', 'F#', 'C++', 'Yahoo!', 'Toys"R"Us', `Dunkin'`, 'Ke$ha', 'I/O', 'Sky+', 'A/C', 'A/V'
|
package/dist/Suppressions.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Suppressions.js","sourceRoot":"","sources":["../src/Suppressions.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,gBAAgB,GAA6B;IACzD,
|
|
1
|
+
{"version":3,"file":"Suppressions.js","sourceRoot":"","sources":["../src/Suppressions.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,gBAAgB,GAA6B;IACzD,6pCAA6pC;IAC7pC,EAAE,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC;IAC/1B,EAAE,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC;IAC9+D,EAAE,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC;IAC92C,EAAE,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,aAAa,CAAC;IACxtB,EAAE,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC;IAC9X,EAAE,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,OAAO,CAAC;IAC17C,EAAE,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC;CAC3J,CAAA;AAED,MAAM,CAAC,MAAM,sBAAsB,GAA8B;IAChE,EAAE,EAAE;QACH,OAAO;KACP;CACD,CAAA;AAED,MAAM,CAAC,MAAM,wCAAwC,GAA6B;IACjF,IAAI,EAAE;QACL,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM;KAC1C;IACD,IAAI,EAAE;QACL,IAAI;KACJ;CACD,CAAA;AAED,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC/B,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK;CACzF,CAAA;AAED,MAAM,CAAC,MAAM,eAAe,GAAG;IAC9B,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK;CACpC,CAAA;AAED,kDAAkD;AAClD,wBAAwB;AACxB,yBAAyB;AACzB,gBAAgB;AAChB,wBAAwB;AACxB,EAAE;AACF,0CAA0C;AAC1C,qEAAqE;AACrE,gDAAgD;AAChD,iFAAiF;AACjF,0FAA0F"}
|
package/dist/Test.js
CHANGED
|
@@ -1,16 +1,24 @@
|
|
|
1
|
-
import { addMissingPunctuationWordsToWordSequence, segmentText, splitToWords, WordSequence } from "./TextSegmentation.js";
|
|
1
|
+
import { addMissingPunctuationWordsToWordSequence, segmentText, segmentWordSequence, splitToWords, WordSequence } from "./TextSegmentation.js";
|
|
2
2
|
import { Timer } from "./utilities/Timer.js";
|
|
3
3
|
const log = console.log;
|
|
4
4
|
async function test1() {
|
|
5
5
|
{
|
|
6
|
-
const
|
|
7
|
-
|
|
6
|
+
//const text = 'Hello! 1. Say who? 2. How are you? This is v2.0 that good. I have 2 344 234ms C# is good games'
|
|
7
|
+
const text = 'The time is 06:12';
|
|
8
|
+
//const text = 'Hello world! 23rd? 123% 3/4/7 Привет мир! 你好世界！'
|
|
9
|
+
const options = {
|
|
10
|
+
language: 'en'
|
|
11
|
+
};
|
|
12
|
+
const wordSequence = await splitToWords(text, options);
|
|
13
|
+
console.log(JSON.stringify(wordSequence.wordArray));
|
|
14
|
+
const segmentedText = await segmentWordSequence(wordSequence);
|
|
15
|
+
const x = 0;
|
|
8
16
|
}
|
|
9
17
|
const { readFileSync, writeFileSync } = await import('fs');
|
|
10
18
|
const text = readFileSync('test-data/Test.txt', 'utf-8');
|
|
11
19
|
const timer = new Timer();
|
|
12
20
|
let result;
|
|
13
|
-
for (let i = 0; i <
|
|
21
|
+
for (let i = 0; i < 5; i++) {
|
|
14
22
|
result = await segmentText(text, {
|
|
15
23
|
language: 'en',
|
|
16
24
|
customSuppressions: [],
|
|
@@ -26,7 +34,7 @@ async function test1() {
|
|
|
26
34
|
const phrases = sentence.phrases;
|
|
27
35
|
for (let phraseIndex = 0; phraseIndex < phrases.length; phraseIndex++) {
|
|
28
36
|
const phrase = phrases[phraseIndex];
|
|
29
|
-
segmentedText += phrase.
|
|
37
|
+
segmentedText += phrase.words.wordArray.join(' | ');
|
|
30
38
|
if (phraseIndex < phrases.length - 1) {
|
|
31
39
|
segmentedText += `\n${'-'.repeat(100)}\n`;
|
|
32
40
|
}
|
package/dist/Test.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Test.js","sourceRoot":"","sources":["../src/Test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wCAAwC,
|
|
1
|
+
{"version":3,"file":"Test.js","sourceRoot":"","sources":["../src/Test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wCAAwC,EAA2C,WAAW,EAAE,mBAAmB,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AACvL,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAE5C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAA;AAEvB,KAAK,UAAU,KAAK;IACnB,CAAC;QACA,+GAA+G;QAC/G,MAAM,IAAI,GAAG,mBAAmB,CAAA;QAChC,iEAAiE;QAEjE,MAAM,OAAO,GAAwB;YACpC,QAAQ,EAAE,IAAI;SACd,CAAA;QAED,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;QAEtD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAA;QAEnD,MAAM,aAAa,GAAG,MAAM,mBAAmB,CAAC,YAAY,CAAC,CAAA;QAE7D,MAAM,CAAC,GAAG,CAAC,CAAA;IACZ,CAAC;IAED,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAA;IAC1D,MAAM,IAAI,GAAG,YAAY,CAAC,oBAAoB,EAAE,OAAO,CAAC,CAAA;IAExD,MAAM,KAAK,GAAG,IAAI,KAAK,EAAE,CAAA;IAEzB,IAAI,MAA0B,CAAA;IAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,MAAM,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE;YAChC,QAAQ,EAAE,IAAI;YACd,kBAAkB,EAAE,EAAE;YACtB,6BAA6B,EAAE,IAAI;SACnC,CAAC,CAAA;QAEF,KAAK,CAAC,aAAa,CAAC,sBAAsB,CAAC,CAAA;IAC5C,CAAC;IAED,GAAG,CAAC,EAAE,CAAC,CAAA;IAEP,EAAE;IAEF,IAAI,aAAa,GAAG,EAAE,CAAA;IAEtB,KAAK,IAAI,aAAa,GAAG,CAAC,EAAE,aAAa,GAAG,MAAO,CAAC,SAAS,CAAC,MAAM,EAAE,aAAa,EAAE,EAAE,CAAC;QACvF,MAAM,QAAQ,GAAG,MAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAA;QACjD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAA;QAEhC,KAAK,IAAI,WAAW,GAAG,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC;YACvE,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAA;YAEnC,aAAa,IAAI,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAEnD,IAAI,WAAW,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtC,aAAa,IAAI,KAAK,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAA;YAC1C,CAAC;QACF,CAAC;QAED,IAAI,aAAa,GAAG,MAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClD,aAAa,IAAI,KAAM,GAAG,CAAC,MAAM,CAAC,GAAG,CAAE,KAAK,CAAA;QAC7C,CAAC;IACF,CAAC;IAED,aAAa,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAA;AAClD,CAAC;AAED,KAAK,UAAU,KAAK;IACnB,MAAM,IAAI,GAAG,mDAAmD,CAAA;IAEhE,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAA;IAEtC,MAAM,yBAAyB,GAAG,KAAK,CAAC,qBAAqB,CAAA;IAC7D,MAAM,0BAA0B,GAAG,IAAI,YAAY,EAAE,CAAA;IACrD,0BAA0B,CAAC,OAAO,GAAG,yBAAyB,CAAA;IAE9D,MAAM,aAAa,GAAG,wCAAwC,CAAC,0BAA0B,EAAE,IAAI,CAAC,CAAA;IAEhG,MAAM,CAAC,GAAG,CAAC,CAAA;AACZ,CAAC;AAED,KAAK,EAAE,CAAA;AACP,SAAS"}
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { WordSequence } from './WordSequence.js';
|
|
2
|
-
export { cldrSuppressions } from './Suppressions.js';
|
|
3
2
|
export { WordSequence, type WordEntry } from './WordSequence.js';
|
|
4
3
|
export declare function segmentText(text: string, options?: SegmentationOptions): Promise<SegmentationResult>;
|
|
5
4
|
export declare function segmentWordSequence(wordSequence: WordSequence): Promise<SegmentationResult>;
|
|
@@ -9,13 +8,13 @@ export declare function addMissingPunctuationWordsToWordSequence(wordSequence: W
|
|
|
9
8
|
originalWordsReverseMapping: Map<number, number>;
|
|
10
9
|
};
|
|
11
10
|
export interface SegmentationResult {
|
|
12
|
-
|
|
11
|
+
words: WordSequence;
|
|
13
12
|
sentences: Sentence[];
|
|
14
13
|
}
|
|
15
14
|
export declare class TextFragment {
|
|
16
15
|
wordRange: Range;
|
|
17
|
-
|
|
18
|
-
constructor(wordRange: Range,
|
|
16
|
+
words: WordSequence;
|
|
17
|
+
constructor(wordRange: Range, words: WordSequence);
|
|
19
18
|
get text(): string;
|
|
20
19
|
get charRange(): Range;
|
|
21
20
|
}
|
package/dist/TextSegmentation.js
CHANGED
|
@@ -1,21 +1,34 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { cldrSuppressions, leadingApostropheContractionSuppressions, nounSuppressions, tldSuppressions } from './Suppressions.js';
|
|
1
|
+
import { buildWordOrNumberPattern as buildWordSplitterPattern, phraseSeparatorRegExp, sentenceSeparatorTrailingPunctuationRegExp, sentenceSeparatorRegExp, whitespacePatternRegExp, letterPatternGlobalRegExp } from './Patterns.js';
|
|
2
|
+
import { cldrSuppressions, additionalSuppressions, leadingApostropheContractionSuppressions, nounSuppressions, tldSuppressions } from './Suppressions.js';
|
|
4
3
|
import { eastAsianCharRangesRegExp } from './EastAsianCharacterPatterns.js';
|
|
5
4
|
import { WordSequence } from './WordSequence.js';
|
|
6
|
-
|
|
5
|
+
import { getShortLanguageCode } from './utilities/Utilities.js';
|
|
7
6
|
export { WordSequence } from './WordSequence.js';
|
|
7
|
+
import { buildRegExp } from 'regexp-composer';
|
|
8
|
+
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
9
|
+
// Exported methods
|
|
10
|
+
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
8
11
|
export async function segmentText(text, options) {
|
|
9
|
-
options = { ...defaultSegmentationOptions, ...(options ?? {}) };
|
|
10
12
|
const wordSequence = await splitToWords(text, options);
|
|
11
13
|
return segmentWordSequence(wordSequence);
|
|
12
14
|
}
|
|
13
15
|
export async function segmentWordSequence(wordSequence) {
|
|
14
16
|
const sentenceWordRanges = [];
|
|
17
|
+
const minimumSentenceLetterCount = 2;
|
|
15
18
|
let sentenceStartWordOffset = 0;
|
|
19
|
+
let currentSentenceLetterCount = 0;
|
|
16
20
|
for (let wordIndex = 0; wordIndex < wordSequence.length; wordIndex++) {
|
|
17
21
|
const word = wordSequence.getWordAt(wordIndex);
|
|
18
|
-
if (
|
|
22
|
+
if (currentSentenceLetterCount < minimumSentenceLetterCount) {
|
|
23
|
+
const matches = word.matchAll(letterPatternGlobalRegExp);
|
|
24
|
+
for (const match of matches) {
|
|
25
|
+
currentSentenceLetterCount += 1;
|
|
26
|
+
if (currentSentenceLetterCount >= minimumSentenceLetterCount) {
|
|
27
|
+
break;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
if (currentSentenceLetterCount >= minimumSentenceLetterCount && sentenceSeparatorRegExp.test(word)) {
|
|
19
32
|
while (wordIndex < wordSequence.length - 1) {
|
|
20
33
|
const nextWord = wordSequence.getWordAt(wordIndex + 1);
|
|
21
34
|
if (!sentenceSeparatorTrailingPunctuationRegExp.test(nextWord)) {
|
|
@@ -28,6 +41,7 @@ export async function segmentWordSequence(wordSequence) {
|
|
|
28
41
|
end: wordIndex + 1
|
|
29
42
|
});
|
|
30
43
|
sentenceStartWordOffset = wordIndex + 1;
|
|
44
|
+
currentSentenceLetterCount = 0;
|
|
31
45
|
}
|
|
32
46
|
}
|
|
33
47
|
if (sentenceStartWordOffset < wordSequence.length) {
|
|
@@ -75,7 +89,7 @@ export async function segmentWordSequence(wordSequence) {
|
|
|
75
89
|
}
|
|
76
90
|
}
|
|
77
91
|
const result = {
|
|
78
|
-
wordSequence,
|
|
92
|
+
words: wordSequence,
|
|
79
93
|
sentences,
|
|
80
94
|
};
|
|
81
95
|
return result;
|
|
@@ -86,6 +100,9 @@ export async function splitToWords(text, options) {
|
|
|
86
100
|
options = {};
|
|
87
101
|
}
|
|
88
102
|
options = { ...defaultSegmentationOptions, ...options };
|
|
103
|
+
if (options.language) {
|
|
104
|
+
options.language = getShortLanguageCode(options.language);
|
|
105
|
+
}
|
|
89
106
|
const optionsAsJson = JSON.stringify(options);
|
|
90
107
|
let wordSplitterRegExp = cachedWordSplitterRegExps.get(optionsAsJson);
|
|
91
108
|
if (!wordSplitterRegExp) {
|
|
@@ -99,7 +116,8 @@ export async function splitToWords(text, options) {
|
|
|
99
116
|
let punctuationWordStartOffset = startOffset;
|
|
100
117
|
function addPunctuationWordIfNeeded() {
|
|
101
118
|
if (charOffset > punctuationWordStartOffset) {
|
|
102
|
-
|
|
119
|
+
const wordText = text.substring(punctuationWordStartOffset, charOffset);
|
|
120
|
+
wordSequence.addWord(wordText, punctuationWordStartOffset, true);
|
|
103
121
|
punctuationWordStartOffset = charOffset;
|
|
104
122
|
}
|
|
105
123
|
}
|
|
@@ -124,7 +142,8 @@ export async function splitToWords(text, options) {
|
|
|
124
142
|
if (matchStartOffset > lastMatchEndOffset) {
|
|
125
143
|
addPunctuationWordsBetween(lastMatchEndOffset, matchStartOffset);
|
|
126
144
|
}
|
|
127
|
-
|
|
145
|
+
const wordText = text.substring(matchStartOffset, matchEndOffset);
|
|
146
|
+
wordSequence.addWord(wordText, matchStartOffset, false);
|
|
128
147
|
lastMatchEndOffset = matchEndOffset;
|
|
129
148
|
}
|
|
130
149
|
addPunctuationWordsBetween(lastMatchEndOffset, text.length);
|
|
@@ -152,36 +171,19 @@ async function postprocessEastAsianWords(containingText, wordSequence) {
|
|
|
152
171
|
}
|
|
153
172
|
const wordBreaks = [...icuSegmentation.createWordBreakIterator(word)];
|
|
154
173
|
for (let i = 0; i < wordBreaks.length - 1; i++) {
|
|
155
|
-
|
|
174
|
+
const subwordStartOffset = wordStartOffset + wordBreaks[i];
|
|
175
|
+
const subwordEndOffset = wordStartOffset + wordBreaks[i + 1];
|
|
176
|
+
const subwordText = containingText.substring(subwordStartOffset, subwordEndOffset);
|
|
177
|
+
newWordSequence.addWord(subwordText, subwordStartOffset, false);
|
|
156
178
|
}
|
|
157
179
|
}
|
|
158
180
|
else {
|
|
159
|
-
|
|
181
|
+
const wordText = containingText.substring(wordEntry.startOffset, wordEntry.endOffset);
|
|
182
|
+
newWordSequence.addWord(wordText, wordEntry.startOffset, wordEntry.isPunctuation);
|
|
160
183
|
}
|
|
161
184
|
}
|
|
162
185
|
return newWordSequence;
|
|
163
186
|
}
|
|
164
|
-
function buildWordSplitterRegExpForOptions(options) {
|
|
165
|
-
const cldrSuppressionsForLang = cldrSuppressions[options.language ?? ''] ?? [];
|
|
166
|
-
const contractionSuppressionsForLang = leadingApostropheContractionSuppressions[options.language ?? ''] ?? [];
|
|
167
|
-
const contractionSuppressionsForLangWithSingleQuote = contractionSuppressionsForLang.map(str => str.replaceAll(`'`, `’`));
|
|
168
|
-
const customSuppressions = options.customSuppressions ?? [];
|
|
169
|
-
let suppressions = [
|
|
170
|
-
...customSuppressions,
|
|
171
|
-
...cldrSuppressionsForLang,
|
|
172
|
-
...contractionSuppressionsForLang,
|
|
173
|
-
...contractionSuppressionsForLangWithSingleQuote,
|
|
174
|
-
...nounSuppressions,
|
|
175
|
-
...tldSuppressions,
|
|
176
|
-
];
|
|
177
|
-
const wordPattern = buildWordSplitterPattern([
|
|
178
|
-
...suppressions,
|
|
179
|
-
...suppressions.map(word => word.toLocaleLowerCase()),
|
|
180
|
-
...suppressions.map(word => word.toLocaleUpperCase()),
|
|
181
|
-
]);
|
|
182
|
-
const wordSplitterRegExp = buildRegExp(wordPattern, { global: true });
|
|
183
|
-
return wordSplitterRegExp;
|
|
184
|
-
}
|
|
185
187
|
// Add any missing punctuation words to a word sequence
|
|
186
188
|
export function addMissingPunctuationWordsToWordSequence(wordSequence, sourceText) {
|
|
187
189
|
const originalWordsReverseMapping = new Map();
|
|
@@ -197,7 +199,8 @@ export function addMissingPunctuationWordsToWordSequence(wordSequence, sourceTex
|
|
|
197
199
|
wordSequenceWithPunctuation.lastEntry.endOffset = charEndOffset;
|
|
198
200
|
}
|
|
199
201
|
else {
|
|
200
|
-
|
|
202
|
+
const wordText = sourceText.substring(charOffset, charEndOffset);
|
|
203
|
+
wordSequenceWithPunctuation.addWord(wordText, charOffset, true);
|
|
201
204
|
}
|
|
202
205
|
charOffset = charEndOffset;
|
|
203
206
|
}
|
|
@@ -223,6 +226,32 @@ export function addMissingPunctuationWordsToWordSequence(wordSequence, sourceTex
|
|
|
223
226
|
}
|
|
224
227
|
return { wordSequenceWithPunctuation, originalWordsReverseMapping };
|
|
225
228
|
}
|
|
229
|
+
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
230
|
+
// Helper methods
|
|
231
|
+
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
232
|
+
function buildWordSplitterRegExpForOptions(options) {
|
|
233
|
+
const cldrSuppressionsForLang = cldrSuppressions[options.language ?? ''] ?? [];
|
|
234
|
+
const extendedSuppressionsForLang = additionalSuppressions[options.language ?? ''] ?? [];
|
|
235
|
+
const contractionSuppressionsForLang = leadingApostropheContractionSuppressions[options.language ?? ''] ?? [];
|
|
236
|
+
const contractionSuppressionsForLangWithSingleQuote = contractionSuppressionsForLang.map(str => str.replaceAll(`'`, `’`));
|
|
237
|
+
const customSuppressions = options.customSuppressions ?? [];
|
|
238
|
+
let suppressions = [
|
|
239
|
+
...customSuppressions,
|
|
240
|
+
...cldrSuppressionsForLang,
|
|
241
|
+
...extendedSuppressionsForLang,
|
|
242
|
+
...contractionSuppressionsForLang,
|
|
243
|
+
...contractionSuppressionsForLangWithSingleQuote,
|
|
244
|
+
...nounSuppressions,
|
|
245
|
+
...tldSuppressions,
|
|
246
|
+
];
|
|
247
|
+
const wordPattern = buildWordSplitterPattern([
|
|
248
|
+
...suppressions,
|
|
249
|
+
...suppressions.map(word => word.toLocaleLowerCase()),
|
|
250
|
+
...suppressions.map(word => word.toLocaleUpperCase()),
|
|
251
|
+
]);
|
|
252
|
+
const wordSplitterRegExp = buildRegExp(wordPattern, { global: true });
|
|
253
|
+
return wordSplitterRegExp;
|
|
254
|
+
}
|
|
226
255
|
async function getIcuSegmentation() {
|
|
227
256
|
try {
|
|
228
257
|
const icuSegmentation = await import('@echogarden/icu-segmentation-wasm');
|
|
@@ -234,18 +263,18 @@ async function getIcuSegmentation() {
|
|
|
234
263
|
}
|
|
235
264
|
export class TextFragment {
|
|
236
265
|
wordRange;
|
|
237
|
-
|
|
238
|
-
constructor(wordRange,
|
|
266
|
+
words;
|
|
267
|
+
constructor(wordRange, words) {
|
|
239
268
|
this.wordRange = wordRange;
|
|
240
|
-
this.
|
|
269
|
+
this.words = words;
|
|
241
270
|
}
|
|
242
271
|
get text() {
|
|
243
|
-
return this.
|
|
272
|
+
return this.words.text;
|
|
244
273
|
}
|
|
245
274
|
get charRange() {
|
|
246
275
|
return {
|
|
247
|
-
start: this.
|
|
248
|
-
end: this.
|
|
276
|
+
start: this.words.firstEntry.startOffset,
|
|
277
|
+
end: this.words.lastEntry.endOffset
|
|
249
278
|
};
|
|
250
279
|
}
|
|
251
280
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"TextSegmentation.js","sourceRoot":"","sources":["../src/TextSegmentation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAC7C,OAAO,EAAE,wBAAwB,IAAI,wBAAwB,EAAE,qBAAqB,EAAE,0CAA0C,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAA;AACzM,OAAO,EAAE,gBAAgB,EAAE,wCAAwC,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AACjI,OAAO,EAAE,yBAAyB,EAAE,MAAM,iCAAiC,CAAA;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAEhD,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AACpD,OAAO,EAAE,YAAY,EAAkB,MAAM,mBAAmB,CAAA;AAEhE,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAY,EAAE,OAA6B;IAC5E,OAAO,GAAG,EAAE,GAAG,0BAA0B,EAAE,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,CAAA;IAE/D,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;IAEtD,OAAO,mBAAmB,CAAC,YAAY,CAAC,CAAA;AACzC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,YAA0B;IACnE,MAAM,kBAAkB,GAAY,EAAE,CAAA;IAEtC,IAAI,uBAAuB,GAAG,CAAC,CAAA;IAE/B,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;QAE9C,IAAI,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACxC,OAAO,SAAS,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5C,MAAM,QAAQ,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC,CAAA;gBAEtD,IAAI,CAAC,0CAA0C,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAChE,MAAK;gBACN,CAAC;gBAED,SAAS,IAAI,CAAC,CAAA;YACf,CAAC;YAED,kBAAkB,CAAC,IAAI,CAAC;gBACvB,KAAK,EAAE,uBAAuB;gBAC9B,GAAG,EAAE,SAAS,GAAG,CAAC;aAClB,CAAC,CAAA;YAEF,uBAAuB,GAAG,SAAS,GAAG,CAAC,CAAA;QACxC,CAAC;IACF,CAAC;IAED,IAAI,uBAAuB,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC;QACnD,kBAAkB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,uBAAuB,EAAE,GAAG,EAAE,YAAY,CAAC,MAAM,EAAE,CAAC,CAAA;IACtF,CAAC;IAED,MAAM,SAAS,GAAe,EAAE,CAAA;IAEhC,KAAK,MAAM,SAAS,IAAI,kBAAkB,EAAE,CAAC;QAC5C,MAAM,oBAAoB,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,CAAA;QAE/E,SAAS,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC,CAAA;IAC9D,CAAC;IAED,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QAClC,MAAM,gBAAgB,GAAY,EAAE,CAAA;QAEpC,IAAI,qBAAqB,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAA;QAClD,IAAI,qBAAqB,GAAG,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAA;QAEpD,KAAK,IAAI,SAAS,GAAG,qBAAqB,EAAE,SAAS,GAAG,qBAAqB,EAAE,SAAS,EAAE,EAAE,CAAC;YAC5F,MAAM,WAAW,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;YAErD,IAAI,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC7C,IAAI,kBAAkB,GAAG,KAAK,CAAA;gBAE9B,OAAO,SAAS,GAAG,qBAAqB,GAAG,CAAC,EAAE,CAAC;oBAC9C,MAAM,QAAQ,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC,CAAA;oBAEtD,IAAI,CAAC,0CAA0C,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAChE,MAAK;oBACN,CAAC;oBAED,IAAI,uBAAuB,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAC5C,kBAAkB,GAAG,IAAI,CAAA;oBAC1B,CAAC;oBAED,IAAI,QAAQ,KAAK,GAAG,IAAI,kBAAkB,EAAE,CAAC;wBAC5C,MAAK;oBACN,CAAC;oBAED,SAAS,IAAI,CAAC,CAAA;gBACf,CAAC;gBAED,gBAAgB,CAAC,IAAI,CAAC;oBACrB,KAAK,EAAE,qBAAqB;oBAC5B,GAAG,EAAE,SAAS,GAAG,CAAC;iBAClB,CAAC,CAAA;gBAEF,qBAAqB,GAAG,SAAS,GAAG,CAAC,CAAA;YACtC,CAAC;QACF,CAAC;QAED,IAAI,qBAAqB,GAAG,qBAAqB,EAAE,CAAC;YACnD,gBAAgB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,qBAAqB,EAAE,GAAG,EAAE,qBAAqB,EAAE,CAAC,CAAA;QACpF,CAAC;QAED,KAAK,MAAM,SAAS,IAAI,gBAAgB,EAAE,CAAC;YAC1C,MAAM,kBAAkB,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,CAAA;YAE7E,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC,CAAA;QACjE,CAAC;IACF,CAAC;IAED,MAAM,MAAM,GAAuB;QAClC,YAAY;QACZ,SAAS;KACT,CAAA;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED,MAAM,yBAAyB,GAAG,IAAI,GAAG,EAAkB,CAAA;AAE3D,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,OAA6B;IAC7E,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,GAAG,EAAE,CAAA;IACb,CAAC;IAED,OAAO,GAAG,EAAE,GAAG,0BAA0B,EAAE,GAAG,OAAO,EAAE,CAAA;IAEvD,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA;IAE7C,IAAI,kBAAkB,GAAG,yBAAyB,CAAC,GAAG,CAAC,aAAa,CAAC,CAAA;IAErE,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACzB,kBAAkB,GAAG,iCAAiC,CAAC,OAAO,CAAC,CAAA;QAE/D,yBAAyB,CAAC,GAAG,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAAA;IACjE,CAAC;IAED,IAAI,YAAY,GAAG,IAAI,YAAY,EAAE,CAAA;IAErC,SAAS,0BAA0B,CAAC,WAAmB,EAAE,SAAiB;QACzE,MAAM,wBAAwB,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,CAAA;QAEvE,IAAI,UAAU,GAAG,WAAW,CAAA;QAC5B,IAAI,0BAA0B,GAAG,WAAW,CAAA;QAE5C,SAAS,0BAA0B;YAClC,IAAI,UAAU,GAAG,0BAA0B,EAAE,CAAC;gBAC7C,YAAY,CAAC,YAAY,CAAC,IAAI,EAAE,0BAA0B,EAAE,UAAU,EAAE,IAAI,CAAC,CAAA;gBAC7E,0BAA0B,GAAG,UAAU,CAAA;YACxC,CAAC;QACF,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,wBAAwB,EAAE,CAAC;YAC7C,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;gBAClB,UAAU,IAAI,CAAC,CAAA;gBAEf,SAAQ;YACT,CAAC;YAED,0BAA0B,EAAE,CAAA;YAE5B,UAAU,IAAI,IAAI,CAAC,MAAM,CAAA;YAEzB,0BAA0B,EAAE,CAAA;QAC7B,CAAC;QAED,0BAA0B,EAAE,CAAA;IAC7B,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAA;IAErD,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAE1B,IAAI,WAAW,EAAE,CAAC;QACjB,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAQ,CAAC,CAAC,CAAC,CAAA;YACjC,MAAM,gBAAgB,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YACnC,MAAM,cAAc,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YAEjC,IAAI,gBAAgB,GAAG,kBAAkB,EAAE,CAAC;gBAC3C,0BAA0B,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAA;YACjE,CAAC;YAED,YAAY,CAAC,YAAY,CAAC,IAAI,EAAE,gBAAgB,EAAE,cAAc,EAAE,KAAK,CAAC,CAAA;YAExE,kBAAkB,GAAG,cAAc,CAAA;QACpC,CAAC;QAED,0BAA0B,CAAC,kBAAkB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IAC5D,CAAC;IAED,IAAI,OAAO,CAAC,6BAA6B,EAAE,CAAC;QAC3C,YAAY,GAAG,MAAM,yBAAyB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;IACnE,CAAC;IAED,OAAO,YAAY,CAAA;AACpB,CAAC;AAED,KAAK,UAAU,yBAAyB,CAAC,cAAsB,EAAE,YAA0B;IAC1F,MAAM,eAAe,GAAG,MAAM,kBAAkB,EAAE,CAAA;IAElD,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;QACnC,OAAO,YAAY,CAAA;IACpB,CAAC;IAED,IAAI,cAAc,GAAG,KAAK,CAAA;IAE1B,MAAM,eAAe,GAAG,IAAI,YAAY,EAAE,CAAA;IAE1C,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;QACjD,MAAM,eAAe,GAAG,SAAS,CAAC,WAAW,CAAA;QAE7C,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;QAE9C,IAAI,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1C,IAAI,CAAC,cAAc,EAAE,CAAC;gBACrB,MAAM,eAAe,CAAC,UAAU,EAAE,CAAA;gBAElC,cAAc,GAAG,IAAI,CAAA;YACtB,CAAC;YAED,MAAM,UAAU,GAAG,CAAC,GAAG,eAAe,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAA;YAErE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChD,eAAe,CAAC,YAAY,CAC3B,cAAc,EACd,eAAe,GAAG,UAAU,CAAC,CAAC,CAAC,EAC/B,eAAe,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,EACnC,KAAK,CACL,CAAA;YACF,CAAC;QACF,CAAC;aAAM,CAAC;YACP,eAAe,CAAC,YAAY,CAAC,cAAc,EAAE,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC,aAAa,CAAC,CAAA;QAClH,CAAC;IACF,CAAC;IAED,OAAO,eAAe,CAAA;AACvB,CAAC;AAED,SAAS,iCAAiC,CAAC,OAA4B;IACtE,MAAM,uBAAuB,GAAG,gBAAgB,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IAC9E,MAAM,8BAA8B,GAAG,wCAAwC,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IAC7G,MAAM,6CAA6C,GAAG,8BAA8B,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAA;IACzH,MAAM,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,EAAE,CAAA;IAE3D,IAAI,YAAY,GAAG;QAClB,GAAG,kBAAkB;QACrB,GAAG,uBAAuB;QAC1B,GAAG,8BAA8B;QACjC,GAAG,6CAA6C;QAChD,GAAG,gBAAgB;QACnB,GAAG,eAAe;KAClB,CAAA;IAED,MAAM,WAAW,GAAG,wBAAwB,CAAC;QAC5C,GAAG,YAAY;QACf,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACrD,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;KACrD,CAAC,CAAA;IAEF,MAAM,kBAAkB,GAAG,WAAW,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;IAErE,OAAO,kBAAkB,CAAA;AAC1B,CAAC;AAED,uDAAuD;AACvD,MAAM,UAAU,wCAAwC,CAAC,YAA0B,EAAE,UAAkB;IACtG,MAAM,2BAA2B,GAAG,IAAI,GAAG,EAAkB,CAAA;IAE7D,MAAM,2BAA2B,GAAG,IAAI,YAAY,EAAE,CAAA;IAEtD,SAAS,0BAA0B,CAAC,SAAiB,EAAE,iBAAyB;QAC/E,IAAI,UAAU,GAAG,iBAAiB,CAAA;QAElC,kFAAkF;QAClF,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC9B,MAAM,aAAa,GAAG,UAAU,GAAG,IAAI,CAAC,MAAM,CAAA;YAE9C,MAAM,SAAS,GAAG,2BAA2B,CAAC,SAAS,CAAA;YAEvD,IAAI,IAAI,KAAK,GAAG,IAAI,SAAS,IAAI,SAAS,CAAC,aAAa,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;gBACvF,2BAA2B,CAAC,SAAS,CAAC,IAAI,IAAI,GAAG,CAAA;gBACjD,2BAA2B,CAAC,SAAS,CAAC,SAAS,GAAG,aAAa,CAAA;YAChE,CAAC;iBAAM,CAAC;gBACP,2BAA2B,CAAC,YAAY,CAAC,UAAU,EAAE,UAAU,EAAE,aAAa,EAAE,IAAI,CAAC,CAAA;YACtF,CAAC;YAED,UAAU,GAAG,aAAa,CAAA;QAC3B,CAAC;IACF,CAAC;IAED,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,MAAM,SAAS,GAAG,YAAY,CAAC,UAAU,CAAC,SAAS,CAAC,CAAA;QACpD,MAAM,eAAe,GAAG,SAAS,CAAC,WAAW,CAAA;QAE7C,MAAM,qBAAqB,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAA;QAE/F,8FAA8F;QAC9F,IAAI,qBAAqB,KAAK,eAAe,EAAE,CAAC;YAC/C,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,qBAAqB,EAAE,eAAe,CAAC,CAAA;YAE9E,0BAA0B,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAA;QAC7D,CAAC;QAED,2BAA2B,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QACnD,2BAA2B,CAAC,GAAG,CAAC,2BAA2B,CAAC,MAAM,GAAG,CAAC,EAAE,SAAS,CAAC,CAAA;QAElF,oEAAoE;QACpE,IAAI,SAAS,KAAK,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3C,IAAI,UAAU,CAAC,MAAM,KAAK,SAAS,CAAC,SAAS,EAAE,CAAC;gBAC/C,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,SAAS,CAAC,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC,CAAA;gBAE9E,0BAA0B,CAAC,SAAS,EAAE,SAAS,CAAC,SAAS,CAAC,CAAA;YAC3D,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,EAAE,2BAA2B,EAAE,2BAA2B,EAAE,CAAA;AACpE,CAAC;AAED,KAAK,UAAU,kBAAkB;IAChC,IAAI,CAAC;QACJ,MAAM,eAAe,GAAG,MAAM,MAAM,CAAC,mCAAmC,CAAC,CAAA;QAEzE,OAAO,eAAe,CAAA;IACvB,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAA;IACjB,CAAC;AACF,CAAC;AAOD,MAAM,OAAO,YAAY;IACxB,SAAS,CAAO;IAChB,YAAY,CAAc;IAE1B,YAAY,SAAgB,EAAE,YAA0B;QACvD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;QAC1B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAA;IACjC,CAAC;IAED,IAAI,IAAI;QACP,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAA;IAC9B,CAAC;IAED,IAAI,SAAS;QACZ,OAAO;YACN,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,WAAW;YAC/C,GAAG,EAAE,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,SAAS;SAC1C,CAAA;IACF,CAAC;CACD;AAED,MAAM,OAAO,QAAS,SAAQ,YAAY;IACzC,OAAO,GAAa,EAAE,CAAA;CACtB;AAED,MAAM,OAAO,MAAO,SAAQ,YAAY;CACvC;AAaD,MAAM,CAAC,MAAM,0BAA0B,GAAwB;IAC9D,QAAQ,EAAE,EAAE;IACZ,kBAAkB,EAAE,EAAE;IACtB,6BAA6B,EAAE,IAAI;CACnC,CAAA"}
|
|
1
|
+
{"version":3,"file":"TextSegmentation.js","sourceRoot":"","sources":["../src/TextSegmentation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wBAAwB,IAAI,wBAAwB,EAAE,qBAAqB,EAAE,0CAA0C,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,yBAAyB,EAAE,MAAM,eAAe,CAAA;AACpO,OAAO,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,wCAAwC,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AACzJ,OAAO,EAAE,yBAAyB,EAAE,MAAM,iCAAiC,CAAA;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAChD,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAA;AAC/D,OAAO,EAAE,YAAY,EAAkB,MAAM,mBAAmB,CAAA;AAEhE,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAE7C,gGAAgG;AAChG,mBAAmB;AACnB,gGAAgG;AAChG,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAY,EAAE,OAA6B;IAC5E,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;IAEtD,OAAO,mBAAmB,CAAC,YAAY,CAAC,CAAA;AACzC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,YAA0B;IACnE,MAAM,kBAAkB,GAAY,EAAE,CAAA;IAEtC,MAAM,0BAA0B,GAAG,CAAC,CAAA;IAEpC,IAAI,uBAAuB,GAAG,CAAC,CAAA;IAC/B,IAAI,0BAA0B,GAAG,CAAC,CAAA;IAElC,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;QAE9C,IAAI,0BAA0B,GAAG,0BAA0B,EAAE,CAAC;YAC7D,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,yBAAyB,CAAC,CAAA;YAExD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC7B,0BAA0B,IAAI,CAAC,CAAA;gBAE/B,IAAI,0BAA0B,IAAI,0BAA0B,EAAE,CAAC;oBAC9D,MAAK;gBACN,CAAC;YACF,CAAC;QACF,CAAC;QAED,IAAI,0BAA0B,IAAI,0BAA0B,IAAI,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACpG,OAAO,SAAS,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5C,MAAM,QAAQ,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC,CAAA;gBAEtD,IAAI,CAAC,0CAA0C,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAChE,MAAK;gBACN,CAAC;gBAED,SAAS,IAAI,CAAC,CAAA;YACf,CAAC;YAED,kBAAkB,CAAC,IAAI,CAAC;gBACvB,KAAK,EAAE,uBAAuB;gBAC9B,GAAG,EAAE,SAAS,GAAG,CAAC;aAClB,CAAC,CAAA;YAEF,uBAAuB,GAAG,SAAS,GAAG,CAAC,CAAA;YACvC,0BAA0B,GAAG,CAAC,CAAA;QAC/B,CAAC;IACF,CAAC;IAED,IAAI,uBAAuB,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC;QACnD,kBAAkB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,uBAAuB,EAAE,GAAG,EAAE,YAAY,CAAC,MAAM,EAAE,CAAC,CAAA;IACtF,CAAC;IAED,MAAM,SAAS,GAAe,EAAE,CAAA;IAEhC,KAAK,MAAM,SAAS,IAAI,kBAAkB,EAAE,CAAC;QAC5C,MAAM,oBAAoB,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,CAAA;QAE/E,SAAS,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC,CAAA;IAC9D,CAAC;IAED,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QAClC,MAAM,gBAAgB,GAAY,EAAE,CAAA;QAEpC,IAAI,qBAAqB,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAA;QAClD,IAAI,qBAAqB,GAAG,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAA;QAEpD,KAAK,IAAI,SAAS,GAAG,qBAAqB,EAAE,SAAS,GAAG,qBAAqB,EAAE,SAAS,EAAE,EAAE,CAAC;YAC5F,MAAM,WAAW,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;YAErD,IAAI,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC7C,IAAI,kBAAkB,GAAG,KAAK,CAAA;gBAE9B,OAAO,SAAS,GAAG,qBAAqB,GAAG,CAAC,EAAE,CAAC;oBAC9C,MAAM,QAAQ,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC,CAAA;oBAEtD,IAAI,CAAC,0CAA0C,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAChE,MAAK;oBACN,CAAC;oBAED,IAAI,uBAAuB,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAC5C,kBAAkB,GAAG,IAAI,CAAA;oBAC1B,CAAC;oBAED,IAAI,QAAQ,KAAK,GAAG,IAAI,kBAAkB,EAAE,CAAC;wBAC5C,MAAK;oBACN,CAAC;oBAED,SAAS,IAAI,CAAC,CAAA;gBACf,CAAC;gBAED,gBAAgB,CAAC,IAAI,CAAC;oBACrB,KAAK,EAAE,qBAAqB;oBAC5B,GAAG,EAAE,SAAS,GAAG,CAAC;iBAClB,CAAC,CAAA;gBAEF,qBAAqB,GAAG,SAAS,GAAG,CAAC,CAAA;YACtC,CAAC;QACF,CAAC;QAED,IAAI,qBAAqB,GAAG,qBAAqB,EAAE,CAAC;YACnD,gBAAgB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,qBAAqB,EAAE,GAAG,EAAE,qBAAqB,EAAE,CAAC,CAAA;QACpF,CAAC;QAED,KAAK,MAAM,SAAS,IAAI,gBAAgB,EAAE,CAAC;YAC1C,MAAM,kBAAkB,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,CAAA;YAE7E,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC,CAAA;QACjE,CAAC;IACF,CAAC;IAED,MAAM,MAAM,GAAuB;QAClC,KAAK,EAAE,YAAY;QACnB,SAAS;KACT,CAAA;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED,MAAM,yBAAyB,GAAG,IAAI,GAAG,EAAkB,CAAA;AAE3D,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,OAA6B;IAC7E,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,GAAG,EAAE,CAAA;IACb,CAAC;IAED,OAAO,GAAG,EAAE,GAAG,0BAA0B,EAAE,GAAG,OAAO,EAAE,CAAA;IAEvD,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACtB,OAAO,CAAC,QAAQ,GAAG,oBAAoB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;IAC1D,CAAC;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA;IAE7C,IAAI,kBAAkB,GAAG,yBAAyB,CAAC,GAAG,CAAC,aAAa,CAAC,CAAA;IAErE,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACzB,kBAAkB,GAAG,iCAAiC,CAAC,OAAO,CAAC,CAAA;QAE/D,yBAAyB,CAAC,GAAG,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAAA;IACjE,CAAC;IAED,IAAI,YAAY,GAAG,IAAI,YAAY,EAAE,CAAA;IAErC,SAAS,0BAA0B,CAAC,WAAmB,EAAE,SAAiB;QACzE,MAAM,wBAAwB,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,CAAA;QAEvE,IAAI,UAAU,GAAG,WAAW,CAAA;QAC5B,IAAI,0BAA0B,GAAG,WAAW,CAAA;QAE5C,SAAS,0BAA0B;YAClC,IAAI,UAAU,GAAG,0BAA0B,EAAE,CAAC;gBAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,0BAA0B,EAAE,UAAU,CAAC,CAAA;gBACvE,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,0BAA0B,EAAE,IAAI,CAAC,CAAA;gBAEhE,0BAA0B,GAAG,UAAU,CAAA;YACxC,CAAC;QACF,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,wBAAwB,EAAE,CAAC;YAC7C,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;gBAClB,UAAU,IAAI,CAAC,CAAA;gBAEf,SAAQ;YACT,CAAC;YAED,0BAA0B,EAAE,CAAA;YAE5B,UAAU,IAAI,IAAI,CAAC,MAAM,CAAA;YAEzB,0BAA0B,EAAE,CAAA;QAC7B,CAAC;QAED,0BAA0B,EAAE,CAAA;IAC7B,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAA;IAErD,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAE1B,IAAI,WAAW,EAAE,CAAC;QACjB,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAQ,CAAC,CAAC,CAAC,CAAA;YACjC,MAAM,gBAAgB,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YACnC,MAAM,cAAc,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YAEjC,IAAI,gBAAgB,GAAG,kBAAkB,EAAE,CAAC;gBAC3C,0BAA0B,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAA;YACjE,CAAC;YAED,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,gBAAgB,EAAE,cAAc,CAAC,CAAA;YACjE,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAA;YAEvD,kBAAkB,GAAG,cAAc,CAAA;QACpC,CAAC;QAED,0BAA0B,CAAC,kBAAkB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IAC5D,CAAC;IAED,IAAI,OAAO,CAAC,6BAA6B,EAAE,CAAC;QAC3C,YAAY,GAAG,MAAM,yBAAyB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;IACnE,CAAC;IAED,OAAO,YAAY,CAAA;AACpB,CAAC;AAED,KAAK,UAAU,yBAAyB,CAAC,cAAsB,EAAE,YAA0B;IAC1F,MAAM,eAAe,GAAG,MAAM,kBAAkB,EAAE,CAAA;IAElD,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;QACnC,OAAO,YAAY,CAAA;IACpB,CAAC;IAED,IAAI,cAAc,GAAG,KAAK,CAAA;IAE1B,MAAM,eAAe,GAAG,IAAI,YAAY,EAAE,CAAA;IAE1C,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;QACjD,MAAM,eAAe,GAAG,SAAS,CAAC,WAAW,CAAA;QAE7C,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;QAE9C,IAAI,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1C,IAAI,CAAC,cAAc,EAAE,CAAC;gBACrB,MAAM,eAAe,CAAC,UAAU,EAAE,CAAA;gBAElC,cAAc,GAAG,IAAI,CAAA;YACtB,CAAC;YAED,MAAM,UAAU,GAAG,CAAC,GAAG,eAAe,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAA;YAErE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChD,MAAM,kBAAkB,GAAG,eAAe,GAAG,UAAU,CAAC,CAAC,CAAC,CAAA;gBAC1D,MAAM,gBAAgB,GAAG,eAAe,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;gBAE5D,MAAM,WAAW,GAAG,cAAc,CAAC,SAAS,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAA;gBAElF,eAAe,CAAC,OAAO,CACtB,WAAW,EACX,kBAAkB,EAClB,KAAK,CACL,CAAA;YACF,CAAC;QACF,CAAC;aAAM,CAAC;YACP,MAAM,QAAQ,GAAG,cAAc,CAAC,SAAS,CAAC,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,SAAS,CAAC,CAAA;YAErF,eAAe,CAAC,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,aAAa,CAAC,CAAA;QAClF,CAAC;IACF,CAAC;IAED,OAAO,eAAe,CAAA;AACvB,CAAC;AAED,uDAAuD;AACvD,MAAM,UAAU,wCAAwC,CAAC,YAA0B,EAAE,UAAkB;IACtG,MAAM,2BAA2B,GAAG,IAAI,GAAG,EAAkB,CAAA;IAE7D,MAAM,2BAA2B,GAAG,IAAI,YAAY,EAAE,CAAA;IAEtD,SAAS,0BAA0B,CAAC,SAAiB,EAAE,iBAAyB;QAC/E,IAAI,UAAU,GAAG,iBAAiB,CAAA;QAElC,kFAAkF;QAClF,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC9B,MAAM,aAAa,GAAG,UAAU,GAAG,IAAI,CAAC,MAAM,CAAA;YAE9C,MAAM,SAAS,GAAG,2BAA2B,CAAC,SAAS,CAAA;YAEvD,IAAI,IAAI,KAAK,GAAG,IAAI,SAAS,IAAI,SAAS,CAAC,aAAa,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;gBACvF,2BAA2B,CAAC,SAAS,CAAC,IAAI,IAAI,GAAG,CAAA;gBACjD,2BAA2B,CAAC,SAAS,CAAC,SAAS,GAAG,aAAa,CAAA;YAChE,CAAC;iBAAM,CAAC;gBACP,MAAM,QAAQ,GAAG,UAAU,CAAC,SAAS,CAAC,UAAU,EAAE,aAAa,CAAC,CAAA;gBAEhE,2BAA2B,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,EAAE,IAAI,CAAC,CAAA;YAChE,CAAC;YAED,UAAU,GAAG,aAAa,CAAA;QAC3B,CAAC;IACF,CAAC;IAED,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,MAAM,SAAS,GAAG,YAAY,CAAC,UAAU,CAAC,SAAS,CAAC,CAAA;QACpD,MAAM,eAAe,GAAG,SAAS,CAAC,WAAW,CAAA;QAE7C,MAAM,qBAAqB,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAA;QAE/F,8FAA8F;QAC9F,IAAI,qBAAqB,KAAK,eAAe,EAAE,CAAC;YAC/C,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,qBAAqB,EAAE,eAAe,CAAC,CAAA;YAE9E,0BAA0B,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAA;QAC7D,CAAC;QAED,2BAA2B,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QACnD,2BAA2B,CAAC,GAAG,CAAC,2BAA2B,CAAC,MAAM,GAAG,CAAC,EAAE,SAAS,CAAC,CAAA;QAElF,oEAAoE;QACpE,IAAI,SAAS,KAAK,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3C,IAAI,UAAU,CAAC,MAAM,KAAK,SAAS,CAAC,SAAS,EAAE,CAAC;gBAC/C,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,SAAS,CAAC,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC,CAAA;gBAE9E,0BAA0B,CAAC,SAAS,EAAE,SAAS,CAAC,SAAS,CAAC,CAAA;YAC3D,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,EAAE,2BAA2B,EAAE,2BAA2B,EAAE,CAAA;AACpE,CAAC;AAED,gGAAgG;AAChG,iBAAiB;AACjB,gGAAgG;AAChG,SAAS,iCAAiC,CAAC,OAA4B;IACtE,MAAM,uBAAuB,GAAG,gBAAgB,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IAC9E,MAAM,2BAA2B,GAAG,sBAAsB,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IACxF,MAAM,8BAA8B,GAAG,wCAAwC,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IAC7G,MAAM,6CAA6C,GAAG,8BAA8B,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAA;IACzH,MAAM,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,EAAE,CAAA;IAE3D,IAAI,YAAY,GAAG;QAClB,GAAG,kBAAkB;QACrB,GAAG,uBAAuB;QAC1B,GAAG,2BAA2B;QAC9B,GAAG,8BAA8B;QACjC,GAAG,6CAA6C;QAChD,GAAG,gBAAgB;QACnB,GAAG,eAAe;KAClB,CAAA;IAED,MAAM,WAAW,GAAG,wBAAwB,CAAC;QAC5C,GAAG,YAAY;QACf,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACrD,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;KACrD,CAAC,CAAA;IAEF,MAAM,kBAAkB,GAAG,WAAW,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;IAErE,OAAO,kBAAkB,CAAA;AAC1B,CAAC;AAED,KAAK,UAAU,kBAAkB;IAChC,IAAI,CAAC;QACJ,MAAM,eAAe,GAAG,MAAM,MAAM,CAAC,mCAAmC,CAAC,CAAA;QAEzE,OAAO,eAAe,CAAA;IACvB,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAA;IACjB,CAAC;AACF,CAAC;AAUD,MAAM,OAAO,YAAY;IACxB,SAAS,CAAO;IAChB,KAAK,CAAc;IAEnB,YAAY,SAAgB,EAAE,KAAmB;QAChD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;QAC1B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAA;IACnB,CAAC;IAED,IAAI,IAAI;QACP,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAA;IACvB,CAAC;IAED,IAAI,SAAS;QACZ,OAAO;YACN,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,WAAW;YACxC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,SAAS;SACnC,CAAA;IACF,CAAC;CACD;AAED,MAAM,OAAO,QAAS,SAAQ,YAAY;IACzC,OAAO,GAAa,EAAE,CAAA;CACtB;AAED,MAAM,OAAO,MAAO,SAAQ,YAAY;CACvC;AAaD,MAAM,CAAC,MAAM,0BAA0B,GAAwB;IAC9D,QAAQ,EAAE,EAAE;IACZ,kBAAkB,EAAE,EAAE;IACtB,6BAA6B,EAAE,IAAI;CACnC,CAAA"}
|
package/dist/WordSequence.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export declare class WordSequence {
|
|
2
2
|
entries: WordEntry[];
|
|
3
|
-
|
|
3
|
+
addWord(text: string, textStartOffset: number, isPunctuation: boolean): void;
|
|
4
4
|
getWordRange(startIndex: number, endIndex: number): string[];
|
|
5
5
|
iterateWordRange(startIndex: number, endIndex: number): Generator<string, void, unknown>;
|
|
6
6
|
getWordAt(index: number): string;
|
|
@@ -9,11 +9,13 @@ export declare class WordSequence {
|
|
|
9
9
|
getEntryAt(index: number): WordEntry;
|
|
10
10
|
clone(): WordSequence;
|
|
11
11
|
slice(startIndex: number, endIndex: number): WordSequence;
|
|
12
|
-
get
|
|
12
|
+
get wordArray(): string[];
|
|
13
13
|
get firstWord(): string;
|
|
14
14
|
get lastWord(): string;
|
|
15
15
|
get firstEntry(): WordEntry;
|
|
16
16
|
get lastEntry(): WordEntry;
|
|
17
|
+
get punctuationEntries(): WordEntry[];
|
|
18
|
+
get punctuationWords(): string[];
|
|
17
19
|
get nonPunctuationEntries(): WordEntry[];
|
|
18
20
|
get nonPunctuationWords(): string[];
|
|
19
21
|
get text(): string;
|
|
@@ -21,7 +23,7 @@ export declare class WordSequence {
|
|
|
21
23
|
}
|
|
22
24
|
export interface WordEntry {
|
|
23
25
|
text: string;
|
|
24
|
-
isPunctuation: boolean;
|
|
25
26
|
startOffset: number;
|
|
26
27
|
endOffset: number;
|
|
28
|
+
isPunctuation: boolean;
|
|
27
29
|
}
|
package/dist/WordSequence.js
CHANGED
|
@@ -1,16 +1,17 @@
|
|
|
1
1
|
export class WordSequence {
|
|
2
2
|
entries = [];
|
|
3
|
-
|
|
4
|
-
const
|
|
3
|
+
addWord(text, textStartOffset, isPunctuation) {
|
|
4
|
+
const startOffset = textStartOffset;
|
|
5
|
+
const endOffset = startOffset + text.length;
|
|
5
6
|
this.entries.push({
|
|
6
|
-
text
|
|
7
|
+
text,
|
|
7
8
|
startOffset,
|
|
8
9
|
endOffset,
|
|
9
10
|
isPunctuation
|
|
10
11
|
});
|
|
11
12
|
}
|
|
12
13
|
getWordRange(startIndex, endIndex) {
|
|
13
|
-
return
|
|
14
|
+
return this.getEntryRange(startIndex, endIndex).map(entry => entry.text);
|
|
14
15
|
}
|
|
15
16
|
*iterateWordRange(startIndex, endIndex) {
|
|
16
17
|
for (let i = startIndex; i < endIndex; i++) {
|
|
@@ -39,7 +40,7 @@ export class WordSequence {
|
|
|
39
40
|
slicedSequence.entries = this.entries.slice(startIndex, endIndex);
|
|
40
41
|
return slicedSequence;
|
|
41
42
|
}
|
|
42
|
-
get
|
|
43
|
+
get wordArray() {
|
|
43
44
|
return this.entries.map(entry => entry.text);
|
|
44
45
|
}
|
|
45
46
|
get firstWord() {
|
|
@@ -54,6 +55,12 @@ export class WordSequence {
|
|
|
54
55
|
get lastEntry() {
|
|
55
56
|
return this.entries[this.length - 1];
|
|
56
57
|
}
|
|
58
|
+
get punctuationEntries() {
|
|
59
|
+
return this.entries.filter(entry => entry.isPunctuation === true);
|
|
60
|
+
}
|
|
61
|
+
get punctuationWords() {
|
|
62
|
+
return this.entries.filter(entry => entry.isPunctuation === true).map(entry => entry.text);
|
|
63
|
+
}
|
|
57
64
|
get nonPunctuationEntries() {
|
|
58
65
|
return this.entries.filter(entry => entry.isPunctuation === false);
|
|
59
66
|
}
|
|
@@ -61,7 +68,7 @@ export class WordSequence {
|
|
|
61
68
|
return this.entries.filter(entry => entry.isPunctuation === false).map(entry => entry.text);
|
|
62
69
|
}
|
|
63
70
|
get text() {
|
|
64
|
-
return this.
|
|
71
|
+
return this.wordArray.join('');
|
|
65
72
|
}
|
|
66
73
|
get length() {
|
|
67
74
|
return this.entries.length;
|
package/dist/WordSequence.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"WordSequence.js","sourceRoot":"","sources":["../src/WordSequence.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,YAAY;IACxB,OAAO,GAAgB,EAAE,CAAA;IAEzB,
|
|
1
|
+
{"version":3,"file":"WordSequence.js","sourceRoot":"","sources":["../src/WordSequence.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,YAAY;IACxB,OAAO,GAAgB,EAAE,CAAA;IAEzB,OAAO,CAAC,IAAY,EAAE,eAAuB,EAAE,aAAsB;QACpE,MAAM,WAAW,GAAG,eAAe,CAAA;QACnC,MAAM,SAAS,GAAG,WAAW,GAAG,IAAI,CAAC,MAAM,CAAA;QAE3C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YACjB,IAAI;YACJ,WAAW;YACX,SAAS;YACT,aAAa;SACb,CAAC,CAAA;IACH,CAAC;IAED,YAAY,CAAC,UAAkB,EAAE,QAAgB;QAChD,OAAO,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IACzE,CAAC;IAED,CAAC,gBAAgB,CAAC,UAAkB,EAAE,QAAgB;QACrD,KAAK,IAAI,CAAC,GAAG,UAAU,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA;QACxB,CAAC;IACF,CAAC;IAED,SAAS,CAAC,KAAa;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAA;IAChC,CAAC;IAED,aAAa,CAAC,UAAkB,EAAE,QAAgB;QACjD,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;IAChD,CAAC;IAED,CAAC,iBAAiB,CAAC,UAAkB,EAAE,QAAgB;QACtD,KAAK,IAAI,CAAC,GAAG,UAAU,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;QACzB,CAAC;IACF,CAAC;IAED,UAAU,CAAC,KAAa;QACvB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IAC3B,CAAC;IAED,KAAK;QACJ,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IAClC,CAAC;IAED,KAAK,CAAC,UAAkB,EAAE,QAAgB;QACzC,MAAM,cAAc,GAAG,IAAI,YAAY,EAAE,CAAA;QAEzC,cAAc,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;QAEjE,OAAO,cAAc,CAAA;IACtB,CAAC;IAED,IAAI,SAAS;QACZ,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC7C,CAAC;IAED,IAAI,SAAS;QACZ,OAAO,IAAI,CAAC,UAAU,EAAE,IAAI,CAAA;IAC7B,CAAC;IAED,IAAI,QAAQ;QACX,OAAO,IAAI,CAAC,SAAS,EAAE,IAAI,CAAA;IAC5B,CAAC;IAED,IAAI,UAAU;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;IACvB,CAAC;IAED,IAAI,SAAS;QACZ,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACrC,CAAC;IAED,IAAI,kBAAkB;QACrB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,aAAa,KAAK,IAAI,CAAC,CAAA;IAClE,CAAC;IAED,IAAI,gBAAgB;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,aAAa,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC3F,CAAC;IAED,IAAI,qBAAqB;QACxB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,aAAa,KAAK,KAAK,CAAC,CAAA;IACnE,CAAC;IAED,IAAI,mBAAmB;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,aAAa,KAAK,KAAK,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC5F,CAAC;IAED,IAAI,IAAI;QACP,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC/B,CAAC;IAED,IAAI,MAAM;QACT,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;IAC3B,CAAC;CACD"}
|