RubyGems - opener-tokenizer-base - Versions diffs - 1.0.0 - Mend

opener-tokenizer-base 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (44) hide show

checksums.yaml +7 -0
data/README.md +148 -0
data/bin/tokenizer-base +5 -0
data/bin/tokenizer-de +5 -0
data/bin/tokenizer-en +5 -0
data/bin/tokenizer-es +5 -0
data/bin/tokenizer-fr +5 -0
data/bin/tokenizer-it +5 -0
data/bin/tokenizer-nl +5 -0
data/core/lib/Data/OptList.pm +256 -0
data/core/lib/Params/Util.pm +866 -0
data/core/lib/Sub/Exporter.pm +1101 -0
data/core/lib/Sub/Exporter/Cookbook.pod +309 -0
data/core/lib/Sub/Exporter/Tutorial.pod +280 -0
data/core/lib/Sub/Exporter/Util.pm +354 -0
data/core/lib/Sub/Install.pm +329 -0
data/core/lib/Time/Stamp.pm +808 -0
data/core/load-prefixes.pl +43 -0
data/core/nonbreaking_prefixes/abbreviation_list.kaf +0 -0
data/core/nonbreaking_prefixes/abbreviation_list.txt +444 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.ca +533 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.de +781 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.el +448 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.en +564 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.es +758 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.fr +1027 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.is +697 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.it +641 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.nl +739 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.pl +729 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.pt +656 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.ro +484 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.ru +705 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.sk +920 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.sl +524 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.sv +492 -0
data/core/split-sentences.pl +114 -0
data/core/text-fixer.pl +169 -0
data/core/tokenizer-cli.pl +363 -0
data/core/tokenizer.pl +145 -0
data/lib/opener/tokenizers/base.rb +84 -0
data/lib/opener/tokenizers/base/version.rb +8 -0
data/opener-tokenizer-base.gemspec +25 -0
metadata +134 -0

data/core/nonbreaking_prefixes/nonbreaking_prefix.sk ADDED

@@ -0,0 +1,920 @@
+Bc
+Mgr
+RNDr
+PharmDr
+PhDr
+JUDr
+PaedDr
+ThDr
+Ing
+MUDr
+MDDr
+MVDr
+Dr
+ThLic
+PhD
+ArtD
+ThDr
+Dr
+DrSc
+CSs
+prof
+obr
+Obr
+Č
+č
+absol
+adj
+admin
+adr
+Adr
+adv
+advok
+afr
+ak
+akad
+akc
+akuz
+et
+al
+alch
+amer
+anat
+angl
+Angl
+anglosas
+anorg
+ap
+apod
+arch
+archeol
+archit
+arg
+art
+astr
+astrol
+astron
+atp
+atď
+austr
+Austr
+aut
+belg
+Belg
+bibl
+Bibl
+biol
+bot
+bud
+bás
+býv
+cest
+chem
+cirk
+csl
+čs
+Čs
+dat
+dep
+det
+dial
+diaľ
+dipl
+distrib
+dokl
+dosl
+dopr
+dram
+duš
+dv
+dvojčl
+dór
+ekol
+ekon
+el
+elektr
+elektrotech
+energet
+epic
+est
+etc
+etonym
+eufem
+európ
+Európ
+ev
+evid
+expr
+fa
+fam
+farm
+fem
+feud
+fil
+filat
+filoz
+fi
+fon
+form
+fot
+fr
+Fr
+franc
+Franc
+fraz
+fut
+fyz
+fyziol
+garb
+gen
+genet
+genpor
+geod
+geogr
+geol
+geom
+germ
+gr
+Gr
+gréc
+Gréc
+gréckokat
+hebr
+herald
+hist
+hlav
+hosp
+hromad
+hud
+hypok
+ident
+i.e
+ident
+imp
+impf
+indoeur
+inf
+inform
+instr
+int
+interj
+inšt
+inštr
+iron
+jap
+Jap
+jaz
+jedn
+juhoamer
+juhových
+juhozáp
+juž
+kanad
+Kanad
+kanc
+kapit
+kpt
+kart
+katastr
+knih
+kniž
+komp
+konj
+konkr
+kozmet
+krajč
+kresť
+kt
+kuch
+lat
+latinskoamer
+lek
+lex
+lingv
+lit
+litur
+log
+lok
+max
+Max
+maď
+Maď
+medzinár
+mest
+metr
+mil
+Mil
+min
+Min
+miner
+ml
+mld
+mn
+mod
+mytol
+napr
+nar
+Nar
+nasl
+nedok
+neg
+negat
+neklas
+nem
+Nem
+neodb
+neos
+neskl
+nesklon
+nespis
+nespráv
+neved
+než
+niekt
+niž
+nom
+náb
+nákl
+námor
+nár
+obch
+obj
+obv
+obyč
+obč
+občian
+odb
+odd
+ods
+ojed
+okr
+Okr
+opt
+opyt
+org
+os
+osob
+ot
+ovoc
+par
+part
+pejor
+pers
+pf
+Pf
+P.f
+p.f
+pl
+Plk
+pod
+podst
+pokl
+polit
+politol
+polygr
+pomn
+popl
+por
+porad
+porov
+posch
+potrav
+použ
+poz
+pozit
+poľ
+poľno
+poľnohosp
+poľov
+pošt
+pož
+prac
+predl
+pren
+prep
+preuk
+priezv
+Priezv
+privl
+prof
+práv
+príd
+príj
+prík
+príp
+prír
+prísl
+príslov
+príč
+psych
+publ
+pís
+písm
+pôv
+refl
+reg
+rep
+resp
+rozk
+rozlič
+rozpráv
+roč
+Roč
+ryb
+rádiotech
+rím
+samohl
+semest
+sev
+severoamer
+severových
+severozáp
+sg
+skr
+skup
+sl
+Sloven
+soc
+soch
+sociol
+sp
+spol
+Spol
+spoloč
+spoluhl
+správ
+spôs
+st
+star
+starogréc
+starorím
+s.r.o
+stol
+stor
+str
+stredoamer
+stredoškol
+subj
+subst
+superl
+sv
+sz
+súkr
+súp
+súvzť
+tal
+Tal
+tech
+tel
+Tel
+telef
+teles
+telev
+teol
+trans
+turist
+tuzem
+typogr
+tzn
+tzv
+ukaz
+ul
+Ul
+umel
+univ
+ust
+ved
+vedľ
+verb
+veter
+vin
+viď
+vl
+vod
+vodohosp
+pnl
+vulg
+vyj
+vys
+vysokoškol
+vzťaž
+vôb
+vých
+výd
+výrob
+výsk
+výsl
+výtv
+výtvar
+význ
+včel
+vš
+všeob
+zahr
+zar
+zariad
+zast
+zastar
+zastaráv
+zb
+zdravot
+združ
+zjemn
+zlat
+zn
+Zn
+zool
+zr
+zried
+zv
+záhr
+zák
+zákl
+zám
+záp
+západoeur
+zázn
+územ
+účt
+čast
+čes
+Čes
+čl
+čísl
+živ
+pr
+fak
+Kr
+p.n.l
+A
+B
+C
+D
+E
+F
+G
+H
+I
+J
+K
+L
+M
+N
+O
+P
+Q
+R
+S
+T
+U
+V
+W
+X
+Y
+Z
+#unified abbreviation list
+Acad
+Adj
+Adm
+Adv
+Affl
+Apr
+Art
+Asst
+Av
+Avg
+B.ches-du-Rh
+Bart
+Bco
+Bldg
+Brig
+Bros
+C.a
+C.p.c.n
+Ca
+Capt
+Cdt
+Cf
+Ch.-Mme
+Chap
+Cie
+Cmdr
+Col
+Comdr
+Con
+Corp
+Cpl
+DR
+DRA
+Da
+Dec
+Dep
+Dn
+Dr
+Dra
+Dras
+Drs
+Eng
+Enga
+Engas
+Engos
+Ens
+Ets
+Euro
+Ev
+Ex
+Excmo
+Exmo
+Exo
+Fa
+Fco
+Feb
+Fig
+Fr
+Gar
+Gen
+Gir
+Gl
+Gov
+Hno
+Hon
+Hosp
+Hr
+Ilmo
+Insp
+J.-C
+Jan
+Jeu
+Jr
+Jul
+Jun
+Lda
+Lieut
+Lt
+Lun
+MM
+MR
+MRS
+MS
+MSc
+Maj
+Mar
+Me
+Mej
+Mer
+Mes
+Messrs
+Mgr
+Mgrs
+Mll
+Mlle
+Mlle(s)
+Mme
+Mme(s)
+Mr
+Mrs
+Ms
+Msgr
+Mw
+Nov
+Npr
+Nr
+O.d.J
+Okt
+Op
+Ord
+Oz
+P
+P.D
+P.ej
+P.p.c
+Pas
+Pfc
+Ph
+Prim
+Prof
+Pte
+Pts
+Pvt
+Rep
+Reps
+Res
+Rev
+Revd
+Rh
+Riv
+Rt
+S.Em
+S.Exc
+S.a.r.l
+Sen
+Sens
+Sep
+Sept
+Sfc
+Sgt
+SGT
+Sl
+Sr
+Sra
+Sras
+Srs
+Srta
+St
+ST
+Sta
+Ste
+Sto
+Supt
+Surg
+Tj
+Tr
+Ud
+Uds
+V.Exc
+Vd
+Vda
+Vds
+Vz
+Z.D
+Z.D.H
+Z.E
+Z.Em
+Z.H
+Z.K.H
+Z.K.M
+Z.M
+a
+a./s
+a.C
+a.g.v
+a.l
+abrev
+abs
+ac
+acc
+acron
+adj
+adm
+adr
+adv
+alt
+anal
+anat
+angl
+appos
+apr
+apr
+asc
+atm
+auj
+aux
+av
+avg
+avr
+b
+b.a.o
+b.a.p
+b.a.r
+bacc
+bat
+bc
+bd
+bde
+bgen
+bijv
+bijz
+br
+bv
+c
+c.-a-d
+c.a.f
+c.i
+cc
+cf
+cft
+ch
+ch.-l
+chbre
+chbs
+chf
+col
+coll
+cpl
+cpt
+cpte
+cta
+d
+d.c
+d.w.z
+dcha
+dec
+def
+dem
+dep
+dept
+dhr
+dipl
+dispo
+div
+dpto
+dr
+dr.h.c
+dra
+dras
+drs
+ds
+dz
+e.c
+e.g
+e.g
+e.k
+eccles
+ecol
+econ
+ed
+ej
+env
+ep
+eq
+et
+etc
+ev
+ex
+exmo
+exo
+exp
+expo
+f.a.c
+fa
+fam
+fasc
+fbg
+feb
+fem
+fevr
+ff
+fl
+fol
+fr
+fs
+fut
+gd
+gde
+gdes
+gds
+gen
+gl
+grd
+h.-t
+hab
+i.e
+i.p.v
+i.s.m
+i.t.t
+i.v.m
+ibid
+id
+imp
+ing
+ir
+iron
+itd
+itn
+itp
+izq
+j
+janv
+jhr
+jkvr
+jr
+l
+lat
+lex
+lgen
+lib
+lieut
+liv
+lkol
+loc
+lof
+m
+m.a.w
+m.b.t
+m.b.v
+m.h.o
+m.i
+m.i.v
+maj
+mar
+mas
+max
+med
+mevr
+min
+mll
+mr
+ms
+mtr
+mtrs
+n
+n
+n.f
+n.f.pl
+n.m
+n.m.pl
+npr
+o
+o.b.s
+obs
+oct
+okt
+ord
+oz
+p
+p
+p.a
+p.ej
+p.ex
+p.g.c.d
+p.i
+p.j
+p.m
+p.o
+p.p
+p.p.c.d
+p.p.c.m
+p.pa
+p.pr
+pl
+plv
+poe
+pp
+pp
+pr
+pr
+pres
+prev
+prof
+px
+q.s
+qqch
+qqf
+qqn
+qqns
+r.-de-ch
+r.p.m
+rc
+rd
+ref
+refl
+reg
+rev
+ro
+rte
+s
+s
+s.a
+s.b.f
+s.d
+s.e
+s.l
+s.l.n.d
+s.l.p
+s.t.p
+s.v.p
+s/c
+sc
+sf
+sgt
+sl
+sr
+sra
+sras
+srs
+ss
+sto
+t
+t.s.v.p
+tec
+tel
+terr
+tg
+tint
+tit
+tj
+tr
+travx
+v
+v.intr
+v.tr
+v.w.t
+var
+vs
+vta
+vx
+z.v
+zool
+Št
+št