RubyGems - opener-tokenizer-base - Versions diffs - 1.0.0 - Mend

opener-tokenizer-base 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (44) hide show

checksums.yaml +7 -0
data/README.md +148 -0
data/bin/tokenizer-base +5 -0
data/bin/tokenizer-de +5 -0
data/bin/tokenizer-en +5 -0
data/bin/tokenizer-es +5 -0
data/bin/tokenizer-fr +5 -0
data/bin/tokenizer-it +5 -0
data/bin/tokenizer-nl +5 -0
data/core/lib/Data/OptList.pm +256 -0
data/core/lib/Params/Util.pm +866 -0
data/core/lib/Sub/Exporter.pm +1101 -0
data/core/lib/Sub/Exporter/Cookbook.pod +309 -0
data/core/lib/Sub/Exporter/Tutorial.pod +280 -0
data/core/lib/Sub/Exporter/Util.pm +354 -0
data/core/lib/Sub/Install.pm +329 -0
data/core/lib/Time/Stamp.pm +808 -0
data/core/load-prefixes.pl +43 -0
data/core/nonbreaking_prefixes/abbreviation_list.kaf +0 -0
data/core/nonbreaking_prefixes/abbreviation_list.txt +444 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.ca +533 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.de +781 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.el +448 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.en +564 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.es +758 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.fr +1027 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.is +697 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.it +641 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.nl +739 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.pl +729 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.pt +656 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.ro +484 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.ru +705 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.sk +920 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.sl +524 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.sv +492 -0
data/core/split-sentences.pl +114 -0
data/core/text-fixer.pl +169 -0
data/core/tokenizer-cli.pl +363 -0
data/core/tokenizer.pl +145 -0
data/lib/opener/tokenizers/base.rb +84 -0
data/lib/opener/tokenizers/base/version.rb +8 -0
data/opener-tokenizer-base.gemspec +25 -0
metadata +134 -0

data/core/nonbreaking_prefixes/nonbreaking_prefix.de ADDED

@@ -0,0 +1,781 @@
+#Anything in this file, followed by a period (and an upper-case word), does NOT indicate an end-of-sentence marker.
+#Special cases are included for prefixes that ONLY appear before 0-9 numbers.
+#any single upper case letter  followed by a period is not a sentence ender (excluding I occasionally, but we leave it in)
+#usually upper case letters are initials in a name
+#no german words end in single lower-case letters, so we throw those in too.
+A
+B
+C
+D
+E
+F
+G
+H
+I
+J
+K
+L
+M
+N
+O
+P
+Q
+R
+S
+T
+U
+V
+W
+X
+Y
+Z
+a
+b
+c
+d
+e
+f
+g
+h
+i
+j
+k
+l
+m
+n
+o
+p
+q
+r
+s
+t
+u
+v
+w
+x
+y
+z
+#Roman Numerals. A dot after one of these is not a sentence break in German.
+I
+II
+III
+IV
+V
+VI
+VII
+VIII
+IX
+X
+XI
+XII
+XIII
+XIV
+XV
+XVI
+XVII
+XVIII
+XIX
+XX
+i
+ii
+iii
+iv
+v
+vi
+vii
+viii
+ix
+x
+xi
+xii
+xiii
+xiv
+xv
+xvi
+xvii
+xviii
+xix
+xx
+#Titles and Honorifics
+Adj
+Adm
+Adv
+Asst
+Bart
+Bldg
+Brig
+Bros
+Capt
+Cmdr
+Col
+Comdr
+Con
+Corp
+Cpl
+DR
+Dr
+Ens
+Gen
+Gov
+Hon
+Hosp
+Insp
+Lt
+MM
+MR
+MRS
+MS
+Maj
+Messrs
+Mlle
+Mme
+Mr
+Mrs
+Ms
+Msgr
+Op
+Ord
+Pfc
+Ph
+Prof
+Pvt
+Rep
+Reps
+Res
+Rev
+Rt
+Sen
+Sens
+Sfc
+Sgt
+Sr
+St
+Supt
+Surg
+#Misc symbols
+Mio
+Mrd
+bzw
+v
+vs
+usw
+d.h
+z.B
+u.a
+etc
+Mrd
+MwSt
+ggf
+d.J
+D.h
+m.E
+vgl
+I.F
+z.T
+sogen
+ff
+u.E
+g.U
+g.g.A
+c.-à-d
+Buchst
+u.s.w
+sog
+u.ä
+Std
+evtl
+Zt
+Chr
+u.U
+o.ä
+Ltd
+b.A
+z.Zt
+spp
+sen
+SA
+k.o
+jun
+i.H.v
+dgl
+dergl
+Co
+zzt
+usf
+s.p.a
+Dkr
+Corp
+bzgl
+BSE
+#Number indicators
+# add #NUMERIC_ONLY# after the word if it should ONLY be non-breaking when a 0-9 digit follows it
+No
+Nos
+Art
+Nr
+pp
+ca
+Ca
+#Ordinals are done with . in German - "1." = "1st" in English
+1
+2
+3
+4
+5
+6
+7
+8
+9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+19
+20
+21
+22
+23
+24
+25
+26
+27
+28
+29
+30
+31
+32
+33
+34
+35
+36
+37
+38
+39
+40
+41
+42
+43
+44
+45
+46
+47
+48
+49
+50
+51
+52
+53
+54
+55
+56
+57
+58
+59
+60
+61
+62
+63
+64
+65
+66
+67
+68
+69
+70
+71
+72
+73
+74
+75
+76
+77
+78
+79
+80
+81
+82
+83
+84
+85
+86
+87
+88
+89
+90
+91
+92
+93
+94
+95
+96
+97
+98
+99
+#unified abbreviation list
+Acad
+Adj
+Adm
+Adv
+Affl
+Apr
+Art
+Asst
+Aug
+Av
+Avg
+B.ches-du-Rh
+Bart
+Bco
+Bldg
+Brig
+Bros
+C.a
+C.p.c.n
+Ca
+Capt
+Cdt
+Cf
+Ch.-Mme
+Chap
+Cie
+Cmdr
+Col
+Comdr
+Con
+Corp
+Cpl
+DR
+DRA
+Da
+Dec
+Dep
+Dez
+Dn
+Dr
+Dra
+Dras
+Drs
+Eng
+Enga
+Engas
+Engos
+Ens
+Ets
+Euro
+Ev
+Ex
+Excmo
+Exmo
+Exo
+Fa
+Fco
+Feb
+Fig
+Fr
+Gar
+Gen
+Gir
+Gl
+Gov
+Hno
+Hon
+Hosp
+Hr
+Ilmo
+Insp
+J.-C
+Jan
+Jän
+Jeu
+Jr
+Jul
+Jun
+Lda
+Lieut
+Lt
+Lun
+MM
+MR
+MRS
+MS
+MSc
+Maj
+Mar
+Me
+Mej
+Mer
+Mes
+Messrs
+Mgr
+Mgrs
+Mll
+Mlle
+Mlle(s)
+Mme
+Mme(s)
+Mr
+Mrs
+Ms
+Msgr
+Mw
+Nov
+Npr
+Nr
+O.d.J
+Okt
+Op
+Ord
+Oz
+P
+P.D
+P.ej
+P.p.c
+Pas
+Pfc
+Ph
+Prim
+Prof
+Pte
+Pts
+Pvt
+Rep
+Reps
+Res
+Rev
+Revd
+Rh
+Riv
+Rt
+S.Em
+S.Exc
+S.a.r.l
+Sen
+Sens
+Sep
+Sept
+Sfc
+Sgt
+SGT
+Sl
+Sr
+Sra
+Sras
+Srs
+Srta
+St
+ST
+Sta
+Ste
+Sto
+Supt
+Surg
+Tj
+Tr
+Ud
+Uds
+V.Exc
+Vd
+Vda
+Vds
+Vz
+Z.D
+Z.D.H
+Z.E
+Z.Em
+Z.H
+Z.K.H
+Z.K.M
+Z.M
+a
+a./s
+a.C
+a.g.v
+a.l
+abrev
+abs
+ac
+acc
+acron
+adj
+adm
+adr
+adv
+alt
+anal
+anat
+angl
+appos
+apr
+apr
+asc
+atm
+aug
+auj
+aux
+av
+avg
+avr
+b
+b.a.o
+b.a.p
+b.a.r
+bacc
+bat
+bc
+bd
+bde
+bgen
+bijv
+bijz
+br
+bv
+c
+c.-a-d
+c.a.f
+c.i
+cc
+cf
+cft
+ch
+ch.-l
+chbre
+chbs
+chf
+col
+coll
+cpl
+cpt
+cpte
+cta
+d
+d.c
+d.w.z
+dcha
+dec
+def
+dem
+dep
+dept
+dez
+dhr
+dipl
+dispo
+div
+dpto
+dr
+dr.h.c
+dra
+dras
+drs
+ds
+dz
+e.c
+e.g
+e.g
+e.k
+eccles
+ecol
+econ
+ed
+ej
+env
+ep
+eq
+et
+etc
+ev
+ex
+exmo
+exo
+exp
+expo
+f.a.c
+fa
+fam
+fasc
+fbg
+feb
+fem
+fevr
+ff
+fl
+fol
+fr
+fs
+fut
+gd
+gde
+gdes
+gds
+gen
+gl
+grd
+h.-t
+hab
+i.e
+i.p.v
+i.s.m
+i.t.t
+i.v.m
+ibid
+id
+imp
+ing
+ir
+iron
+itd
+itn
+itp
+izq
+j
+jan
+jän
+janv
+jhr
+jkvr
+jr
+l
+lat
+lex
+lgen
+lib
+lieut
+liv
+lkol
+loc
+lof
+m
+m.a.w
+m.b.t
+m.b.v
+m.h.o
+m.i
+m.i.v
+maj
+mar
+mas
+max
+med
+mevr
+min
+mll
+mr
+ms
+mtr
+mtrs
+n
+n
+n.f
+n.f.pl
+n.m
+n.m.pl
+nov
+npr
+o
+o.b.s
+obs
+oct
+okt
+ord
+oz
+p
+p
+p.a
+p.ej
+p.ex
+p.g.c.d
+p.i
+p.j
+p.m
+p.o
+p.p
+p.p.c.d
+p.p.c.m
+p.pa
+p.pr
+pl
+plv
+poe
+pp
+pp
+pr
+pr
+pres
+prev
+prof
+px
+q.s
+qqch
+qqf
+qqn
+qqns
+r.-de-ch
+r.p.m
+rc
+rd
+ref
+refl
+reg
+rev
+ro
+rte
+s
+s
+s.a
+s.b.f
+s.d
+s.e
+s.l
+s.l.n.d
+s.l.p
+s.t.p
+s.v.p
+s/c
+sc
+sep
+sept
+sf
+sgt
+sl
+sr
+sra
+sras
+srs
+ss
+sto
+t
+t.s.v.p
+tec
+tel
+terr
+tg
+tint
+tit
+tj
+tr
+travx
+v
+v.intr
+v.tr
+v.w.t
+var
+vs
+vta
+vx
+z.v
+zool
+Št
+št