RubyGems - opener-tokenizer-base - Versions diffs - 1.0.0 - Mend

opener-tokenizer-base 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (44) hide show

checksums.yaml +7 -0
data/README.md +148 -0
data/bin/tokenizer-base +5 -0
data/bin/tokenizer-de +5 -0
data/bin/tokenizer-en +5 -0
data/bin/tokenizer-es +5 -0
data/bin/tokenizer-fr +5 -0
data/bin/tokenizer-it +5 -0
data/bin/tokenizer-nl +5 -0
data/core/lib/Data/OptList.pm +256 -0
data/core/lib/Params/Util.pm +866 -0
data/core/lib/Sub/Exporter.pm +1101 -0
data/core/lib/Sub/Exporter/Cookbook.pod +309 -0
data/core/lib/Sub/Exporter/Tutorial.pod +280 -0
data/core/lib/Sub/Exporter/Util.pm +354 -0
data/core/lib/Sub/Install.pm +329 -0
data/core/lib/Time/Stamp.pm +808 -0
data/core/load-prefixes.pl +43 -0
data/core/nonbreaking_prefixes/abbreviation_list.kaf +0 -0
data/core/nonbreaking_prefixes/abbreviation_list.txt +444 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.ca +533 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.de +781 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.el +448 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.en +564 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.es +758 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.fr +1027 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.is +697 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.it +641 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.nl +739 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.pl +729 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.pt +656 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.ro +484 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.ru +705 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.sk +920 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.sl +524 -0
data/core/nonbreaking_prefixes/nonbreaking_prefix.sv +492 -0
data/core/split-sentences.pl +114 -0
data/core/text-fixer.pl +169 -0
data/core/tokenizer-cli.pl +363 -0
data/core/tokenizer.pl +145 -0
data/lib/opener/tokenizers/base.rb +84 -0
data/lib/opener/tokenizers/base/version.rb +8 -0
data/opener-tokenizer-base.gemspec +25 -0
metadata +134 -0

data/core/nonbreaking_prefixes/nonbreaking_prefix.pt ADDED

@@ -0,0 +1,656 @@
+#File adapted for PT by H. Leal Fontes from the EN & DE versions published with moses-2009-04-13. Last update: 10.11.2009.
+#Anything in this file, followed by a period (and an upper-case word), does NOT indicate an end-of-sentence marker.
+#Special cases are included for prefixes that ONLY appear before 0-9 numbers.
+#any single upper case letter  followed by a period is not a sentence ender (excluding I occasionally, but we leave it in)
+#usually upper case letters are initials in a name
+A
+B
+C
+D
+E
+F
+G
+H
+I
+J
+K
+L
+M
+N
+O
+P
+Q
+R
+S
+T
+U
+V
+W
+X
+Y
+Z
+a
+b
+c
+d
+e
+f
+g
+h
+i
+j
+k
+l
+m
+n
+o
+p
+q
+r
+s
+t
+u
+v
+w
+x
+y
+z
+#Roman Numerals. A dot after one of these is not a sentence break in Portuguese.
+I
+II
+III
+IV
+V
+VI
+VII
+VIII
+IX
+X
+XI
+XII
+XIII
+XIV
+XV
+XVI
+XVII
+XVIII
+XIX
+XX
+i
+ii
+iii
+iv
+v
+vi
+vii
+viii
+ix
+x
+xi
+xii
+xiii
+xiv
+xv
+xvi
+xvii
+xviii
+xix
+xx
+#List of titles. These are often followed by upper-case names, but do not indicate sentence breaks
+Adj
+Adm
+Adv
+Art
+Ca
+Capt
+Cmdr
+Col
+Comdr
+Con
+Corp
+Cpl
+DR
+DRA
+Dr
+Dra
+Dras
+Drs
+Eng
+Enga
+Engas
+Engos
+Ex
+Exo
+Exmo
+Fig
+Gen
+Hosp
+Insp
+Lda
+MM
+MR
+MRS
+MS
+Maj
+Mrs
+Ms
+Msgr
+Op
+Ord
+Pfc
+Ph
+Prof
+Pvt
+Rep
+Reps
+Res
+Rev
+Rt
+Sen
+Sens
+Sfc
+Sgt
+SGT
+Sr
+Sra
+Sras
+Srs
+Sto
+Supt
+Surg
+adj
+adm
+adv
+art
+cit
+col
+con
+corp
+cpl
+dr
+dra
+dras
+drs
+eng
+enga
+engas
+engos
+ex
+exo
+exmo
+fig
+op
+prof
+sr
+sra
+sras
+srs
+sto
+#misc - odd period-ending items that NEVER indicate breaks (p.m. does NOT fall into this category - it sometimes ends a sentence)
+v
+vs
+i.e
+rev
+e.g
+#Numbers only. These should only induce breaks when followed by a numeric sequence
+# add NUMERIC_ONLY after the word for this function
+#This case is mostly for the english "No." which can either be a sentence of its own, or
+#if followed by a number, a non-breaking prefix
+No #NUMERIC_ONLY#
+Nos
+Art #NUMERIC_ONLY#
+Nr
+p #NUMERIC_ONLY#
+pp #NUMERIC_ONLY#
+#unified abbreviation list
+Acad
+Adj
+Adm
+Adv
+Affl
+Apr
+Art
+Asst
+Av
+Avg
+B.ches-du-Rh
+Bart
+Bco
+Bldg
+Brig
+Bros
+C.a
+C.p.c.n
+Ca
+Capt
+Cdt
+Cf
+Ch.-Mme
+Chap
+Cie
+Cmdr
+Col
+Comdr
+Con
+Corp
+Cpl
+DR
+DRA
+Da
+Dec
+Dep
+Dn
+Dr
+Dra
+Dras
+Drs
+Eng
+Enga
+Engas
+Engos
+Ens
+Ets
+Euro
+Ev
+Ex
+Excmo
+Exmo
+Exo
+Fa
+Fco
+Feb
+Fig
+Fr
+Gar
+Gen
+Gir
+Gl
+Gov
+Hno
+Hon
+Hosp
+Hr
+Ilmo
+Insp
+J.-C
+Jan
+Jeu
+Jr
+Jul
+Jun
+Lda
+Lieut
+Lt
+Lun
+MM
+MR
+MRS
+MS
+MSc
+Maj
+Mar
+Me
+Mej
+Mer
+Mes
+Messrs
+Mgr
+Mgrs
+Mll
+Mlle
+Mlle(s)
+Mme
+Mme(s)
+Mr
+Mrs
+Ms
+Msgr
+Mw
+Nov
+Npr
+Nr
+O.d.J
+Okt
+Op
+Ord
+Oz
+P
+P.D
+P.ej
+P.p.c
+Pas
+Pfc
+Ph
+Prim
+Prof
+Pte
+Pts
+Pvt
+Rep
+Reps
+Res
+Rev
+Revd
+Rh
+Riv
+Rt
+S.Em
+S.Exc
+S.a.r.l
+Sen
+Sens
+Sep
+Sept
+Sfc
+Sgt
+Sl
+Sr
+Sra
+Sras
+Srs
+Srta
+St
+ST
+Sta
+Ste
+Sto
+Supt
+Surg
+Tj
+Tr
+Ud
+Uds
+V.Exc
+Vd
+Vda
+Vds
+Vz
+Z.D
+Z.D.H
+Z.E
+Z.Em
+Z.H
+Z.K.H
+Z.K.M
+Z.M
+a
+a./s
+a.C
+a.g.v
+a.l
+abrev
+abs
+ac
+acc
+acron
+adj
+adm
+adr
+adv
+alt
+anal
+anat
+angl
+appos
+apr
+apr
+asc
+atm
+auj
+aux
+av
+avg
+avr
+b
+b.a.o
+b.a.p
+b.a.r
+bacc
+bat
+bc
+bd
+bde
+bgen
+bijv
+bijz
+br
+bv
+c
+c.-a-d
+c.a.f
+c.i
+cc
+cf
+cft
+ch
+ch.-l
+chbre
+chbs
+chf
+col
+coll
+cpl
+cpt
+cpte
+cta
+d
+d.c
+d.w.z
+dcha
+dec
+def
+dem
+dep
+dept
+dhr
+dipl
+dispo
+div
+dpto
+dr
+dr.h.c
+dra
+dras
+drs
+ds
+dz
+e.c
+e.g
+e.g
+e.k
+eccles
+ecol
+econ
+ed
+ej
+env
+ep
+eq
+et
+etc
+ev
+ex
+exmo
+exo
+exp
+expo
+f.a.c
+fa
+fam
+fasc
+fbg
+feb
+fem
+fevr
+ff
+fl
+fol
+fr
+fs
+fut
+gd
+gde
+gdes
+gds
+gen
+gl
+grd
+h.-t
+hab
+i.e
+i.p.v
+i.s.m
+i.t.t
+i.v.m
+ibid
+id
+imp
+ing
+ir
+iron
+itd
+itn
+itp
+izq
+j
+janv
+jhr
+jkvr
+jr
+l
+lat
+lex
+lgen
+lib
+lieut
+liv
+lkol
+loc
+lof
+m
+m.a.w
+m.b.t
+m.b.v
+m.h.o
+m.i
+m.i.v
+maj
+mar
+mas
+max
+med
+mevr
+min
+mll
+mr
+ms
+mtr
+mtrs
+n
+n
+n.f
+n.f.pl
+n.m
+n.m.pl
+npr
+o
+o.b.s
+obs
+oct
+okt
+ord
+oz
+p
+p
+p.a
+p.ej
+p.ex
+p.g.c.d
+p.i
+p.j
+p.m
+p.o
+p.p
+p.p.c.d
+p.p.c.m
+p.pa
+p.pr
+pl
+plv
+poe
+pp
+pp
+pr
+pr
+pres
+prev
+prof
+px
+q.s
+qqch
+qqf
+qqn
+qqns
+r.-de-ch
+r.p.m
+rc
+rd
+ref
+refl
+reg
+rev
+ro
+rte
+s
+s
+s.a
+s.b.f
+s.d
+s.e
+s.l
+s.l.n.d
+s.l.p
+s.t.p
+s.v.p
+s/c
+sc
+sf
+sgt
+sl
+sr
+sra
+sras
+srs
+ss
+sto
+t
+t.s.v.p
+tec
+tel
+terr
+tg
+tint
+tit
+tj
+tr
+travx
+v
+v.intr
+v.tr
+v.w.t
+var
+vs
+vta
+vx
+z.v
+zool
+Št
+št