opener-tokenizer-base 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/README.md +148 -0
- data/bin/tokenizer-base +5 -0
- data/bin/tokenizer-de +5 -0
- data/bin/tokenizer-en +5 -0
- data/bin/tokenizer-es +5 -0
- data/bin/tokenizer-fr +5 -0
- data/bin/tokenizer-it +5 -0
- data/bin/tokenizer-nl +5 -0
- data/core/lib/Data/OptList.pm +256 -0
- data/core/lib/Params/Util.pm +866 -0
- data/core/lib/Sub/Exporter.pm +1101 -0
- data/core/lib/Sub/Exporter/Cookbook.pod +309 -0
- data/core/lib/Sub/Exporter/Tutorial.pod +280 -0
- data/core/lib/Sub/Exporter/Util.pm +354 -0
- data/core/lib/Sub/Install.pm +329 -0
- data/core/lib/Time/Stamp.pm +808 -0
- data/core/load-prefixes.pl +43 -0
- data/core/nonbreaking_prefixes/abbreviation_list.kaf +0 -0
- data/core/nonbreaking_prefixes/abbreviation_list.txt +444 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.ca +533 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.de +781 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.el +448 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.en +564 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.es +758 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.fr +1027 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.is +697 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.it +641 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.nl +739 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.pl +729 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.pt +656 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.ro +484 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.ru +705 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.sk +920 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.sl +524 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.sv +492 -0
- data/core/split-sentences.pl +114 -0
- data/core/text-fixer.pl +169 -0
- data/core/tokenizer-cli.pl +363 -0
- data/core/tokenizer.pl +145 -0
- data/lib/opener/tokenizers/base.rb +84 -0
- data/lib/opener/tokenizers/base/version.rb +8 -0
- data/opener-tokenizer-base.gemspec +25 -0
- metadata +134 -0
@@ -0,0 +1,697 @@
|
|
1
|
+
no #NUMERIC_ONLY#
|
2
|
+
No #NUMERIC_ONLY#
|
3
|
+
nr #NUMERIC_ONLY#
|
4
|
+
Nr #NUMERIC_ONLY#
|
5
|
+
nR #NUMERIC_ONLY#
|
6
|
+
NR #NUMERIC_ONLY#
|
7
|
+
a
|
8
|
+
b
|
9
|
+
c
|
10
|
+
d
|
11
|
+
e
|
12
|
+
f
|
13
|
+
g
|
14
|
+
h
|
15
|
+
i
|
16
|
+
j
|
17
|
+
k
|
18
|
+
l
|
19
|
+
m
|
20
|
+
n
|
21
|
+
o
|
22
|
+
p
|
23
|
+
q
|
24
|
+
r
|
25
|
+
s
|
26
|
+
t
|
27
|
+
u
|
28
|
+
v
|
29
|
+
w
|
30
|
+
x
|
31
|
+
y
|
32
|
+
z
|
33
|
+
^
|
34
|
+
í
|
35
|
+
á
|
36
|
+
ó
|
37
|
+
æ
|
38
|
+
A
|
39
|
+
B
|
40
|
+
C
|
41
|
+
D
|
42
|
+
E
|
43
|
+
F
|
44
|
+
G
|
45
|
+
H
|
46
|
+
I
|
47
|
+
J
|
48
|
+
K
|
49
|
+
L
|
50
|
+
M
|
51
|
+
N
|
52
|
+
O
|
53
|
+
P
|
54
|
+
Q
|
55
|
+
R
|
56
|
+
S
|
57
|
+
T
|
58
|
+
U
|
59
|
+
V
|
60
|
+
W
|
61
|
+
X
|
62
|
+
Y
|
63
|
+
Z
|
64
|
+
ab.fn
|
65
|
+
a.fn
|
66
|
+
afs
|
67
|
+
al
|
68
|
+
alm
|
69
|
+
alg
|
70
|
+
andh
|
71
|
+
ath
|
72
|
+
aths
|
73
|
+
atr
|
74
|
+
ao
|
75
|
+
au
|
76
|
+
aukaf
|
77
|
+
áfn
|
78
|
+
áhrl.s
|
79
|
+
áhrs
|
80
|
+
ákv.gr
|
81
|
+
ákv
|
82
|
+
bh
|
83
|
+
bls
|
84
|
+
dr
|
85
|
+
e.Kr
|
86
|
+
et
|
87
|
+
ef
|
88
|
+
efn
|
89
|
+
ennfr
|
90
|
+
eink
|
91
|
+
end
|
92
|
+
e.st
|
93
|
+
erl
|
94
|
+
fél
|
95
|
+
fskj
|
96
|
+
fh
|
97
|
+
f.hl
|
98
|
+
físl
|
99
|
+
fl
|
100
|
+
fn
|
101
|
+
fo
|
102
|
+
forl
|
103
|
+
frb
|
104
|
+
frl
|
105
|
+
frh
|
106
|
+
frt
|
107
|
+
fsl
|
108
|
+
fsh
|
109
|
+
fs
|
110
|
+
fsk
|
111
|
+
fst
|
112
|
+
f.Kr
|
113
|
+
ft
|
114
|
+
fv
|
115
|
+
fyrrn
|
116
|
+
fyrrv
|
117
|
+
germ
|
118
|
+
gm
|
119
|
+
gr
|
120
|
+
hdl
|
121
|
+
hdr
|
122
|
+
hf
|
123
|
+
hl
|
124
|
+
hlsk
|
125
|
+
hljsk
|
126
|
+
hljv
|
127
|
+
hljóðv
|
128
|
+
hr
|
129
|
+
hv
|
130
|
+
hvk
|
131
|
+
holl
|
132
|
+
Hos
|
133
|
+
höf
|
134
|
+
hk
|
135
|
+
hrl
|
136
|
+
ísl
|
137
|
+
kaf
|
138
|
+
kap
|
139
|
+
Khöfn
|
140
|
+
kk
|
141
|
+
kg
|
142
|
+
kk
|
143
|
+
km
|
144
|
+
kl
|
145
|
+
klst
|
146
|
+
kr
|
147
|
+
kt
|
148
|
+
kgúrsk
|
149
|
+
kvk
|
150
|
+
leturbr
|
151
|
+
lh
|
152
|
+
lh.nt
|
153
|
+
lh.þt
|
154
|
+
lo
|
155
|
+
ltr
|
156
|
+
mlja
|
157
|
+
mljó
|
158
|
+
millj
|
159
|
+
mm
|
160
|
+
mms
|
161
|
+
m.fl
|
162
|
+
miðm
|
163
|
+
mgr
|
164
|
+
mst
|
165
|
+
mín
|
166
|
+
nf
|
167
|
+
nh
|
168
|
+
nhm
|
169
|
+
nl
|
170
|
+
nk
|
171
|
+
nmgr
|
172
|
+
no
|
173
|
+
núv
|
174
|
+
nt
|
175
|
+
o.áfr
|
176
|
+
o.m.fl
|
177
|
+
ohf
|
178
|
+
o.fl
|
179
|
+
o.s.frv
|
180
|
+
ófn
|
181
|
+
ób
|
182
|
+
óákv.gr
|
183
|
+
óákv
|
184
|
+
pfn
|
185
|
+
PR
|
186
|
+
pr
|
187
|
+
Ritstj
|
188
|
+
Rvík
|
189
|
+
Rvk
|
190
|
+
samb
|
191
|
+
samhlj
|
192
|
+
samn
|
193
|
+
samn
|
194
|
+
sbr
|
195
|
+
sek
|
196
|
+
sérn
|
197
|
+
sf
|
198
|
+
sfn
|
199
|
+
sh
|
200
|
+
sfn
|
201
|
+
sh
|
202
|
+
s.hl
|
203
|
+
sk
|
204
|
+
skv
|
205
|
+
sl
|
206
|
+
sn
|
207
|
+
so
|
208
|
+
ss.us
|
209
|
+
s.st
|
210
|
+
samþ
|
211
|
+
sbr
|
212
|
+
shlj
|
213
|
+
sign
|
214
|
+
skál
|
215
|
+
st
|
216
|
+
st.s
|
217
|
+
stk
|
218
|
+
sþ
|
219
|
+
teg
|
220
|
+
tbl
|
221
|
+
tfn
|
222
|
+
tl
|
223
|
+
tvíhlj
|
224
|
+
tvt
|
225
|
+
till
|
226
|
+
to
|
227
|
+
umr
|
228
|
+
uh
|
229
|
+
us
|
230
|
+
uppl
|
231
|
+
útg
|
232
|
+
vb
|
233
|
+
Vf
|
234
|
+
vh
|
235
|
+
vkf
|
236
|
+
Vl
|
237
|
+
vl
|
238
|
+
vlf
|
239
|
+
vmf
|
240
|
+
8vo
|
241
|
+
vsk
|
242
|
+
vth
|
243
|
+
þt
|
244
|
+
þf
|
245
|
+
þjs
|
246
|
+
þgf
|
247
|
+
þlt
|
248
|
+
þolm
|
249
|
+
þm
|
250
|
+
þml
|
251
|
+
þýð
|
252
|
+
|
253
|
+
#unified abbreviation list
|
254
|
+
Acad
|
255
|
+
Adj
|
256
|
+
Adm
|
257
|
+
Adv
|
258
|
+
Affl
|
259
|
+
Apr
|
260
|
+
Art
|
261
|
+
Asst
|
262
|
+
Av
|
263
|
+
Avg
|
264
|
+
B.ches-du-Rh
|
265
|
+
Bart
|
266
|
+
Bco
|
267
|
+
Bldg
|
268
|
+
Brig
|
269
|
+
Bros
|
270
|
+
C.a
|
271
|
+
C.p.c.n
|
272
|
+
Ca
|
273
|
+
Capt
|
274
|
+
Cdt
|
275
|
+
Cf
|
276
|
+
Ch.-Mme
|
277
|
+
Chap
|
278
|
+
Cie
|
279
|
+
Cmdr
|
280
|
+
Col
|
281
|
+
Comdr
|
282
|
+
Con
|
283
|
+
Corp
|
284
|
+
Cpl
|
285
|
+
DR
|
286
|
+
DRA
|
287
|
+
Da
|
288
|
+
Dec
|
289
|
+
Dep
|
290
|
+
Dn
|
291
|
+
Dr
|
292
|
+
Dra
|
293
|
+
Dras
|
294
|
+
Drs
|
295
|
+
Eng
|
296
|
+
Enga
|
297
|
+
Engas
|
298
|
+
Engos
|
299
|
+
Ens
|
300
|
+
Ets
|
301
|
+
Euro
|
302
|
+
Ev
|
303
|
+
Ex
|
304
|
+
Excmo
|
305
|
+
Exmo
|
306
|
+
Exo
|
307
|
+
Fa
|
308
|
+
Fco
|
309
|
+
Feb
|
310
|
+
Fig
|
311
|
+
Fr
|
312
|
+
Gar
|
313
|
+
Gen
|
314
|
+
Gir
|
315
|
+
Gl
|
316
|
+
Gov
|
317
|
+
Hno
|
318
|
+
Hon
|
319
|
+
Hosp
|
320
|
+
Hr
|
321
|
+
Ilmo
|
322
|
+
Insp
|
323
|
+
J.-C
|
324
|
+
Jan
|
325
|
+
Jeu
|
326
|
+
Jr
|
327
|
+
Jul
|
328
|
+
Jun
|
329
|
+
Lda
|
330
|
+
Lieut
|
331
|
+
Lt
|
332
|
+
Lun
|
333
|
+
MM
|
334
|
+
MR
|
335
|
+
MRS
|
336
|
+
MS
|
337
|
+
MSc
|
338
|
+
Maj
|
339
|
+
Mar
|
340
|
+
Me
|
341
|
+
Mej
|
342
|
+
Mer
|
343
|
+
Mes
|
344
|
+
Messrs
|
345
|
+
Mgr
|
346
|
+
Mgrs
|
347
|
+
Mll
|
348
|
+
Mlle
|
349
|
+
Mlle(s)
|
350
|
+
Mme
|
351
|
+
Mme(s)
|
352
|
+
Mr
|
353
|
+
Mrs
|
354
|
+
Ms
|
355
|
+
Msgr
|
356
|
+
Mw
|
357
|
+
Nov
|
358
|
+
Npr
|
359
|
+
Nr
|
360
|
+
O.d.J
|
361
|
+
Okt
|
362
|
+
Op
|
363
|
+
Ord
|
364
|
+
Oz
|
365
|
+
P
|
366
|
+
P.D
|
367
|
+
P.ej
|
368
|
+
P.p.c
|
369
|
+
Pas
|
370
|
+
Pfc
|
371
|
+
Ph
|
372
|
+
Prim
|
373
|
+
Prof
|
374
|
+
Pte
|
375
|
+
Pts
|
376
|
+
Pvt
|
377
|
+
Rep
|
378
|
+
Reps
|
379
|
+
Res
|
380
|
+
Rev
|
381
|
+
Revd
|
382
|
+
Rh
|
383
|
+
Riv
|
384
|
+
Rt
|
385
|
+
S.Em
|
386
|
+
S.Exc
|
387
|
+
S.a.r.l
|
388
|
+
Sen
|
389
|
+
Sens
|
390
|
+
Sep
|
391
|
+
Sept
|
392
|
+
Sfc
|
393
|
+
Sgt
|
394
|
+
SGT
|
395
|
+
Sl
|
396
|
+
Sr
|
397
|
+
Sra
|
398
|
+
Sras
|
399
|
+
Srs
|
400
|
+
Srta
|
401
|
+
St
|
402
|
+
ST
|
403
|
+
Sta
|
404
|
+
Ste
|
405
|
+
Sto
|
406
|
+
Supt
|
407
|
+
Surg
|
408
|
+
Tj
|
409
|
+
Tr
|
410
|
+
Ud
|
411
|
+
Uds
|
412
|
+
V.Exc
|
413
|
+
Vd
|
414
|
+
Vda
|
415
|
+
Vds
|
416
|
+
Vz
|
417
|
+
Z.D
|
418
|
+
Z.D.H
|
419
|
+
Z.E
|
420
|
+
Z.Em
|
421
|
+
Z.H
|
422
|
+
Z.K.H
|
423
|
+
Z.K.M
|
424
|
+
Z.M
|
425
|
+
a
|
426
|
+
a./s
|
427
|
+
a.C
|
428
|
+
a.g.v
|
429
|
+
a.l
|
430
|
+
abrev
|
431
|
+
abs
|
432
|
+
ac
|
433
|
+
acc
|
434
|
+
acron
|
435
|
+
adj
|
436
|
+
adm
|
437
|
+
adr
|
438
|
+
adv
|
439
|
+
alt
|
440
|
+
anal
|
441
|
+
anat
|
442
|
+
angl
|
443
|
+
appos
|
444
|
+
apr
|
445
|
+
apr
|
446
|
+
asc
|
447
|
+
atm
|
448
|
+
auj
|
449
|
+
aux
|
450
|
+
av
|
451
|
+
avg
|
452
|
+
avr
|
453
|
+
b
|
454
|
+
b.a.o
|
455
|
+
b.a.p
|
456
|
+
b.a.r
|
457
|
+
bacc
|
458
|
+
bat
|
459
|
+
bc
|
460
|
+
bd
|
461
|
+
bde
|
462
|
+
bgen
|
463
|
+
bijv
|
464
|
+
bijz
|
465
|
+
br
|
466
|
+
bv
|
467
|
+
c
|
468
|
+
c.-a-d
|
469
|
+
c.a.f
|
470
|
+
c.i
|
471
|
+
cc
|
472
|
+
cf
|
473
|
+
cft
|
474
|
+
ch
|
475
|
+
ch.-l
|
476
|
+
chbre
|
477
|
+
chbs
|
478
|
+
chf
|
479
|
+
col
|
480
|
+
coll
|
481
|
+
cpl
|
482
|
+
cpt
|
483
|
+
cpte
|
484
|
+
cta
|
485
|
+
d
|
486
|
+
d.c
|
487
|
+
d.w.z
|
488
|
+
dcha
|
489
|
+
dec
|
490
|
+
def
|
491
|
+
dem
|
492
|
+
dep
|
493
|
+
dept
|
494
|
+
dhr
|
495
|
+
dipl
|
496
|
+
dispo
|
497
|
+
div
|
498
|
+
dpto
|
499
|
+
dr
|
500
|
+
dr.h.c
|
501
|
+
dra
|
502
|
+
dras
|
503
|
+
drs
|
504
|
+
ds
|
505
|
+
dz
|
506
|
+
e.c
|
507
|
+
e.g
|
508
|
+
e.g
|
509
|
+
e.k
|
510
|
+
eccles
|
511
|
+
ecol
|
512
|
+
econ
|
513
|
+
ed
|
514
|
+
ej
|
515
|
+
env
|
516
|
+
ep
|
517
|
+
eq
|
518
|
+
et
|
519
|
+
etc
|
520
|
+
ev
|
521
|
+
ex
|
522
|
+
exmo
|
523
|
+
exo
|
524
|
+
exp
|
525
|
+
expo
|
526
|
+
f.a.c
|
527
|
+
fa
|
528
|
+
fam
|
529
|
+
fasc
|
530
|
+
fbg
|
531
|
+
feb
|
532
|
+
fem
|
533
|
+
fevr
|
534
|
+
ff
|
535
|
+
fl
|
536
|
+
fol
|
537
|
+
fr
|
538
|
+
fs
|
539
|
+
fut
|
540
|
+
gd
|
541
|
+
gde
|
542
|
+
gdes
|
543
|
+
gds
|
544
|
+
gen
|
545
|
+
gl
|
546
|
+
grd
|
547
|
+
h.-t
|
548
|
+
hab
|
549
|
+
i.e
|
550
|
+
i.p.v
|
551
|
+
i.s.m
|
552
|
+
i.t.t
|
553
|
+
i.v.m
|
554
|
+
ibid
|
555
|
+
id
|
556
|
+
imp
|
557
|
+
ing
|
558
|
+
ir
|
559
|
+
iron
|
560
|
+
itd
|
561
|
+
itn
|
562
|
+
itp
|
563
|
+
izq
|
564
|
+
j
|
565
|
+
janv
|
566
|
+
jhr
|
567
|
+
jkvr
|
568
|
+
jr
|
569
|
+
l
|
570
|
+
lat
|
571
|
+
lex
|
572
|
+
lgen
|
573
|
+
lib
|
574
|
+
lieut
|
575
|
+
liv
|
576
|
+
lkol
|
577
|
+
loc
|
578
|
+
lof
|
579
|
+
m
|
580
|
+
m.a.w
|
581
|
+
m.b.t
|
582
|
+
m.b.v
|
583
|
+
m.h.o
|
584
|
+
m.i
|
585
|
+
m.i.v
|
586
|
+
maj
|
587
|
+
mar
|
588
|
+
mas
|
589
|
+
max
|
590
|
+
med
|
591
|
+
mevr
|
592
|
+
min
|
593
|
+
mll
|
594
|
+
mr
|
595
|
+
ms
|
596
|
+
mtr
|
597
|
+
mtrs
|
598
|
+
n
|
599
|
+
n
|
600
|
+
n.f
|
601
|
+
n.f.pl
|
602
|
+
n.m
|
603
|
+
n.m.pl
|
604
|
+
npr
|
605
|
+
o
|
606
|
+
o.b.s
|
607
|
+
obs
|
608
|
+
oct
|
609
|
+
okt
|
610
|
+
ord
|
611
|
+
oz
|
612
|
+
p
|
613
|
+
p
|
614
|
+
p.a
|
615
|
+
p.ej
|
616
|
+
p.ex
|
617
|
+
p.g.c.d
|
618
|
+
p.i
|
619
|
+
p.j
|
620
|
+
p.m
|
621
|
+
p.o
|
622
|
+
p.p
|
623
|
+
p.p.c.d
|
624
|
+
p.p.c.m
|
625
|
+
p.pa
|
626
|
+
p.pr
|
627
|
+
pl
|
628
|
+
plv
|
629
|
+
poe
|
630
|
+
pp
|
631
|
+
pp
|
632
|
+
pr
|
633
|
+
pr
|
634
|
+
pres
|
635
|
+
prev
|
636
|
+
prof
|
637
|
+
px
|
638
|
+
q.s
|
639
|
+
qqch
|
640
|
+
qqf
|
641
|
+
qqn
|
642
|
+
qqns
|
643
|
+
r.-de-ch
|
644
|
+
r.p.m
|
645
|
+
rc
|
646
|
+
rd
|
647
|
+
ref
|
648
|
+
refl
|
649
|
+
reg
|
650
|
+
rev
|
651
|
+
ro
|
652
|
+
rte
|
653
|
+
s
|
654
|
+
s
|
655
|
+
s.a
|
656
|
+
s.b.f
|
657
|
+
s.d
|
658
|
+
s.e
|
659
|
+
s.l
|
660
|
+
s.l.n.d
|
661
|
+
s.l.p
|
662
|
+
s.t.p
|
663
|
+
s.v.p
|
664
|
+
s/c
|
665
|
+
sc
|
666
|
+
sf
|
667
|
+
sgt
|
668
|
+
sl
|
669
|
+
sr
|
670
|
+
sra
|
671
|
+
sras
|
672
|
+
srs
|
673
|
+
ss
|
674
|
+
sto
|
675
|
+
t
|
676
|
+
t.s.v.p
|
677
|
+
tec
|
678
|
+
tel
|
679
|
+
terr
|
680
|
+
tg
|
681
|
+
tint
|
682
|
+
tit
|
683
|
+
tj
|
684
|
+
tr
|
685
|
+
travx
|
686
|
+
v
|
687
|
+
v.intr
|
688
|
+
v.tr
|
689
|
+
v.w.t
|
690
|
+
var
|
691
|
+
vs
|
692
|
+
vta
|
693
|
+
vx
|
694
|
+
z.v
|
695
|
+
zool
|
696
|
+
Št
|
697
|
+
št
|