opener-tokenizer-base 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/README.md +148 -0
- data/bin/tokenizer-base +5 -0
- data/bin/tokenizer-de +5 -0
- data/bin/tokenizer-en +5 -0
- data/bin/tokenizer-es +5 -0
- data/bin/tokenizer-fr +5 -0
- data/bin/tokenizer-it +5 -0
- data/bin/tokenizer-nl +5 -0
- data/core/lib/Data/OptList.pm +256 -0
- data/core/lib/Params/Util.pm +866 -0
- data/core/lib/Sub/Exporter.pm +1101 -0
- data/core/lib/Sub/Exporter/Cookbook.pod +309 -0
- data/core/lib/Sub/Exporter/Tutorial.pod +280 -0
- data/core/lib/Sub/Exporter/Util.pm +354 -0
- data/core/lib/Sub/Install.pm +329 -0
- data/core/lib/Time/Stamp.pm +808 -0
- data/core/load-prefixes.pl +43 -0
- data/core/nonbreaking_prefixes/abbreviation_list.kaf +0 -0
- data/core/nonbreaking_prefixes/abbreviation_list.txt +444 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.ca +533 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.de +781 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.el +448 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.en +564 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.es +758 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.fr +1027 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.is +697 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.it +641 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.nl +739 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.pl +729 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.pt +656 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.ro +484 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.ru +705 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.sk +920 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.sl +524 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.sv +492 -0
- data/core/split-sentences.pl +114 -0
- data/core/text-fixer.pl +169 -0
- data/core/tokenizer-cli.pl +363 -0
- data/core/tokenizer.pl +145 -0
- data/lib/opener/tokenizers/base.rb +84 -0
- data/lib/opener/tokenizers/base/version.rb +8 -0
- data/opener-tokenizer-base.gemspec +25 -0
- metadata +134 -0
@@ -0,0 +1,729 @@
|
|
1
|
+
adw
|
2
|
+
afr
|
3
|
+
akad
|
4
|
+
al
|
5
|
+
Al
|
6
|
+
am
|
7
|
+
amer
|
8
|
+
arch
|
9
|
+
art
|
10
|
+
Art
|
11
|
+
artyst
|
12
|
+
astr
|
13
|
+
austr
|
14
|
+
bałt
|
15
|
+
bdb
|
16
|
+
bł
|
17
|
+
bm
|
18
|
+
br
|
19
|
+
bryg
|
20
|
+
bryt
|
21
|
+
centr
|
22
|
+
ces
|
23
|
+
chem
|
24
|
+
chiń
|
25
|
+
chir
|
26
|
+
c.k
|
27
|
+
c.o
|
28
|
+
cyg
|
29
|
+
cyw
|
30
|
+
cyt
|
31
|
+
czes
|
32
|
+
czw
|
33
|
+
cd
|
34
|
+
Cd
|
35
|
+
czyt
|
36
|
+
ćw
|
37
|
+
ćwicz
|
38
|
+
daw
|
39
|
+
dcn
|
40
|
+
dekl
|
41
|
+
demokr
|
42
|
+
det
|
43
|
+
diec
|
44
|
+
dł
|
45
|
+
dn
|
46
|
+
dot
|
47
|
+
dol
|
48
|
+
dop
|
49
|
+
dost
|
50
|
+
dosł
|
51
|
+
h.c
|
52
|
+
ds
|
53
|
+
dst
|
54
|
+
duszp
|
55
|
+
dypl
|
56
|
+
egz
|
57
|
+
ekol
|
58
|
+
ekon
|
59
|
+
elektr
|
60
|
+
em
|
61
|
+
ew
|
62
|
+
fab
|
63
|
+
farm
|
64
|
+
fot
|
65
|
+
fr
|
66
|
+
gat
|
67
|
+
gastr
|
68
|
+
geogr
|
69
|
+
geol
|
70
|
+
gimn
|
71
|
+
głęb
|
72
|
+
gm
|
73
|
+
godz
|
74
|
+
górn
|
75
|
+
gosp
|
76
|
+
gr
|
77
|
+
gram
|
78
|
+
hist
|
79
|
+
hiszp
|
80
|
+
hr
|
81
|
+
Hr
|
82
|
+
hot
|
83
|
+
id
|
84
|
+
in
|
85
|
+
im
|
86
|
+
iron
|
87
|
+
jn
|
88
|
+
kard
|
89
|
+
kat
|
90
|
+
katol
|
91
|
+
k.k
|
92
|
+
kk
|
93
|
+
kol
|
94
|
+
kl
|
95
|
+
k.p.a
|
96
|
+
kpc
|
97
|
+
k.p.c
|
98
|
+
kpt
|
99
|
+
kr
|
100
|
+
k.r
|
101
|
+
krak
|
102
|
+
k.r.o
|
103
|
+
kryt
|
104
|
+
kult
|
105
|
+
laic
|
106
|
+
łac
|
107
|
+
niem
|
108
|
+
woj
|
109
|
+
nb
|
110
|
+
np
|
111
|
+
Nb
|
112
|
+
Np
|
113
|
+
pol
|
114
|
+
pow
|
115
|
+
m.in
|
116
|
+
pt
|
117
|
+
ps
|
118
|
+
Pt
|
119
|
+
Ps
|
120
|
+
cdn
|
121
|
+
jw
|
122
|
+
ryc
|
123
|
+
rys
|
124
|
+
Ryc
|
125
|
+
Rys
|
126
|
+
tj
|
127
|
+
tzw
|
128
|
+
Tzw
|
129
|
+
tzn
|
130
|
+
zob
|
131
|
+
ang
|
132
|
+
ub
|
133
|
+
ul
|
134
|
+
pw
|
135
|
+
pn
|
136
|
+
pl
|
137
|
+
al
|
138
|
+
k
|
139
|
+
n
|
140
|
+
nr #NUMERIC_ONLY#
|
141
|
+
Nr #NUMERIC_ONLY#
|
142
|
+
ww
|
143
|
+
wł
|
144
|
+
ur
|
145
|
+
zm
|
146
|
+
żyd
|
147
|
+
żarg
|
148
|
+
żyw
|
149
|
+
wył
|
150
|
+
bp
|
151
|
+
bp
|
152
|
+
wyst
|
153
|
+
tow
|
154
|
+
Tow
|
155
|
+
o
|
156
|
+
sp
|
157
|
+
Sp
|
158
|
+
st
|
159
|
+
spółdz
|
160
|
+
Spółdz
|
161
|
+
społ
|
162
|
+
spółgł
|
163
|
+
stoł
|
164
|
+
stow
|
165
|
+
Stoł
|
166
|
+
Stow
|
167
|
+
zn
|
168
|
+
zew
|
169
|
+
zewn
|
170
|
+
zdr
|
171
|
+
zazw
|
172
|
+
zast
|
173
|
+
zaw
|
174
|
+
zał
|
175
|
+
zal
|
176
|
+
zam
|
177
|
+
zak
|
178
|
+
zakł
|
179
|
+
zagr
|
180
|
+
zach
|
181
|
+
adw
|
182
|
+
Adw
|
183
|
+
lek
|
184
|
+
Lek
|
185
|
+
med
|
186
|
+
mec
|
187
|
+
Mec
|
188
|
+
doc
|
189
|
+
Doc
|
190
|
+
dyw
|
191
|
+
dyr
|
192
|
+
Dyw
|
193
|
+
Dyr
|
194
|
+
inż
|
195
|
+
Inż
|
196
|
+
mgr
|
197
|
+
Mgr
|
198
|
+
dh
|
199
|
+
dr
|
200
|
+
Dh
|
201
|
+
Dr
|
202
|
+
p
|
203
|
+
P
|
204
|
+
red
|
205
|
+
Red
|
206
|
+
prof
|
207
|
+
prok
|
208
|
+
Prof
|
209
|
+
Prok
|
210
|
+
hab
|
211
|
+
płk
|
212
|
+
Płk
|
213
|
+
nadkom
|
214
|
+
Nadkom
|
215
|
+
podkom
|
216
|
+
Podkom
|
217
|
+
ks
|
218
|
+
Ks
|
219
|
+
gen
|
220
|
+
Gen
|
221
|
+
por
|
222
|
+
Por
|
223
|
+
reż
|
224
|
+
Reż
|
225
|
+
przyp
|
226
|
+
Przyp
|
227
|
+
śp
|
228
|
+
św
|
229
|
+
śW
|
230
|
+
Śp
|
231
|
+
Św
|
232
|
+
ŚW
|
233
|
+
szer
|
234
|
+
Szer
|
235
|
+
pkt #NUMERIC_ONLY#
|
236
|
+
str #NUMERIC_ONLY#
|
237
|
+
tab #NUMERIC_ONLY#
|
238
|
+
Tab #NUMERIC_ONLY#
|
239
|
+
tel
|
240
|
+
ust #NUMERIC_ONLY#
|
241
|
+
par #NUMERIC_ONLY#
|
242
|
+
poz
|
243
|
+
pok
|
244
|
+
oo
|
245
|
+
oO
|
246
|
+
Oo
|
247
|
+
OO
|
248
|
+
r #NUMERIC_ONLY#
|
249
|
+
l #NUMERIC_ONLY#
|
250
|
+
s #NUMERIC_ONLY#
|
251
|
+
najśw
|
252
|
+
Najśw
|
253
|
+
A
|
254
|
+
B
|
255
|
+
C
|
256
|
+
D
|
257
|
+
E
|
258
|
+
F
|
259
|
+
G
|
260
|
+
H
|
261
|
+
I
|
262
|
+
J
|
263
|
+
K
|
264
|
+
L
|
265
|
+
M
|
266
|
+
N
|
267
|
+
O
|
268
|
+
P
|
269
|
+
Q
|
270
|
+
R
|
271
|
+
S
|
272
|
+
T
|
273
|
+
U
|
274
|
+
V
|
275
|
+
W
|
276
|
+
X
|
277
|
+
Y
|
278
|
+
Z
|
279
|
+
Ś
|
280
|
+
Ć
|
281
|
+
Ż
|
282
|
+
Ź
|
283
|
+
Dz
|
284
|
+
|
285
|
+
#unified abbreviation list
|
286
|
+
Acad
|
287
|
+
Adj
|
288
|
+
Adm
|
289
|
+
Adv
|
290
|
+
Affl
|
291
|
+
Apr
|
292
|
+
Art
|
293
|
+
Asst
|
294
|
+
Av
|
295
|
+
Avg
|
296
|
+
B.ches-du-Rh
|
297
|
+
Bart
|
298
|
+
Bco
|
299
|
+
Bldg
|
300
|
+
Brig
|
301
|
+
Bros
|
302
|
+
C.a
|
303
|
+
C.p.c.n
|
304
|
+
Ca
|
305
|
+
Capt
|
306
|
+
Cdt
|
307
|
+
Cf
|
308
|
+
Ch.-Mme
|
309
|
+
Chap
|
310
|
+
Cie
|
311
|
+
Cmdr
|
312
|
+
Col
|
313
|
+
Comdr
|
314
|
+
Con
|
315
|
+
Corp
|
316
|
+
Cpl
|
317
|
+
DR
|
318
|
+
DRA
|
319
|
+
Da
|
320
|
+
Dec
|
321
|
+
Dep
|
322
|
+
Dn
|
323
|
+
Dr
|
324
|
+
Dra
|
325
|
+
Dras
|
326
|
+
Drs
|
327
|
+
Eng
|
328
|
+
Enga
|
329
|
+
Engas
|
330
|
+
Engos
|
331
|
+
Ens
|
332
|
+
Ets
|
333
|
+
Euro
|
334
|
+
Ev
|
335
|
+
Ex
|
336
|
+
Excmo
|
337
|
+
Exmo
|
338
|
+
Exo
|
339
|
+
Fa
|
340
|
+
Fco
|
341
|
+
Feb
|
342
|
+
Fig
|
343
|
+
Fr
|
344
|
+
Gar
|
345
|
+
Gen
|
346
|
+
Gir
|
347
|
+
Gl
|
348
|
+
Gov
|
349
|
+
Hno
|
350
|
+
Hon
|
351
|
+
Hosp
|
352
|
+
Hr
|
353
|
+
Ilmo
|
354
|
+
Insp
|
355
|
+
J.-C
|
356
|
+
Jan
|
357
|
+
Jeu
|
358
|
+
Jr
|
359
|
+
Jul
|
360
|
+
Jun
|
361
|
+
Lda
|
362
|
+
Lieut
|
363
|
+
Lt
|
364
|
+
Lun
|
365
|
+
MM
|
366
|
+
MR
|
367
|
+
MRS
|
368
|
+
MS
|
369
|
+
MSc
|
370
|
+
Maj
|
371
|
+
Mar
|
372
|
+
Me
|
373
|
+
Mej
|
374
|
+
Mer
|
375
|
+
Mes
|
376
|
+
Messrs
|
377
|
+
Mgr
|
378
|
+
Mgrs
|
379
|
+
Mll
|
380
|
+
Mlle
|
381
|
+
Mlle(s)
|
382
|
+
Mme
|
383
|
+
Mme(s)
|
384
|
+
Mr
|
385
|
+
Mrs
|
386
|
+
Ms
|
387
|
+
Msgr
|
388
|
+
Mw
|
389
|
+
Nov
|
390
|
+
Npr
|
391
|
+
Nr
|
392
|
+
O.d.J
|
393
|
+
Okt
|
394
|
+
Op
|
395
|
+
Ord
|
396
|
+
Oz
|
397
|
+
P
|
398
|
+
P.D
|
399
|
+
P.ej
|
400
|
+
P.p.c
|
401
|
+
Pas
|
402
|
+
Pfc
|
403
|
+
Ph
|
404
|
+
Prim
|
405
|
+
Prof
|
406
|
+
Pte
|
407
|
+
Pts
|
408
|
+
Pvt
|
409
|
+
Rep
|
410
|
+
Reps
|
411
|
+
Res
|
412
|
+
Rev
|
413
|
+
Revd
|
414
|
+
Rh
|
415
|
+
Riv
|
416
|
+
Rt
|
417
|
+
S.Em
|
418
|
+
S.Exc
|
419
|
+
S.a.r.l
|
420
|
+
Sen
|
421
|
+
Sens
|
422
|
+
Sep
|
423
|
+
Sept
|
424
|
+
Sfc
|
425
|
+
Sgt
|
426
|
+
SGT
|
427
|
+
Sl
|
428
|
+
Sr
|
429
|
+
Sra
|
430
|
+
Sras
|
431
|
+
Srs
|
432
|
+
Srta
|
433
|
+
St
|
434
|
+
ST
|
435
|
+
Sta
|
436
|
+
Ste
|
437
|
+
Sto
|
438
|
+
Supt
|
439
|
+
Surg
|
440
|
+
Tj
|
441
|
+
Tr
|
442
|
+
Ud
|
443
|
+
Uds
|
444
|
+
V.Exc
|
445
|
+
Vd
|
446
|
+
Vda
|
447
|
+
Vds
|
448
|
+
Vz
|
449
|
+
Z.D
|
450
|
+
Z.D.H
|
451
|
+
Z.E
|
452
|
+
Z.Em
|
453
|
+
Z.H
|
454
|
+
Z.K.H
|
455
|
+
Z.K.M
|
456
|
+
Z.M
|
457
|
+
a
|
458
|
+
a./s
|
459
|
+
a.C
|
460
|
+
a.g.v
|
461
|
+
a.l
|
462
|
+
abrev
|
463
|
+
abs
|
464
|
+
ac
|
465
|
+
acc
|
466
|
+
acron
|
467
|
+
adj
|
468
|
+
adm
|
469
|
+
adr
|
470
|
+
adv
|
471
|
+
alt
|
472
|
+
anal
|
473
|
+
anat
|
474
|
+
angl
|
475
|
+
appos
|
476
|
+
apr
|
477
|
+
apr
|
478
|
+
asc
|
479
|
+
atm
|
480
|
+
auj
|
481
|
+
aux
|
482
|
+
av
|
483
|
+
avg
|
484
|
+
avr
|
485
|
+
b
|
486
|
+
b.a.o
|
487
|
+
b.a.p
|
488
|
+
b.a.r
|
489
|
+
bacc
|
490
|
+
bat
|
491
|
+
bc
|
492
|
+
bd
|
493
|
+
bde
|
494
|
+
bgen
|
495
|
+
bijv
|
496
|
+
bijz
|
497
|
+
br
|
498
|
+
bv
|
499
|
+
c
|
500
|
+
c.-a-d
|
501
|
+
c.a.f
|
502
|
+
c.i
|
503
|
+
cc
|
504
|
+
cf
|
505
|
+
cft
|
506
|
+
ch
|
507
|
+
ch.-l
|
508
|
+
chbre
|
509
|
+
chbs
|
510
|
+
chf
|
511
|
+
col
|
512
|
+
coll
|
513
|
+
cpl
|
514
|
+
cpt
|
515
|
+
cpte
|
516
|
+
cta
|
517
|
+
d
|
518
|
+
d.c
|
519
|
+
d.w.z
|
520
|
+
dcha
|
521
|
+
dec
|
522
|
+
def
|
523
|
+
dem
|
524
|
+
dep
|
525
|
+
dept
|
526
|
+
dhr
|
527
|
+
dipl
|
528
|
+
dispo
|
529
|
+
div
|
530
|
+
dpto
|
531
|
+
dr
|
532
|
+
dr.h.c
|
533
|
+
dra
|
534
|
+
dras
|
535
|
+
drs
|
536
|
+
ds
|
537
|
+
dz
|
538
|
+
e.c
|
539
|
+
e.g
|
540
|
+
e.g
|
541
|
+
e.k
|
542
|
+
eccles
|
543
|
+
ecol
|
544
|
+
econ
|
545
|
+
ed
|
546
|
+
ej
|
547
|
+
env
|
548
|
+
ep
|
549
|
+
eq
|
550
|
+
et
|
551
|
+
etc
|
552
|
+
ev
|
553
|
+
ex
|
554
|
+
exmo
|
555
|
+
exo
|
556
|
+
exp
|
557
|
+
expo
|
558
|
+
f.a.c
|
559
|
+
fa
|
560
|
+
fam
|
561
|
+
fasc
|
562
|
+
fbg
|
563
|
+
feb
|
564
|
+
fem
|
565
|
+
fevr
|
566
|
+
ff
|
567
|
+
fl
|
568
|
+
fol
|
569
|
+
fr
|
570
|
+
fs
|
571
|
+
fut
|
572
|
+
gd
|
573
|
+
gde
|
574
|
+
gdes
|
575
|
+
gds
|
576
|
+
gen
|
577
|
+
gl
|
578
|
+
grd
|
579
|
+
h.-t
|
580
|
+
hab
|
581
|
+
i.e
|
582
|
+
i.p.v
|
583
|
+
i.s.m
|
584
|
+
i.t.t
|
585
|
+
i.v.m
|
586
|
+
ibid
|
587
|
+
id
|
588
|
+
imp
|
589
|
+
ing
|
590
|
+
ir
|
591
|
+
iron
|
592
|
+
itd
|
593
|
+
itn
|
594
|
+
itp
|
595
|
+
izq
|
596
|
+
j
|
597
|
+
janv
|
598
|
+
jhr
|
599
|
+
jkvr
|
600
|
+
jr
|
601
|
+
l
|
602
|
+
lat
|
603
|
+
lex
|
604
|
+
lgen
|
605
|
+
lib
|
606
|
+
lieut
|
607
|
+
liv
|
608
|
+
lkol
|
609
|
+
loc
|
610
|
+
lof
|
611
|
+
m
|
612
|
+
m.a.w
|
613
|
+
m.b.t
|
614
|
+
m.b.v
|
615
|
+
m.h.o
|
616
|
+
m.i
|
617
|
+
m.i.v
|
618
|
+
maj
|
619
|
+
mar
|
620
|
+
mas
|
621
|
+
max
|
622
|
+
med
|
623
|
+
mevr
|
624
|
+
min
|
625
|
+
mll
|
626
|
+
mr
|
627
|
+
ms
|
628
|
+
mtr
|
629
|
+
mtrs
|
630
|
+
n
|
631
|
+
n
|
632
|
+
n.f
|
633
|
+
n.f.pl
|
634
|
+
n.m
|
635
|
+
n.m.pl
|
636
|
+
npr
|
637
|
+
o
|
638
|
+
o.b.s
|
639
|
+
obs
|
640
|
+
oct
|
641
|
+
okt
|
642
|
+
ord
|
643
|
+
oz
|
644
|
+
p
|
645
|
+
p
|
646
|
+
p.a
|
647
|
+
p.ej
|
648
|
+
p.ex
|
649
|
+
p.g.c.d
|
650
|
+
p.i
|
651
|
+
p.j
|
652
|
+
p.m
|
653
|
+
p.o
|
654
|
+
p.p
|
655
|
+
p.p.c.d
|
656
|
+
p.p.c.m
|
657
|
+
p.pa
|
658
|
+
p.pr
|
659
|
+
pl
|
660
|
+
plv
|
661
|
+
poe
|
662
|
+
pp
|
663
|
+
pp
|
664
|
+
pr
|
665
|
+
pr
|
666
|
+
pres
|
667
|
+
prev
|
668
|
+
prof
|
669
|
+
px
|
670
|
+
q.s
|
671
|
+
qqch
|
672
|
+
qqf
|
673
|
+
qqn
|
674
|
+
qqns
|
675
|
+
r.-de-ch
|
676
|
+
r.p.m
|
677
|
+
rc
|
678
|
+
rd
|
679
|
+
ref
|
680
|
+
refl
|
681
|
+
reg
|
682
|
+
rev
|
683
|
+
ro
|
684
|
+
rte
|
685
|
+
s
|
686
|
+
s
|
687
|
+
s.a
|
688
|
+
s.b.f
|
689
|
+
s.d
|
690
|
+
s.e
|
691
|
+
s.l
|
692
|
+
s.l.n.d
|
693
|
+
s.l.p
|
694
|
+
s.t.p
|
695
|
+
s.v.p
|
696
|
+
s/c
|
697
|
+
sc
|
698
|
+
sf
|
699
|
+
sgt
|
700
|
+
sl
|
701
|
+
sr
|
702
|
+
sra
|
703
|
+
sras
|
704
|
+
srs
|
705
|
+
ss
|
706
|
+
sto
|
707
|
+
t
|
708
|
+
t.s.v.p
|
709
|
+
tec
|
710
|
+
tel
|
711
|
+
terr
|
712
|
+
tg
|
713
|
+
tint
|
714
|
+
tit
|
715
|
+
tj
|
716
|
+
tr
|
717
|
+
travx
|
718
|
+
v
|
719
|
+
v.intr
|
720
|
+
v.tr
|
721
|
+
v.w.t
|
722
|
+
var
|
723
|
+
vs
|
724
|
+
vta
|
725
|
+
vx
|
726
|
+
z.v
|
727
|
+
zool
|
728
|
+
Št
|
729
|
+
št
|