opener-tokenizer-base 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/README.md +148 -0
- data/bin/tokenizer-base +5 -0
- data/bin/tokenizer-de +5 -0
- data/bin/tokenizer-en +5 -0
- data/bin/tokenizer-es +5 -0
- data/bin/tokenizer-fr +5 -0
- data/bin/tokenizer-it +5 -0
- data/bin/tokenizer-nl +5 -0
- data/core/lib/Data/OptList.pm +256 -0
- data/core/lib/Params/Util.pm +866 -0
- data/core/lib/Sub/Exporter.pm +1101 -0
- data/core/lib/Sub/Exporter/Cookbook.pod +309 -0
- data/core/lib/Sub/Exporter/Tutorial.pod +280 -0
- data/core/lib/Sub/Exporter/Util.pm +354 -0
- data/core/lib/Sub/Install.pm +329 -0
- data/core/lib/Time/Stamp.pm +808 -0
- data/core/load-prefixes.pl +43 -0
- data/core/nonbreaking_prefixes/abbreviation_list.kaf +0 -0
- data/core/nonbreaking_prefixes/abbreviation_list.txt +444 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.ca +533 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.de +781 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.el +448 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.en +564 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.es +758 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.fr +1027 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.is +697 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.it +641 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.nl +739 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.pl +729 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.pt +656 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.ro +484 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.ru +705 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.sk +920 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.sl +524 -0
- data/core/nonbreaking_prefixes/nonbreaking_prefix.sv +492 -0
- data/core/split-sentences.pl +114 -0
- data/core/text-fixer.pl +169 -0
- data/core/tokenizer-cli.pl +363 -0
- data/core/tokenizer.pl +145 -0
- data/lib/opener/tokenizers/base.rb +84 -0
- data/lib/opener/tokenizers/base/version.rb +8 -0
- data/opener-tokenizer-base.gemspec +25 -0
- metadata +134 -0
@@ -0,0 +1,705 @@
|
|
1
|
+
TBD: Russian uppercase alphabet [А-Я]
|
2
|
+
A
|
3
|
+
B
|
4
|
+
C
|
5
|
+
D
|
6
|
+
E
|
7
|
+
F
|
8
|
+
G
|
9
|
+
H
|
10
|
+
I
|
11
|
+
J
|
12
|
+
K
|
13
|
+
L
|
14
|
+
M
|
15
|
+
N
|
16
|
+
O
|
17
|
+
P
|
18
|
+
Q
|
19
|
+
R
|
20
|
+
S
|
21
|
+
T
|
22
|
+
U
|
23
|
+
V
|
24
|
+
W
|
25
|
+
X
|
26
|
+
Y
|
27
|
+
Z
|
28
|
+
0гг
|
29
|
+
1гг
|
30
|
+
2гг
|
31
|
+
3гг
|
32
|
+
4гг
|
33
|
+
5гг
|
34
|
+
6гг
|
35
|
+
7гг
|
36
|
+
8гг
|
37
|
+
9гг
|
38
|
+
0г
|
39
|
+
1г
|
40
|
+
2г
|
41
|
+
3г
|
42
|
+
4г
|
43
|
+
5г
|
44
|
+
6г
|
45
|
+
7г
|
46
|
+
8г
|
47
|
+
9г
|
48
|
+
Xвв
|
49
|
+
Vвв
|
50
|
+
Iвв
|
51
|
+
Lвв
|
52
|
+
Mвв
|
53
|
+
Cвв
|
54
|
+
Xв
|
55
|
+
Vв
|
56
|
+
Iв
|
57
|
+
Lв
|
58
|
+
Mв
|
59
|
+
Cв
|
60
|
+
0м
|
61
|
+
1м
|
62
|
+
2м
|
63
|
+
3м
|
64
|
+
4м
|
65
|
+
5м
|
66
|
+
6м
|
67
|
+
7м
|
68
|
+
8м
|
69
|
+
9м
|
70
|
+
0мм
|
71
|
+
1мм
|
72
|
+
2мм
|
73
|
+
3мм
|
74
|
+
4мм
|
75
|
+
5мм
|
76
|
+
6мм
|
77
|
+
7мм
|
78
|
+
8мм
|
79
|
+
9мм
|
80
|
+
0см
|
81
|
+
1см
|
82
|
+
2см
|
83
|
+
3см
|
84
|
+
4см
|
85
|
+
5см
|
86
|
+
6см
|
87
|
+
7см
|
88
|
+
8см
|
89
|
+
9см
|
90
|
+
0дм
|
91
|
+
1дм
|
92
|
+
2дм
|
93
|
+
3дм
|
94
|
+
4дм
|
95
|
+
5дм
|
96
|
+
6дм
|
97
|
+
7дм
|
98
|
+
8дм
|
99
|
+
9дм
|
100
|
+
0л
|
101
|
+
1л
|
102
|
+
2л
|
103
|
+
3л
|
104
|
+
4л
|
105
|
+
5л
|
106
|
+
6л
|
107
|
+
7л
|
108
|
+
8л
|
109
|
+
9л
|
110
|
+
0км
|
111
|
+
1км
|
112
|
+
2км
|
113
|
+
3км
|
114
|
+
4км
|
115
|
+
5км
|
116
|
+
6км
|
117
|
+
7км
|
118
|
+
8км
|
119
|
+
9км
|
120
|
+
0га
|
121
|
+
1га
|
122
|
+
2га
|
123
|
+
3га
|
124
|
+
4га
|
125
|
+
5га
|
126
|
+
6га
|
127
|
+
7га
|
128
|
+
8га
|
129
|
+
9га
|
130
|
+
0кг
|
131
|
+
1кг
|
132
|
+
2кг
|
133
|
+
3кг
|
134
|
+
4кг
|
135
|
+
5кг
|
136
|
+
6кг
|
137
|
+
7кг
|
138
|
+
8кг
|
139
|
+
9кг
|
140
|
+
0т
|
141
|
+
1т
|
142
|
+
2т
|
143
|
+
3т
|
144
|
+
4т
|
145
|
+
5т
|
146
|
+
6т
|
147
|
+
7т
|
148
|
+
8т
|
149
|
+
9т
|
150
|
+
0г
|
151
|
+
1г
|
152
|
+
2г
|
153
|
+
3г
|
154
|
+
4г
|
155
|
+
5г
|
156
|
+
6г
|
157
|
+
7г
|
158
|
+
8г
|
159
|
+
9г
|
160
|
+
0мг
|
161
|
+
1мг
|
162
|
+
2мг
|
163
|
+
3мг
|
164
|
+
4мг
|
165
|
+
5мг
|
166
|
+
6мг
|
167
|
+
7мг
|
168
|
+
8мг
|
169
|
+
9мг
|
170
|
+
бульв
|
171
|
+
в
|
172
|
+
вв
|
173
|
+
г
|
174
|
+
га
|
175
|
+
гг
|
176
|
+
гл
|
177
|
+
гос
|
178
|
+
д
|
179
|
+
дм
|
180
|
+
доп
|
181
|
+
др
|
182
|
+
е
|
183
|
+
ед
|
184
|
+
ед
|
185
|
+
зам
|
186
|
+
и
|
187
|
+
инд
|
188
|
+
исп
|
189
|
+
Исп
|
190
|
+
к
|
191
|
+
кап
|
192
|
+
кг
|
193
|
+
кв
|
194
|
+
кл
|
195
|
+
км
|
196
|
+
кол
|
197
|
+
комн
|
198
|
+
коп
|
199
|
+
куб
|
200
|
+
л
|
201
|
+
лиц
|
202
|
+
лл
|
203
|
+
м
|
204
|
+
макс
|
205
|
+
мг
|
206
|
+
мин
|
207
|
+
мл
|
208
|
+
млн
|
209
|
+
млрд
|
210
|
+
мм
|
211
|
+
н
|
212
|
+
наб
|
213
|
+
нач
|
214
|
+
неуд
|
215
|
+
ном
|
216
|
+
о
|
217
|
+
обл
|
218
|
+
обр
|
219
|
+
общ
|
220
|
+
ок
|
221
|
+
ост
|
222
|
+
отл
|
223
|
+
п
|
224
|
+
пер
|
225
|
+
перераб
|
226
|
+
пл
|
227
|
+
пос
|
228
|
+
пр
|
229
|
+
просп
|
230
|
+
проф
|
231
|
+
р
|
232
|
+
ред
|
233
|
+
руб
|
234
|
+
с
|
235
|
+
сб
|
236
|
+
св
|
237
|
+
см
|
238
|
+
соч
|
239
|
+
ср
|
240
|
+
ст
|
241
|
+
стр
|
242
|
+
т
|
243
|
+
тел
|
244
|
+
Тел
|
245
|
+
тех
|
246
|
+
тт
|
247
|
+
туп
|
248
|
+
тыс
|
249
|
+
уд
|
250
|
+
ул
|
251
|
+
уч
|
252
|
+
физ
|
253
|
+
х
|
254
|
+
хор
|
255
|
+
ч
|
256
|
+
чел
|
257
|
+
шт
|
258
|
+
экз
|
259
|
+
э
|
260
|
+
|
261
|
+
#unified abbreviation list
|
262
|
+
Acad
|
263
|
+
Adj
|
264
|
+
Adm
|
265
|
+
Adv
|
266
|
+
Affl
|
267
|
+
Apr
|
268
|
+
Art
|
269
|
+
Asst
|
270
|
+
Av
|
271
|
+
Avg
|
272
|
+
B.ches-du-Rh
|
273
|
+
Bart
|
274
|
+
Bco
|
275
|
+
Bldg
|
276
|
+
Brig
|
277
|
+
Bros
|
278
|
+
C.a
|
279
|
+
C.p.c.n
|
280
|
+
Ca
|
281
|
+
Capt
|
282
|
+
Cdt
|
283
|
+
Cf
|
284
|
+
Ch.-Mme
|
285
|
+
Chap
|
286
|
+
Cie
|
287
|
+
Cmdr
|
288
|
+
Col
|
289
|
+
Comdr
|
290
|
+
Con
|
291
|
+
Corp
|
292
|
+
Cpl
|
293
|
+
DR
|
294
|
+
DRA
|
295
|
+
Da
|
296
|
+
Dec
|
297
|
+
Dep
|
298
|
+
Dn
|
299
|
+
Dr
|
300
|
+
Dra
|
301
|
+
Dras
|
302
|
+
Drs
|
303
|
+
Eng
|
304
|
+
Enga
|
305
|
+
Engas
|
306
|
+
Engos
|
307
|
+
Ens
|
308
|
+
Ets
|
309
|
+
Euro
|
310
|
+
Ev
|
311
|
+
Ex
|
312
|
+
Excmo
|
313
|
+
Exmo
|
314
|
+
Exo
|
315
|
+
Fa
|
316
|
+
Fco
|
317
|
+
Feb
|
318
|
+
Fig
|
319
|
+
Fr
|
320
|
+
Gar
|
321
|
+
Gen
|
322
|
+
Gir
|
323
|
+
Gl
|
324
|
+
Gov
|
325
|
+
Hno
|
326
|
+
Hon
|
327
|
+
Hosp
|
328
|
+
Hr
|
329
|
+
Ilmo
|
330
|
+
Insp
|
331
|
+
J.-C
|
332
|
+
Jan
|
333
|
+
Jeu
|
334
|
+
Jr
|
335
|
+
Jul
|
336
|
+
Jun
|
337
|
+
Lda
|
338
|
+
Lieut
|
339
|
+
Lt
|
340
|
+
Lun
|
341
|
+
MM
|
342
|
+
MR
|
343
|
+
MRS
|
344
|
+
MS
|
345
|
+
MSc
|
346
|
+
Maj
|
347
|
+
Mar
|
348
|
+
Me
|
349
|
+
Mej
|
350
|
+
Mer
|
351
|
+
Mes
|
352
|
+
Messrs
|
353
|
+
Mgr
|
354
|
+
Mgrs
|
355
|
+
Mll
|
356
|
+
Mlle
|
357
|
+
Mlle(s)
|
358
|
+
Mme
|
359
|
+
Mme(s)
|
360
|
+
Mr
|
361
|
+
Mrs
|
362
|
+
Ms
|
363
|
+
Msgr
|
364
|
+
Mw
|
365
|
+
Nov
|
366
|
+
Npr
|
367
|
+
Nr
|
368
|
+
O.d.J
|
369
|
+
Okt
|
370
|
+
Op
|
371
|
+
Ord
|
372
|
+
Oz
|
373
|
+
P
|
374
|
+
P.D
|
375
|
+
P.ej
|
376
|
+
P.p.c
|
377
|
+
Pas
|
378
|
+
Pfc
|
379
|
+
Ph
|
380
|
+
Prim
|
381
|
+
Prof
|
382
|
+
Pte
|
383
|
+
Pts
|
384
|
+
Pvt
|
385
|
+
Rep
|
386
|
+
Reps
|
387
|
+
Res
|
388
|
+
Rev
|
389
|
+
Revd
|
390
|
+
Rh
|
391
|
+
Riv
|
392
|
+
Rt
|
393
|
+
S.Em
|
394
|
+
S.Exc
|
395
|
+
S.a.r.l
|
396
|
+
Sen
|
397
|
+
Sens
|
398
|
+
Sep
|
399
|
+
Sept
|
400
|
+
Sfc
|
401
|
+
Sgt
|
402
|
+
SGT
|
403
|
+
Sl
|
404
|
+
Sr
|
405
|
+
Sra
|
406
|
+
Sras
|
407
|
+
Srs
|
408
|
+
Srta
|
409
|
+
St
|
410
|
+
ST
|
411
|
+
Sta
|
412
|
+
Ste
|
413
|
+
Sto
|
414
|
+
Supt
|
415
|
+
Surg
|
416
|
+
Tj
|
417
|
+
Tr
|
418
|
+
Ud
|
419
|
+
Uds
|
420
|
+
V.Exc
|
421
|
+
Vd
|
422
|
+
Vda
|
423
|
+
Vds
|
424
|
+
Vz
|
425
|
+
Z.D
|
426
|
+
Z.D.H
|
427
|
+
Z.E
|
428
|
+
Z.Em
|
429
|
+
Z.H
|
430
|
+
Z.K.H
|
431
|
+
Z.K.M
|
432
|
+
Z.M
|
433
|
+
a
|
434
|
+
a./s
|
435
|
+
a.C
|
436
|
+
a.g.v
|
437
|
+
a.l
|
438
|
+
abrev
|
439
|
+
abs
|
440
|
+
ac
|
441
|
+
acc
|
442
|
+
acron
|
443
|
+
adj
|
444
|
+
adm
|
445
|
+
adr
|
446
|
+
adv
|
447
|
+
alt
|
448
|
+
anal
|
449
|
+
anat
|
450
|
+
angl
|
451
|
+
appos
|
452
|
+
apr
|
453
|
+
apr
|
454
|
+
asc
|
455
|
+
atm
|
456
|
+
auj
|
457
|
+
aux
|
458
|
+
av
|
459
|
+
avg
|
460
|
+
avr
|
461
|
+
b
|
462
|
+
b.a.o
|
463
|
+
b.a.p
|
464
|
+
b.a.r
|
465
|
+
bacc
|
466
|
+
bat
|
467
|
+
bc
|
468
|
+
bd
|
469
|
+
bde
|
470
|
+
bgen
|
471
|
+
bijv
|
472
|
+
bijz
|
473
|
+
br
|
474
|
+
bv
|
475
|
+
c
|
476
|
+
c.-a-d
|
477
|
+
c.a.f
|
478
|
+
c.i
|
479
|
+
cc
|
480
|
+
cf
|
481
|
+
cft
|
482
|
+
ch
|
483
|
+
ch.-l
|
484
|
+
chbre
|
485
|
+
chbs
|
486
|
+
chf
|
487
|
+
col
|
488
|
+
coll
|
489
|
+
cpl
|
490
|
+
cpt
|
491
|
+
cpte
|
492
|
+
cta
|
493
|
+
d
|
494
|
+
d.c
|
495
|
+
d.w.z
|
496
|
+
dcha
|
497
|
+
dec
|
498
|
+
def
|
499
|
+
dem
|
500
|
+
dep
|
501
|
+
dept
|
502
|
+
dhr
|
503
|
+
dipl
|
504
|
+
dispo
|
505
|
+
div
|
506
|
+
dpto
|
507
|
+
dr
|
508
|
+
dr.h.c
|
509
|
+
dra
|
510
|
+
dras
|
511
|
+
drs
|
512
|
+
ds
|
513
|
+
dz
|
514
|
+
e.c
|
515
|
+
e.g
|
516
|
+
e.g
|
517
|
+
e.k
|
518
|
+
eccles
|
519
|
+
ecol
|
520
|
+
econ
|
521
|
+
ed
|
522
|
+
ej
|
523
|
+
env
|
524
|
+
ep
|
525
|
+
eq
|
526
|
+
et
|
527
|
+
etc
|
528
|
+
ev
|
529
|
+
ex
|
530
|
+
exmo
|
531
|
+
exo
|
532
|
+
exp
|
533
|
+
expo
|
534
|
+
f.a.c
|
535
|
+
fa
|
536
|
+
fam
|
537
|
+
fasc
|
538
|
+
fbg
|
539
|
+
feb
|
540
|
+
fem
|
541
|
+
fevr
|
542
|
+
ff
|
543
|
+
fl
|
544
|
+
fol
|
545
|
+
fr
|
546
|
+
fs
|
547
|
+
fut
|
548
|
+
gd
|
549
|
+
gde
|
550
|
+
gdes
|
551
|
+
gds
|
552
|
+
gen
|
553
|
+
gl
|
554
|
+
grd
|
555
|
+
h.-t
|
556
|
+
hab
|
557
|
+
i.e
|
558
|
+
i.p.v
|
559
|
+
i.s.m
|
560
|
+
i.t.t
|
561
|
+
i.v.m
|
562
|
+
ibid
|
563
|
+
id
|
564
|
+
imp
|
565
|
+
ing
|
566
|
+
ir
|
567
|
+
iron
|
568
|
+
itd
|
569
|
+
itn
|
570
|
+
itp
|
571
|
+
izq
|
572
|
+
j
|
573
|
+
janv
|
574
|
+
jhr
|
575
|
+
jkvr
|
576
|
+
jr
|
577
|
+
l
|
578
|
+
lat
|
579
|
+
lex
|
580
|
+
lgen
|
581
|
+
lib
|
582
|
+
lieut
|
583
|
+
liv
|
584
|
+
lkol
|
585
|
+
loc
|
586
|
+
lof
|
587
|
+
m
|
588
|
+
m.a.w
|
589
|
+
m.b.t
|
590
|
+
m.b.v
|
591
|
+
m.h.o
|
592
|
+
m.i
|
593
|
+
m.i.v
|
594
|
+
maj
|
595
|
+
mar
|
596
|
+
mas
|
597
|
+
max
|
598
|
+
med
|
599
|
+
mevr
|
600
|
+
min
|
601
|
+
mll
|
602
|
+
mr
|
603
|
+
ms
|
604
|
+
mtr
|
605
|
+
mtrs
|
606
|
+
n
|
607
|
+
n
|
608
|
+
n.f
|
609
|
+
n.f.pl
|
610
|
+
n.m
|
611
|
+
n.m.pl
|
612
|
+
npr
|
613
|
+
o
|
614
|
+
o.b.s
|
615
|
+
obs
|
616
|
+
oct
|
617
|
+
okt
|
618
|
+
ord
|
619
|
+
oz
|
620
|
+
p
|
621
|
+
p
|
622
|
+
p.a
|
623
|
+
p.ej
|
624
|
+
p.ex
|
625
|
+
p.g.c.d
|
626
|
+
p.i
|
627
|
+
p.j
|
628
|
+
p.m
|
629
|
+
p.o
|
630
|
+
p.p
|
631
|
+
p.p.c.d
|
632
|
+
p.p.c.m
|
633
|
+
p.pa
|
634
|
+
p.pr
|
635
|
+
pl
|
636
|
+
plv
|
637
|
+
poe
|
638
|
+
pp
|
639
|
+
pp
|
640
|
+
pr
|
641
|
+
pr
|
642
|
+
pres
|
643
|
+
prev
|
644
|
+
prof
|
645
|
+
px
|
646
|
+
q.s
|
647
|
+
qqch
|
648
|
+
qqf
|
649
|
+
qqn
|
650
|
+
qqns
|
651
|
+
r.-de-ch
|
652
|
+
r.p.m
|
653
|
+
rc
|
654
|
+
rd
|
655
|
+
ref
|
656
|
+
refl
|
657
|
+
reg
|
658
|
+
rev
|
659
|
+
ro
|
660
|
+
rte
|
661
|
+
s
|
662
|
+
s
|
663
|
+
s.a
|
664
|
+
s.b.f
|
665
|
+
s.d
|
666
|
+
s.e
|
667
|
+
s.l
|
668
|
+
s.l.n.d
|
669
|
+
s.l.p
|
670
|
+
s.t.p
|
671
|
+
s.v.p
|
672
|
+
s/c
|
673
|
+
sc
|
674
|
+
sf
|
675
|
+
sgt
|
676
|
+
sl
|
677
|
+
sr
|
678
|
+
sra
|
679
|
+
sras
|
680
|
+
srs
|
681
|
+
ss
|
682
|
+
sto
|
683
|
+
t
|
684
|
+
t.s.v.p
|
685
|
+
tec
|
686
|
+
tel
|
687
|
+
terr
|
688
|
+
tg
|
689
|
+
tint
|
690
|
+
tit
|
691
|
+
tj
|
692
|
+
tr
|
693
|
+
travx
|
694
|
+
v
|
695
|
+
v.intr
|
696
|
+
v.tr
|
697
|
+
v.w.t
|
698
|
+
var
|
699
|
+
vs
|
700
|
+
vta
|
701
|
+
vx
|
702
|
+
z.v
|
703
|
+
zool
|
704
|
+
Št
|
705
|
+
št
|