opener-tokenizer-base 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +7 -0
  2. data/README.md +148 -0
  3. data/bin/tokenizer-base +5 -0
  4. data/bin/tokenizer-de +5 -0
  5. data/bin/tokenizer-en +5 -0
  6. data/bin/tokenizer-es +5 -0
  7. data/bin/tokenizer-fr +5 -0
  8. data/bin/tokenizer-it +5 -0
  9. data/bin/tokenizer-nl +5 -0
  10. data/core/lib/Data/OptList.pm +256 -0
  11. data/core/lib/Params/Util.pm +866 -0
  12. data/core/lib/Sub/Exporter.pm +1101 -0
  13. data/core/lib/Sub/Exporter/Cookbook.pod +309 -0
  14. data/core/lib/Sub/Exporter/Tutorial.pod +280 -0
  15. data/core/lib/Sub/Exporter/Util.pm +354 -0
  16. data/core/lib/Sub/Install.pm +329 -0
  17. data/core/lib/Time/Stamp.pm +808 -0
  18. data/core/load-prefixes.pl +43 -0
  19. data/core/nonbreaking_prefixes/abbreviation_list.kaf +0 -0
  20. data/core/nonbreaking_prefixes/abbreviation_list.txt +444 -0
  21. data/core/nonbreaking_prefixes/nonbreaking_prefix.ca +533 -0
  22. data/core/nonbreaking_prefixes/nonbreaking_prefix.de +781 -0
  23. data/core/nonbreaking_prefixes/nonbreaking_prefix.el +448 -0
  24. data/core/nonbreaking_prefixes/nonbreaking_prefix.en +564 -0
  25. data/core/nonbreaking_prefixes/nonbreaking_prefix.es +758 -0
  26. data/core/nonbreaking_prefixes/nonbreaking_prefix.fr +1027 -0
  27. data/core/nonbreaking_prefixes/nonbreaking_prefix.is +697 -0
  28. data/core/nonbreaking_prefixes/nonbreaking_prefix.it +641 -0
  29. data/core/nonbreaking_prefixes/nonbreaking_prefix.nl +739 -0
  30. data/core/nonbreaking_prefixes/nonbreaking_prefix.pl +729 -0
  31. data/core/nonbreaking_prefixes/nonbreaking_prefix.pt +656 -0
  32. data/core/nonbreaking_prefixes/nonbreaking_prefix.ro +484 -0
  33. data/core/nonbreaking_prefixes/nonbreaking_prefix.ru +705 -0
  34. data/core/nonbreaking_prefixes/nonbreaking_prefix.sk +920 -0
  35. data/core/nonbreaking_prefixes/nonbreaking_prefix.sl +524 -0
  36. data/core/nonbreaking_prefixes/nonbreaking_prefix.sv +492 -0
  37. data/core/split-sentences.pl +114 -0
  38. data/core/text-fixer.pl +169 -0
  39. data/core/tokenizer-cli.pl +363 -0
  40. data/core/tokenizer.pl +145 -0
  41. data/lib/opener/tokenizers/base.rb +84 -0
  42. data/lib/opener/tokenizers/base/version.rb +8 -0
  43. data/opener-tokenizer-base.gemspec +25 -0
  44. metadata +134 -0
@@ -0,0 +1,758 @@
1
+
2
+ #Special cases are included for prefixes that ONLY appear before 0-9 numbers.
3
+
4
+ #any single upper case letter followed by a period is not a sentence ender (excluding I occasionally, but we leave it in)
5
+ #usually upper case letters are initials in a name
6
+ #no german words end in single lower-case letters, so we throw those in too.
7
+ A
8
+ B
9
+ C
10
+ D
11
+ E
12
+ F
13
+ G
14
+ H
15
+ I
16
+ J
17
+ K
18
+ L
19
+ M
20
+ N
21
+ O
22
+ P
23
+ Q
24
+ R
25
+ S
26
+ T
27
+ U
28
+ V
29
+ W
30
+ X
31
+ Y
32
+ Z
33
+ a
34
+ b
35
+ c
36
+ d
37
+ e
38
+ f
39
+ g
40
+ h
41
+ i
42
+ j
43
+ k
44
+ l
45
+ #m
46
+ n
47
+ o
48
+ p
49
+ q
50
+ r
51
+ s
52
+ t
53
+ u
54
+ v
55
+ w
56
+ x
57
+ y
58
+ z
59
+
60
+
61
+ #Roman Numerals. A dot after one of these is not a sentence break in German.
62
+ I
63
+ II
64
+ III
65
+ IV
66
+ V
67
+ VI
68
+ VII
69
+ VIII
70
+ IX
71
+ X
72
+ XI
73
+ XII
74
+ XIII
75
+ XIV
76
+ XV
77
+ XVI
78
+ XVII
79
+ XVIII
80
+ XIX
81
+ XX
82
+ i
83
+ ii
84
+ iii
85
+ iv
86
+ v
87
+ vi
88
+ vii
89
+ viii
90
+ ix
91
+ x
92
+ xi
93
+ xii
94
+ xiii
95
+ xiv
96
+ xv
97
+ xvi
98
+ xvii
99
+ xviii
100
+ xix
101
+ xx
102
+
103
+ #Titles and Honorifics
104
+
105
+ a.C
106
+ Av
107
+ Bco
108
+ cta
109
+ d.c
110
+ Da
111
+ dcha
112
+ Dep
113
+ Dn
114
+ dpto
115
+ Dr
116
+ Dra
117
+ ej
118
+ entlo
119
+ Excmo
120
+ Hno
121
+ i.e
122
+ Ilmo
123
+ izq
124
+ p
125
+ p.a
126
+ P.D
127
+ p.ej
128
+ p.p
129
+ p.pa
130
+ pral
131
+ Prof
132
+ Pts
133
+ r.p.m
134
+ rte
135
+ s
136
+ s.a
137
+ Sr
138
+ Sra
139
+ Srta
140
+ St
141
+ ST
142
+ Sta
143
+ Sto
144
+ Ud
145
+ Uds
146
+ Vd
147
+ Vda
148
+ Vds
149
+ vol
150
+ ac
151
+ acc
152
+ fem
153
+ mas
154
+ pl
155
+ sing
156
+ obs
157
+ pop
158
+ col
159
+ sl
160
+ vulg
161
+ comp
162
+ geogr
163
+ iron
164
+ med
165
+ mus
166
+ fil
167
+ fis
168
+ poe
169
+ psic
170
+ psych
171
+ jur
172
+ dep
173
+ tec
174
+ a
175
+ c
176
+ col
177
+ fol
178
+ fs
179
+ gl
180
+ itp
181
+ l
182
+ lib
183
+ n
184
+ ord
185
+ p
186
+ pp
187
+ pr
188
+ rc
189
+ rd
190
+ ro
191
+ s
192
+ ss
193
+ tit
194
+ vol
195
+ vta
196
+ mtr
197
+ mtrs
198
+ mll
199
+ Mll
200
+
201
+
202
+
203
+ #Ordinals are done with . in German - "1." = "1st" in English
204
+ #1
205
+ #2
206
+ #3
207
+ #4
208
+ #5
209
+ #6
210
+ #7
211
+ #8
212
+ #9
213
+ #10
214
+ #11
215
+ #12
216
+ #13
217
+ #14
218
+ #15
219
+ #16
220
+ #17
221
+ #18
222
+ #19
223
+ #20
224
+ #21
225
+ #22
226
+ #23
227
+ #24
228
+ #25
229
+ #26
230
+ #27
231
+ #28
232
+ #29
233
+ #30
234
+ #31
235
+ #32
236
+ #33
237
+ #34
238
+ #35
239
+ #36
240
+ #37
241
+ #38
242
+ #39
243
+ #40
244
+ #41
245
+ #42
246
+ #43
247
+ #44
248
+ #45
249
+ #46
250
+ #47
251
+ #48
252
+ #49
253
+ #50
254
+ #51
255
+ #52
256
+ #53
257
+ #54
258
+ #55
259
+ #56
260
+ #57
261
+ #58
262
+ #59
263
+ #60
264
+ #61
265
+ #62
266
+ #63
267
+ #64
268
+ #65
269
+ #66
270
+ #67
271
+ #68
272
+ #69
273
+ #70
274
+ #71
275
+ #72
276
+ #73
277
+ #74
278
+ #75
279
+ #76
280
+ #77
281
+ #78
282
+ #79
283
+ #80
284
+ #81
285
+ #82
286
+ #83
287
+ #84
288
+ #85
289
+ #86
290
+ #87
291
+ #88
292
+ #89
293
+ #90
294
+ #91
295
+ #92
296
+ #93
297
+ #94
298
+ #95
299
+ #96
300
+ #97
301
+ #98
302
+ #99
303
+
304
+ #unified abbreviation list
305
+ Abr
306
+ Acad
307
+ Adj
308
+ Adm
309
+ Adv
310
+ Affl
311
+ Apr
312
+ Art
313
+ Asst
314
+ Av
315
+ Avg
316
+ B.ches-du-Rh
317
+ Bart
318
+ Bco
319
+ Bldg
320
+ Brig
321
+ Bros
322
+ C.a
323
+ C.p.c.n
324
+ Ca
325
+ Capt
326
+ Cdt
327
+ Cf
328
+ Ch.-Mme
329
+ Chap
330
+ Cie
331
+ Cmdr
332
+ Col
333
+ Comdr
334
+ Con
335
+ Corp
336
+ Cpl
337
+ DR
338
+ DRA
339
+ Da
340
+ Dec
341
+ Dic
342
+ Dep
343
+ Dn
344
+ Dr
345
+ Dra
346
+ Dras
347
+ Drs
348
+ Eng
349
+ Enga
350
+ Engas
351
+ Engos
352
+ Ens
353
+ Ets
354
+ Euro
355
+ Ev
356
+ Ex
357
+ Excmo
358
+ Exmo
359
+ Exo
360
+ Fa
361
+ Fco
362
+ Feb
363
+ Fig
364
+ Fr
365
+ Gar
366
+ Gen
367
+ Gir
368
+ Gl
369
+ Gov
370
+ Hno
371
+ Hon
372
+ Hosp
373
+ Hr
374
+ Ilmo
375
+ Insp
376
+ J.-C
377
+ Jan
378
+ Jeu
379
+ Jr
380
+ Jul
381
+ Jun
382
+ Lda
383
+ Lieut
384
+ Lt
385
+ Lun
386
+ MM
387
+ MR
388
+ MRS
389
+ MS
390
+ MSc
391
+ Maj
392
+ Mar
393
+ Me
394
+ Mej
395
+ Mer
396
+ Mes
397
+ Messrs
398
+ Mgr
399
+ Mgrs
400
+ Mll
401
+ Mlle
402
+ Mlle(s)
403
+ Mme
404
+ Mme(s)
405
+ Mr
406
+ Mrs
407
+ Ms
408
+ Msgr
409
+ Mw
410
+ Nov
411
+ Npr
412
+ Nr
413
+ O.d.J
414
+ Oct
415
+ Okt
416
+ Op
417
+ Ord
418
+ Oz
419
+ P
420
+ P.D
421
+ P.ej
422
+ P.p.c
423
+ Pas
424
+ Pfc
425
+ Ph
426
+ Prim
427
+ Prof
428
+ Pte
429
+ Pts
430
+ Pvt
431
+ Rep
432
+ Reps
433
+ Res
434
+ Rev
435
+ Revd
436
+ Rh
437
+ Riv
438
+ Rt
439
+ S.Em
440
+ S.Exc
441
+ S.a.r.l
442
+ Sen
443
+ Sens
444
+ Sep
445
+ Sept
446
+ Sfc
447
+ Sgt
448
+ SGT
449
+ Sl
450
+ Sr
451
+ Sra
452
+ Sras
453
+ Srs
454
+ Srta
455
+ St
456
+ ST
457
+ Sta
458
+ Ste
459
+ Sto
460
+ Supt
461
+ Surg
462
+ Tj
463
+ Tr
464
+ Ud
465
+ Uds
466
+ V.Exc
467
+ Vd
468
+ Vda
469
+ Vds
470
+ Vz
471
+ Z.D
472
+ Z.D.H
473
+ Z.E
474
+ Z.Em
475
+ Z.H
476
+ Z.K.H
477
+ Z.K.M
478
+ Z.M
479
+ a
480
+ a./s
481
+ a.C
482
+ a.g.v
483
+ a.l
484
+ abr
485
+ abrev
486
+ abs
487
+ ac
488
+ acc
489
+ acron
490
+ adj
491
+ adm
492
+ adr
493
+ adv
494
+ alt
495
+ anal
496
+ anat
497
+ angl
498
+ appos
499
+ apr
500
+ apr
501
+ asc
502
+ atm
503
+ auj
504
+ aux
505
+ av
506
+ avg
507
+ avr
508
+ b
509
+ b.a.o
510
+ b.a.p
511
+ b.a.r
512
+ bacc
513
+ bat
514
+ bc
515
+ bd
516
+ bde
517
+ bgen
518
+ bijv
519
+ bijz
520
+ br
521
+ bv
522
+ c
523
+ c.-a-d
524
+ c.a.f
525
+ c.i
526
+ cc
527
+ cf
528
+ cft
529
+ ch
530
+ ch.-l
531
+ chbre
532
+ chbs
533
+ chf
534
+ col
535
+ coll
536
+ cpl
537
+ cpt
538
+ cpte
539
+ cta
540
+ d
541
+ d.c
542
+ d.w.z
543
+ dcha
544
+ dec
545
+ def
546
+ dem
547
+ dep
548
+ dept
549
+ dhr
550
+ dic
551
+ dipl
552
+ dispo
553
+ div
554
+ dpto
555
+ dr
556
+ dr.h.c
557
+ dra
558
+ dras
559
+ drs
560
+ ds
561
+ dz
562
+ e.c
563
+ e.g
564
+ e.g
565
+ e.k
566
+ eccles
567
+ ecol
568
+ econ
569
+ ed
570
+ ej
571
+ env
572
+ ep
573
+ eq
574
+ et
575
+ etc
576
+ ev
577
+ ex
578
+ exmo
579
+ exo
580
+ exp
581
+ expo
582
+ f.a.c
583
+ fa
584
+ fam
585
+ fasc
586
+ fbg
587
+ feb
588
+ fem
589
+ fevr
590
+ ff
591
+ fl
592
+ fol
593
+ fr
594
+ fs
595
+ fut
596
+ gd
597
+ gde
598
+ gdes
599
+ gds
600
+ gen
601
+ gl
602
+ grd
603
+ h.-t
604
+ hab
605
+ i.e
606
+ i.p.v
607
+ i.s.m
608
+ i.t.t
609
+ i.v.m
610
+ ibid
611
+ id
612
+ imp
613
+ ing
614
+ ir
615
+ iron
616
+ itd
617
+ itn
618
+ itp
619
+ izq
620
+ j
621
+ janv
622
+ jhr
623
+ jkvr
624
+ jr
625
+ jun
626
+ jul
627
+ l
628
+ lat
629
+ lex
630
+ lgen
631
+ lib
632
+ lieut
633
+ liv
634
+ lkol
635
+ loc
636
+ lof
637
+ #m
638
+ m.a.w
639
+ m.b.t
640
+ m.b.v
641
+ m.h.o
642
+ m.i
643
+ m.i.v
644
+ maj
645
+ mar
646
+ mas
647
+ max
648
+ med
649
+ mevr
650
+ min
651
+ mll
652
+ mr
653
+ ms
654
+ mtr
655
+ mtrs
656
+ n
657
+ n
658
+ n.f
659
+ n.f.pl
660
+ n.m
661
+ n.m.pl
662
+ nov
663
+ npr
664
+ o
665
+ o.b.s
666
+ obs
667
+ oct
668
+ okt
669
+ ord
670
+ oz
671
+ p
672
+ p
673
+ p.a
674
+ p.ej
675
+ p.ex
676
+ p.g.c.d
677
+ p.i
678
+ p.j
679
+ p.m
680
+ p.o
681
+ p.p
682
+ p.p.c.d
683
+ p.p.c.m
684
+ p.pa
685
+ p.pr
686
+ pl
687
+ plv
688
+ poe
689
+ pp
690
+ pp
691
+ pr
692
+ pr
693
+ pres
694
+ prev
695
+ prof
696
+ px
697
+ q.s
698
+ qqch
699
+ qqf
700
+ qqn
701
+ qqns
702
+ r.-de-ch
703
+ r.p.m
704
+ rc
705
+ rd
706
+ ref
707
+ refl
708
+ reg
709
+ rev
710
+ ro
711
+ rte
712
+ s
713
+ s
714
+ s.a
715
+ s.b.f
716
+ s.d
717
+ s.e
718
+ s.l
719
+ s.l.n.d
720
+ s.l.p
721
+ s.t.p
722
+ s.v.p
723
+ s/c
724
+ sc
725
+ sep
726
+ sept
727
+ sf
728
+ sgt
729
+ sl
730
+ sr
731
+ sra
732
+ sras
733
+ srs
734
+ ss
735
+ sto
736
+ t
737
+ t.s.v.p
738
+ tec
739
+ tel
740
+ terr
741
+ tg
742
+ tint
743
+ tit
744
+ tj
745
+ tr
746
+ travx
747
+ v
748
+ v.intr
749
+ v.tr
750
+ v.w.t
751
+ var
752
+ vs
753
+ vta
754
+ vx
755
+ z.v
756
+ zool
757
+ Št
758
+ št