@qvac/translation-nmtcpp 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +470 -0
- package/binding.js +1 -0
- package/index.d.ts +82 -0
- package/index.js +188 -0
- package/lib/error.js +65 -0
- package/marian.js +186 -0
- package/package.json +69 -0
- package/prebuilds/android-arm/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/android-arm64/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/android-ia32/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/android-x64/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/darwin-arm64/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/darwin-arm64/qvac__translation-nmtcpp.bare.exports +3622 -0
- package/prebuilds/darwin-x64/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/darwin-x64/qvac__translation-nmtcpp.bare.exports +3731 -0
- package/prebuilds/ios-arm64/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/ios-arm64/qvac__translation-nmtcpp.bare.exports +3603 -0
- package/prebuilds/ios-arm64-simulator/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/ios-arm64-simulator/qvac__translation-nmtcpp.bare.exports +3603 -0
- package/prebuilds/ios-x64-simulator/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/ios-x64-simulator/qvac__translation-nmtcpp.bare.exports +3720 -0
- package/prebuilds/linux-x64/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/win32-x64/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/win32-x64/qvac__translation-nmtcpp.bare.exports +0 -0
- package/third-party/indic-processor-deps/indicnlp/INDIC_NLP_LICENCE +9 -0
- package/third-party/indic-processor-deps/indicnlp/index.js +11 -0
- package/third-party/indic-processor-deps/indicnlp/indic_detokenize.js +141 -0
- package/third-party/indic-processor-deps/indicnlp/indic_normalize.js +1213 -0
- package/third-party/indic-processor-deps/indicnlp/indic_tokenize.js +123 -0
- package/third-party/indic-processor-deps/indicnlp/langinfo.js +609 -0
- package/third-party/indic-processor-deps/indicnlp/sinhala_transliterator.js +197 -0
- package/third-party/indic-processor-deps/indicnlp/unicode_transliterator.js +120 -0
- package/third-party/indic-processor-deps/sacremoses/SACREMOSES_LICENCE +21 -0
- package/third-party/indic-processor-deps/sacremoses/cjk.js +202 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/README.txt +8 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.as +65 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.bn +65 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.ca +75 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.cs +390 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.de +325 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.el +1568 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.en +123 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.es +118 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.et +138 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.fi +138 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.fr +153 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.ga +48 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.gu +105 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.hi +113 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.hu +103 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.is +251 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.it +180 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.kn +70 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.lt +698 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.lv +100 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.ml +67 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.mni +65 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.mr +113 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.nl +115 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.or +101 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.pa +102 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.pl +283 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.pt +210 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.ro +38 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.ru +293 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.sk +474 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.sl +78 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.sv +97 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.ta +71 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.tdt +210 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.te +70 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.yue +53 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.zh +53 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/CJK.txt +23246 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/CJKSymbols.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Close_Punctuation.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Currency_Symbol.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Han.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Hangul.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Hangul_Syllables.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Hiragana.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsAlnum-unichars-au.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsAlnum.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsAlpha-unichars-au.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsAlpha.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsLower.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsN.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsPf.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsPi.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsSc.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsSo.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsUpper.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Katakana.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Line_Separator.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Lowercase_Letter.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Number.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Open_Punctuation.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Punctuation.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Separator.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Symbol.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Titlecase_Letter.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Uppercase_Letter.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/index.js +8 -0
- package/third-party/indic-processor-deps/sacremoses/indic.js +76 -0
- package/third-party/indic-processor-deps/sacremoses/normalizer.js +264 -0
- package/third-party/indic-processor-deps/sacremoses/pernuliprops.js +287 -0
- package/third-party/indic-processor-deps/sacremoses/tokenizer.js +1217 -0
- package/third-party/indic-processor.js +565 -0
package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.en
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
#Anything in this file, followed by a period (and an upper-case word), does NOT indicate an end-of-sentence marker.
|
|
2
|
+
#Special cases are included for prefixes that ONLY appear before 0-9 numbers.
|
|
3
|
+
|
|
4
|
+
#any single upper case letter followed by a period is not a sentence ender (excluding I occasionally, but we leave it in)
|
|
5
|
+
#usually upper case letters are initials in a name
|
|
6
|
+
A
|
|
7
|
+
B
|
|
8
|
+
C
|
|
9
|
+
D
|
|
10
|
+
E
|
|
11
|
+
F
|
|
12
|
+
G
|
|
13
|
+
H
|
|
14
|
+
I
|
|
15
|
+
J
|
|
16
|
+
K
|
|
17
|
+
L
|
|
18
|
+
M
|
|
19
|
+
N
|
|
20
|
+
O
|
|
21
|
+
P
|
|
22
|
+
Q
|
|
23
|
+
R
|
|
24
|
+
S
|
|
25
|
+
T
|
|
26
|
+
U
|
|
27
|
+
V
|
|
28
|
+
W
|
|
29
|
+
X
|
|
30
|
+
Y
|
|
31
|
+
Z
|
|
32
|
+
|
|
33
|
+
#List of titles. These are often followed by upper-case names, but do not indicate sentence breaks
|
|
34
|
+
Adj
|
|
35
|
+
Adm
|
|
36
|
+
Adv
|
|
37
|
+
Asst
|
|
38
|
+
Bart
|
|
39
|
+
Bldg
|
|
40
|
+
Brig
|
|
41
|
+
Bros
|
|
42
|
+
Capt
|
|
43
|
+
Cmdr
|
|
44
|
+
Col
|
|
45
|
+
Comdr
|
|
46
|
+
Con
|
|
47
|
+
Corp
|
|
48
|
+
Cpl
|
|
49
|
+
DR
|
|
50
|
+
Dr
|
|
51
|
+
Drs
|
|
52
|
+
Ens
|
|
53
|
+
Gen
|
|
54
|
+
Gov
|
|
55
|
+
Hon
|
|
56
|
+
Hr
|
|
57
|
+
Hosp
|
|
58
|
+
Insp
|
|
59
|
+
Lt
|
|
60
|
+
MM
|
|
61
|
+
MR
|
|
62
|
+
MRS
|
|
63
|
+
MS
|
|
64
|
+
Maj
|
|
65
|
+
Messrs
|
|
66
|
+
Mlle
|
|
67
|
+
Mme
|
|
68
|
+
Mr
|
|
69
|
+
Mrs
|
|
70
|
+
Ms
|
|
71
|
+
Msgr
|
|
72
|
+
Op
|
|
73
|
+
Ord
|
|
74
|
+
Pfc
|
|
75
|
+
Ph
|
|
76
|
+
Prof
|
|
77
|
+
Pvt
|
|
78
|
+
Rep
|
|
79
|
+
Reps
|
|
80
|
+
Res
|
|
81
|
+
Rev
|
|
82
|
+
Rt
|
|
83
|
+
Sen
|
|
84
|
+
Sens
|
|
85
|
+
Sfc
|
|
86
|
+
Sgt
|
|
87
|
+
Sr
|
|
88
|
+
St
|
|
89
|
+
Supt
|
|
90
|
+
Surg
|
|
91
|
+
|
|
92
|
+
#misc - odd period-ending items that NEVER indicate breaks (p.m. does NOT fall into this category - it sometimes ends a sentence)
|
|
93
|
+
v
|
|
94
|
+
vs
|
|
95
|
+
i.e
|
|
96
|
+
rev
|
|
97
|
+
e.g
|
|
98
|
+
# rupees
|
|
99
|
+
Rs
|
|
100
|
+
|
|
101
|
+
#Numbers only. These should only induce breaks when followed by a numeric sequence
|
|
102
|
+
# add NUMERIC_ONLY after the word for this function
|
|
103
|
+
#This case is mostly for the english "No." which can either be a sentence of its own, or
|
|
104
|
+
#if followed by a number, a non-breaking prefix
|
|
105
|
+
No #NUMERIC_ONLY#
|
|
106
|
+
Nos
|
|
107
|
+
Art #NUMERIC_ONLY#
|
|
108
|
+
Nr
|
|
109
|
+
pp #NUMERIC_ONLY#
|
|
110
|
+
|
|
111
|
+
#month abbreviations
|
|
112
|
+
Jan
|
|
113
|
+
Feb
|
|
114
|
+
Mar
|
|
115
|
+
Apr
|
|
116
|
+
#May is a full word
|
|
117
|
+
Jun
|
|
118
|
+
Jul
|
|
119
|
+
Aug
|
|
120
|
+
Sep
|
|
121
|
+
Oct
|
|
122
|
+
Nov
|
|
123
|
+
Dec
|
package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.es
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
#Anything in this file, followed by a period (and an upper-case word), does NOT indicate an end-of-sentence marker.
|
|
2
|
+
#Special cases are included for prefixes that ONLY appear before 0-9 numbers.
|
|
3
|
+
|
|
4
|
+
#any single upper case letter followed by a period is not a sentence ender
|
|
5
|
+
#usually upper case letters are initials in a name
|
|
6
|
+
A
|
|
7
|
+
B
|
|
8
|
+
C
|
|
9
|
+
D
|
|
10
|
+
E
|
|
11
|
+
F
|
|
12
|
+
G
|
|
13
|
+
H
|
|
14
|
+
I
|
|
15
|
+
J
|
|
16
|
+
K
|
|
17
|
+
L
|
|
18
|
+
M
|
|
19
|
+
N
|
|
20
|
+
O
|
|
21
|
+
P
|
|
22
|
+
Q
|
|
23
|
+
R
|
|
24
|
+
S
|
|
25
|
+
T
|
|
26
|
+
U
|
|
27
|
+
V
|
|
28
|
+
W
|
|
29
|
+
X
|
|
30
|
+
Y
|
|
31
|
+
Z
|
|
32
|
+
|
|
33
|
+
# Period-final abbreviation list from http://www.ctspanish.com/words/abbreviations.htm
|
|
34
|
+
|
|
35
|
+
A.C
|
|
36
|
+
Apdo
|
|
37
|
+
Av
|
|
38
|
+
Bco
|
|
39
|
+
CC.AA
|
|
40
|
+
Da
|
|
41
|
+
Dep
|
|
42
|
+
Dn
|
|
43
|
+
Dr
|
|
44
|
+
Dra
|
|
45
|
+
EE.UU
|
|
46
|
+
Excmo
|
|
47
|
+
FF.CC
|
|
48
|
+
Fil
|
|
49
|
+
Gral
|
|
50
|
+
J.C
|
|
51
|
+
Let
|
|
52
|
+
Lic
|
|
53
|
+
N.B
|
|
54
|
+
P.D
|
|
55
|
+
P.V.P
|
|
56
|
+
Prof
|
|
57
|
+
Pts
|
|
58
|
+
Rte
|
|
59
|
+
S.A
|
|
60
|
+
S.A.R
|
|
61
|
+
S.E
|
|
62
|
+
S.L
|
|
63
|
+
S.R.C
|
|
64
|
+
Sr
|
|
65
|
+
Sra
|
|
66
|
+
Srta
|
|
67
|
+
Sta
|
|
68
|
+
Sto
|
|
69
|
+
T.V.E
|
|
70
|
+
Tel
|
|
71
|
+
Ud
|
|
72
|
+
Uds
|
|
73
|
+
V.B
|
|
74
|
+
V.E
|
|
75
|
+
Vd
|
|
76
|
+
Vds
|
|
77
|
+
a/c
|
|
78
|
+
adj
|
|
79
|
+
admón
|
|
80
|
+
afmo
|
|
81
|
+
apdo
|
|
82
|
+
av
|
|
83
|
+
c
|
|
84
|
+
c.f
|
|
85
|
+
c.g
|
|
86
|
+
cap
|
|
87
|
+
cm
|
|
88
|
+
cta
|
|
89
|
+
dcha
|
|
90
|
+
doc
|
|
91
|
+
ej
|
|
92
|
+
entlo
|
|
93
|
+
esq
|
|
94
|
+
etc
|
|
95
|
+
f.c
|
|
96
|
+
gr
|
|
97
|
+
grs
|
|
98
|
+
izq
|
|
99
|
+
kg
|
|
100
|
+
km
|
|
101
|
+
mg
|
|
102
|
+
mm
|
|
103
|
+
núm
|
|
104
|
+
núm
|
|
105
|
+
p
|
|
106
|
+
p.a
|
|
107
|
+
p.ej
|
|
108
|
+
ptas
|
|
109
|
+
pág
|
|
110
|
+
págs
|
|
111
|
+
pág
|
|
112
|
+
págs
|
|
113
|
+
q.e.g.e
|
|
114
|
+
q.e.s.m
|
|
115
|
+
s
|
|
116
|
+
s.s.s
|
|
117
|
+
vid
|
|
118
|
+
vol
|
package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.et
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
#Anything in this file, followed by a period (and an upper-case word), does NOT
|
|
2
|
+
#indicate an end-of-sentence marker. Special cases are included for prefixes
|
|
3
|
+
#that ONLY appear before 0-9 numbers.
|
|
4
|
+
|
|
5
|
+
#This list is compiled from omorfi <http://code.google.com/p/omorfi> database
|
|
6
|
+
#by Tommi A Pirinen.
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
#any single upper case letter followed by a period is not a sentence ender
|
|
10
|
+
A
|
|
11
|
+
B
|
|
12
|
+
C
|
|
13
|
+
D
|
|
14
|
+
E
|
|
15
|
+
F
|
|
16
|
+
G
|
|
17
|
+
H
|
|
18
|
+
I
|
|
19
|
+
J
|
|
20
|
+
K
|
|
21
|
+
L
|
|
22
|
+
M
|
|
23
|
+
N
|
|
24
|
+
O
|
|
25
|
+
P
|
|
26
|
+
Q
|
|
27
|
+
R
|
|
28
|
+
S
|
|
29
|
+
T
|
|
30
|
+
U
|
|
31
|
+
V
|
|
32
|
+
W
|
|
33
|
+
X
|
|
34
|
+
Y
|
|
35
|
+
Z
|
|
36
|
+
Å
|
|
37
|
+
Ä
|
|
38
|
+
Ö
|
|
39
|
+
|
|
40
|
+
#List of titles. These are often followed by upper-case names, but do not indicate sentence breaks
|
|
41
|
+
alik
|
|
42
|
+
alil
|
|
43
|
+
amir
|
|
44
|
+
apul
|
|
45
|
+
apul.prof
|
|
46
|
+
arkkit
|
|
47
|
+
ass
|
|
48
|
+
assist
|
|
49
|
+
dipl
|
|
50
|
+
dipl.arkkit
|
|
51
|
+
dipl.ekon
|
|
52
|
+
dipl.ins
|
|
53
|
+
dipl.kielenk
|
|
54
|
+
dipl.kirjeenv
|
|
55
|
+
dipl.kosm
|
|
56
|
+
dipl.urk
|
|
57
|
+
dos
|
|
58
|
+
erikoiseläinl
|
|
59
|
+
erikoishammasl
|
|
60
|
+
erikoisl
|
|
61
|
+
erikoist
|
|
62
|
+
ev.luutn
|
|
63
|
+
evp
|
|
64
|
+
fil
|
|
65
|
+
ft
|
|
66
|
+
hallinton
|
|
67
|
+
hallintot
|
|
68
|
+
hammaslääket
|
|
69
|
+
jatk
|
|
70
|
+
jääk
|
|
71
|
+
kansaned
|
|
72
|
+
kapt
|
|
73
|
+
kapt.luutn
|
|
74
|
+
kenr
|
|
75
|
+
kenr.luutn
|
|
76
|
+
kenr.maj
|
|
77
|
+
kers
|
|
78
|
+
kirjeenv
|
|
79
|
+
kom
|
|
80
|
+
kom.kapt
|
|
81
|
+
komm
|
|
82
|
+
konst
|
|
83
|
+
korpr
|
|
84
|
+
luutn
|
|
85
|
+
maist
|
|
86
|
+
maj
|
|
87
|
+
Mr
|
|
88
|
+
Mrs
|
|
89
|
+
Ms
|
|
90
|
+
M.Sc
|
|
91
|
+
neuv
|
|
92
|
+
nimim
|
|
93
|
+
Ph.D
|
|
94
|
+
prof
|
|
95
|
+
puh.joht
|
|
96
|
+
pääll
|
|
97
|
+
res
|
|
98
|
+
san
|
|
99
|
+
siht
|
|
100
|
+
suom
|
|
101
|
+
sähköp
|
|
102
|
+
säv
|
|
103
|
+
toht
|
|
104
|
+
toim
|
|
105
|
+
toim.apul
|
|
106
|
+
toim.joht
|
|
107
|
+
toim.siht
|
|
108
|
+
tuom
|
|
109
|
+
ups
|
|
110
|
+
vänr
|
|
111
|
+
vääp
|
|
112
|
+
ye.ups
|
|
113
|
+
ylik
|
|
114
|
+
ylil
|
|
115
|
+
ylim
|
|
116
|
+
ylimatr
|
|
117
|
+
yliop
|
|
118
|
+
yliopp
|
|
119
|
+
ylip
|
|
120
|
+
yliv
|
|
121
|
+
|
|
122
|
+
#misc - odd period-ending items that NEVER indicate breaks (p.m. does NOT fall
|
|
123
|
+
#into this category - it sometimes ends a sentence)
|
|
124
|
+
e.g
|
|
125
|
+
ent
|
|
126
|
+
esim
|
|
127
|
+
huom
|
|
128
|
+
i.e
|
|
129
|
+
ilm
|
|
130
|
+
l
|
|
131
|
+
mm
|
|
132
|
+
myöh
|
|
133
|
+
nk
|
|
134
|
+
nyk
|
|
135
|
+
par
|
|
136
|
+
po
|
|
137
|
+
t
|
|
138
|
+
v
|
package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.fi
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
#Anything in this file, followed by a period (and an upper-case word), does NOT
|
|
2
|
+
#indicate an end-of-sentence marker. Special cases are included for prefixes
|
|
3
|
+
#that ONLY appear before 0-9 numbers.
|
|
4
|
+
|
|
5
|
+
#This list is compiled from omorfi <http://code.google.com/p/omorfi> database
|
|
6
|
+
#by Tommi A Pirinen.
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
#any single upper case letter followed by a period is not a sentence ender
|
|
10
|
+
A
|
|
11
|
+
B
|
|
12
|
+
C
|
|
13
|
+
D
|
|
14
|
+
E
|
|
15
|
+
F
|
|
16
|
+
G
|
|
17
|
+
H
|
|
18
|
+
I
|
|
19
|
+
J
|
|
20
|
+
K
|
|
21
|
+
L
|
|
22
|
+
M
|
|
23
|
+
N
|
|
24
|
+
O
|
|
25
|
+
P
|
|
26
|
+
Q
|
|
27
|
+
R
|
|
28
|
+
S
|
|
29
|
+
T
|
|
30
|
+
U
|
|
31
|
+
V
|
|
32
|
+
W
|
|
33
|
+
X
|
|
34
|
+
Y
|
|
35
|
+
Z
|
|
36
|
+
Å
|
|
37
|
+
Ä
|
|
38
|
+
Ö
|
|
39
|
+
|
|
40
|
+
#List of titles. These are often followed by upper-case names, but do not indicate sentence breaks
|
|
41
|
+
alik
|
|
42
|
+
alil
|
|
43
|
+
amir
|
|
44
|
+
apul
|
|
45
|
+
apul.prof
|
|
46
|
+
arkkit
|
|
47
|
+
ass
|
|
48
|
+
assist
|
|
49
|
+
dipl
|
|
50
|
+
dipl.arkkit
|
|
51
|
+
dipl.ekon
|
|
52
|
+
dipl.ins
|
|
53
|
+
dipl.kielenk
|
|
54
|
+
dipl.kirjeenv
|
|
55
|
+
dipl.kosm
|
|
56
|
+
dipl.urk
|
|
57
|
+
dos
|
|
58
|
+
erikoiseläinl
|
|
59
|
+
erikoishammasl
|
|
60
|
+
erikoisl
|
|
61
|
+
erikoist
|
|
62
|
+
ev.luutn
|
|
63
|
+
evp
|
|
64
|
+
fil
|
|
65
|
+
ft
|
|
66
|
+
hallinton
|
|
67
|
+
hallintot
|
|
68
|
+
hammaslääket
|
|
69
|
+
jatk
|
|
70
|
+
jääk
|
|
71
|
+
kansaned
|
|
72
|
+
kapt
|
|
73
|
+
kapt.luutn
|
|
74
|
+
kenr
|
|
75
|
+
kenr.luutn
|
|
76
|
+
kenr.maj
|
|
77
|
+
kers
|
|
78
|
+
kirjeenv
|
|
79
|
+
kom
|
|
80
|
+
kom.kapt
|
|
81
|
+
komm
|
|
82
|
+
konst
|
|
83
|
+
korpr
|
|
84
|
+
luutn
|
|
85
|
+
maist
|
|
86
|
+
maj
|
|
87
|
+
Mr
|
|
88
|
+
Mrs
|
|
89
|
+
Ms
|
|
90
|
+
M.Sc
|
|
91
|
+
neuv
|
|
92
|
+
nimim
|
|
93
|
+
Ph.D
|
|
94
|
+
prof
|
|
95
|
+
puh.joht
|
|
96
|
+
pääll
|
|
97
|
+
res
|
|
98
|
+
san
|
|
99
|
+
siht
|
|
100
|
+
suom
|
|
101
|
+
sähköp
|
|
102
|
+
säv
|
|
103
|
+
toht
|
|
104
|
+
toim
|
|
105
|
+
toim.apul
|
|
106
|
+
toim.joht
|
|
107
|
+
toim.siht
|
|
108
|
+
tuom
|
|
109
|
+
ups
|
|
110
|
+
vänr
|
|
111
|
+
vääp
|
|
112
|
+
ye.ups
|
|
113
|
+
ylik
|
|
114
|
+
ylil
|
|
115
|
+
ylim
|
|
116
|
+
ylimatr
|
|
117
|
+
yliop
|
|
118
|
+
yliopp
|
|
119
|
+
ylip
|
|
120
|
+
yliv
|
|
121
|
+
|
|
122
|
+
#misc - odd period-ending items that NEVER indicate breaks (p.m. does NOT fall
|
|
123
|
+
#into this category - it sometimes ends a sentence)
|
|
124
|
+
e.g
|
|
125
|
+
ent
|
|
126
|
+
esim
|
|
127
|
+
huom
|
|
128
|
+
i.e
|
|
129
|
+
ilm
|
|
130
|
+
l
|
|
131
|
+
mm
|
|
132
|
+
myöh
|
|
133
|
+
nk
|
|
134
|
+
nyk
|
|
135
|
+
par
|
|
136
|
+
po
|
|
137
|
+
t
|
|
138
|
+
v
|
package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.fr
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
#Anything in this file, followed by a period (and an upper-case word), does NOT indicate an end-of-sentence marker.
|
|
2
|
+
#Special cases are included for prefixes that ONLY appear before 0-9 numbers.
|
|
3
|
+
#
|
|
4
|
+
#any single upper case letter followed by a period is not a sentence ender
|
|
5
|
+
#usually upper case letters are initials in a name
|
|
6
|
+
#no French words end in single lower-case letters, so we throw those in too?
|
|
7
|
+
A
|
|
8
|
+
B
|
|
9
|
+
C
|
|
10
|
+
D
|
|
11
|
+
E
|
|
12
|
+
F
|
|
13
|
+
G
|
|
14
|
+
H
|
|
15
|
+
I
|
|
16
|
+
J
|
|
17
|
+
K
|
|
18
|
+
L
|
|
19
|
+
M
|
|
20
|
+
N
|
|
21
|
+
O
|
|
22
|
+
P
|
|
23
|
+
Q
|
|
24
|
+
R
|
|
25
|
+
S
|
|
26
|
+
T
|
|
27
|
+
U
|
|
28
|
+
V
|
|
29
|
+
W
|
|
30
|
+
X
|
|
31
|
+
Y
|
|
32
|
+
Z
|
|
33
|
+
#a
|
|
34
|
+
b
|
|
35
|
+
c
|
|
36
|
+
d
|
|
37
|
+
e
|
|
38
|
+
f
|
|
39
|
+
g
|
|
40
|
+
h
|
|
41
|
+
i
|
|
42
|
+
j
|
|
43
|
+
k
|
|
44
|
+
l
|
|
45
|
+
m
|
|
46
|
+
n
|
|
47
|
+
o
|
|
48
|
+
p
|
|
49
|
+
q
|
|
50
|
+
r
|
|
51
|
+
s
|
|
52
|
+
t
|
|
53
|
+
u
|
|
54
|
+
v
|
|
55
|
+
w
|
|
56
|
+
x
|
|
57
|
+
y
|
|
58
|
+
z
|
|
59
|
+
|
|
60
|
+
# Period-final abbreviation list for French
|
|
61
|
+
A.C.N
|
|
62
|
+
A.M
|
|
63
|
+
art
|
|
64
|
+
ann
|
|
65
|
+
apr
|
|
66
|
+
av
|
|
67
|
+
auj
|
|
68
|
+
lib
|
|
69
|
+
B.P
|
|
70
|
+
boul
|
|
71
|
+
ca
|
|
72
|
+
c.-à-d
|
|
73
|
+
cf
|
|
74
|
+
ch.-l
|
|
75
|
+
chap
|
|
76
|
+
contr
|
|
77
|
+
C.P.I
|
|
78
|
+
C.Q.F.D
|
|
79
|
+
C.N
|
|
80
|
+
C.N.S
|
|
81
|
+
C.S
|
|
82
|
+
dir
|
|
83
|
+
éd
|
|
84
|
+
e.g
|
|
85
|
+
env
|
|
86
|
+
al
|
|
87
|
+
etc
|
|
88
|
+
E.V
|
|
89
|
+
ex
|
|
90
|
+
fasc
|
|
91
|
+
fém
|
|
92
|
+
fig
|
|
93
|
+
fr
|
|
94
|
+
hab
|
|
95
|
+
ibid
|
|
96
|
+
id
|
|
97
|
+
i.e
|
|
98
|
+
inf
|
|
99
|
+
LL.AA
|
|
100
|
+
LL.AA.II
|
|
101
|
+
LL.AA.RR
|
|
102
|
+
LL.AA.SS
|
|
103
|
+
L.D
|
|
104
|
+
LL.EE
|
|
105
|
+
LL.MM
|
|
106
|
+
LL.MM.II.RR
|
|
107
|
+
loc.cit
|
|
108
|
+
masc
|
|
109
|
+
MM
|
|
110
|
+
ms
|
|
111
|
+
N.B
|
|
112
|
+
N.D.A
|
|
113
|
+
N.D.L.R
|
|
114
|
+
N.D.T
|
|
115
|
+
n/réf
|
|
116
|
+
NN.SS
|
|
117
|
+
N.S
|
|
118
|
+
N.D
|
|
119
|
+
N.P.A.I
|
|
120
|
+
p.c.c
|
|
121
|
+
pl
|
|
122
|
+
pp
|
|
123
|
+
p.ex
|
|
124
|
+
p.j
|
|
125
|
+
P.S
|
|
126
|
+
R.A.S
|
|
127
|
+
R.-V
|
|
128
|
+
R.P
|
|
129
|
+
R.I.P
|
|
130
|
+
SS
|
|
131
|
+
S.S
|
|
132
|
+
S.A
|
|
133
|
+
S.A.I
|
|
134
|
+
S.A.R
|
|
135
|
+
S.A.S
|
|
136
|
+
S.E
|
|
137
|
+
sec
|
|
138
|
+
sect
|
|
139
|
+
sing
|
|
140
|
+
S.M
|
|
141
|
+
S.M.I.R
|
|
142
|
+
sq
|
|
143
|
+
sqq
|
|
144
|
+
suiv
|
|
145
|
+
sup
|
|
146
|
+
suppl
|
|
147
|
+
tél
|
|
148
|
+
T.S.V.P
|
|
149
|
+
vb
|
|
150
|
+
vol
|
|
151
|
+
vs
|
|
152
|
+
X.O
|
|
153
|
+
Z.I
|
package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.ga
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
|
|
2
|
+
A
|
|
3
|
+
B
|
|
4
|
+
C
|
|
5
|
+
D
|
|
6
|
+
E
|
|
7
|
+
F
|
|
8
|
+
G
|
|
9
|
+
H
|
|
10
|
+
I
|
|
11
|
+
J
|
|
12
|
+
K
|
|
13
|
+
L
|
|
14
|
+
M
|
|
15
|
+
N
|
|
16
|
+
O
|
|
17
|
+
P
|
|
18
|
+
Q
|
|
19
|
+
R
|
|
20
|
+
S
|
|
21
|
+
T
|
|
22
|
+
U
|
|
23
|
+
V
|
|
24
|
+
W
|
|
25
|
+
X
|
|
26
|
+
Y
|
|
27
|
+
Z
|
|
28
|
+
Á
|
|
29
|
+
É
|
|
30
|
+
Í
|
|
31
|
+
Ó
|
|
32
|
+
Ú
|
|
33
|
+
|
|
34
|
+
Uacht
|
|
35
|
+
Dr
|
|
36
|
+
B.Arch
|
|
37
|
+
|
|
38
|
+
m.sh
|
|
39
|
+
.i
|
|
40
|
+
Co
|
|
41
|
+
Cf
|
|
42
|
+
cf
|
|
43
|
+
i.e
|
|
44
|
+
r
|
|
45
|
+
Chr
|
|
46
|
+
lch #NUMERIC_ONLY#
|
|
47
|
+
lgh #NUMERIC_ONLY#
|
|
48
|
+
uimh #NUMERIC_ONLY#
|