@qvac/translation-nmtcpp 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +470 -0
- package/binding.js +1 -0
- package/index.d.ts +82 -0
- package/index.js +188 -0
- package/lib/error.js +65 -0
- package/marian.js +186 -0
- package/package.json +69 -0
- package/prebuilds/android-arm/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/android-arm64/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/android-ia32/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/android-x64/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/darwin-arm64/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/darwin-arm64/qvac__translation-nmtcpp.bare.exports +3622 -0
- package/prebuilds/darwin-x64/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/darwin-x64/qvac__translation-nmtcpp.bare.exports +3731 -0
- package/prebuilds/ios-arm64/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/ios-arm64/qvac__translation-nmtcpp.bare.exports +3603 -0
- package/prebuilds/ios-arm64-simulator/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/ios-arm64-simulator/qvac__translation-nmtcpp.bare.exports +3603 -0
- package/prebuilds/ios-x64-simulator/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/ios-x64-simulator/qvac__translation-nmtcpp.bare.exports +3720 -0
- package/prebuilds/linux-x64/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/win32-x64/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/win32-x64/qvac__translation-nmtcpp.bare.exports +0 -0
- package/third-party/indic-processor-deps/indicnlp/INDIC_NLP_LICENCE +9 -0
- package/third-party/indic-processor-deps/indicnlp/index.js +11 -0
- package/third-party/indic-processor-deps/indicnlp/indic_detokenize.js +141 -0
- package/third-party/indic-processor-deps/indicnlp/indic_normalize.js +1213 -0
- package/third-party/indic-processor-deps/indicnlp/indic_tokenize.js +123 -0
- package/third-party/indic-processor-deps/indicnlp/langinfo.js +609 -0
- package/third-party/indic-processor-deps/indicnlp/sinhala_transliterator.js +197 -0
- package/third-party/indic-processor-deps/indicnlp/unicode_transliterator.js +120 -0
- package/third-party/indic-processor-deps/sacremoses/SACREMOSES_LICENCE +21 -0
- package/third-party/indic-processor-deps/sacremoses/cjk.js +202 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/README.txt +8 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.as +65 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.bn +65 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.ca +75 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.cs +390 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.de +325 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.el +1568 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.en +123 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.es +118 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.et +138 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.fi +138 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.fr +153 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.ga +48 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.gu +105 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.hi +113 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.hu +103 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.is +251 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.it +180 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.kn +70 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.lt +698 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.lv +100 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.ml +67 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.mni +65 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.mr +113 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.nl +115 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.or +101 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.pa +102 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.pl +283 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.pt +210 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.ro +38 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.ru +293 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.sk +474 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.sl +78 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.sv +97 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.ta +71 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.tdt +210 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.te +70 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.yue +53 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.zh +53 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/CJK.txt +23246 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/CJKSymbols.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Close_Punctuation.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Currency_Symbol.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Han.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Hangul.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Hangul_Syllables.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Hiragana.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsAlnum-unichars-au.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsAlnum.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsAlpha-unichars-au.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsAlpha.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsLower.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsN.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsPf.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsPi.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsSc.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsSo.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsUpper.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Katakana.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Line_Separator.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Lowercase_Letter.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Number.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Open_Punctuation.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Punctuation.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Separator.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Symbol.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Titlecase_Letter.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Uppercase_Letter.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/index.js +8 -0
- package/third-party/indic-processor-deps/sacremoses/indic.js +76 -0
- package/third-party/indic-processor-deps/sacremoses/normalizer.js +264 -0
- package/third-party/indic-processor-deps/sacremoses/pernuliprops.js +287 -0
- package/third-party/indic-processor-deps/sacremoses/tokenizer.js +1217 -0
- package/third-party/indic-processor.js +565 -0
package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.lv
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
#Anything in this file, followed by a period (and an upper-case word), does NOT indicate an end-of-sentence marker.
|
|
2
|
+
#Special cases are included for prefixes that ONLY appear before 0-9 numbers.
|
|
3
|
+
|
|
4
|
+
#any single upper case letter followed by a period is not a sentence ender (excluding I occasionally, but we leave it in)
|
|
5
|
+
#usually upper case letters are initials in a name
|
|
6
|
+
A
|
|
7
|
+
Ā
|
|
8
|
+
B
|
|
9
|
+
C
|
|
10
|
+
Č
|
|
11
|
+
D
|
|
12
|
+
E
|
|
13
|
+
Ē
|
|
14
|
+
F
|
|
15
|
+
G
|
|
16
|
+
Ģ
|
|
17
|
+
H
|
|
18
|
+
I
|
|
19
|
+
Ī
|
|
20
|
+
J
|
|
21
|
+
K
|
|
22
|
+
Ķ
|
|
23
|
+
L
|
|
24
|
+
Ļ
|
|
25
|
+
M
|
|
26
|
+
N
|
|
27
|
+
Ņ
|
|
28
|
+
O
|
|
29
|
+
P
|
|
30
|
+
Q
|
|
31
|
+
R
|
|
32
|
+
S
|
|
33
|
+
Š
|
|
34
|
+
T
|
|
35
|
+
U
|
|
36
|
+
Ū
|
|
37
|
+
V
|
|
38
|
+
W
|
|
39
|
+
X
|
|
40
|
+
Y
|
|
41
|
+
Z
|
|
42
|
+
Ž
|
|
43
|
+
|
|
44
|
+
#List of titles. These are often followed by upper-case names, but do not indicate sentence breaks
|
|
45
|
+
dr
|
|
46
|
+
Dr
|
|
47
|
+
med
|
|
48
|
+
prof
|
|
49
|
+
Prof
|
|
50
|
+
inž
|
|
51
|
+
Inž
|
|
52
|
+
ist.loc
|
|
53
|
+
Ist.loc
|
|
54
|
+
kor.loc
|
|
55
|
+
Kor.loc
|
|
56
|
+
v.i
|
|
57
|
+
vietn
|
|
58
|
+
Vietn
|
|
59
|
+
|
|
60
|
+
#misc - odd period-ending items that NEVER indicate breaks (p.m. does NOT fall into this category - it sometimes ends a sentence)
|
|
61
|
+
a.l
|
|
62
|
+
t.p
|
|
63
|
+
pārb
|
|
64
|
+
Pārb
|
|
65
|
+
vec
|
|
66
|
+
Vec
|
|
67
|
+
inv
|
|
68
|
+
Inv
|
|
69
|
+
sk
|
|
70
|
+
Sk
|
|
71
|
+
spec
|
|
72
|
+
Spec
|
|
73
|
+
vienk
|
|
74
|
+
Vienk
|
|
75
|
+
virz
|
|
76
|
+
Virz
|
|
77
|
+
māksl
|
|
78
|
+
Māksl
|
|
79
|
+
mūz
|
|
80
|
+
Mūz
|
|
81
|
+
akad
|
|
82
|
+
Akad
|
|
83
|
+
soc
|
|
84
|
+
Soc
|
|
85
|
+
galv
|
|
86
|
+
Galv
|
|
87
|
+
vad
|
|
88
|
+
Vad
|
|
89
|
+
sertif
|
|
90
|
+
Sertif
|
|
91
|
+
folkl
|
|
92
|
+
Folkl
|
|
93
|
+
hum
|
|
94
|
+
Hum
|
|
95
|
+
|
|
96
|
+
#Numbers only. These should only induce breaks when followed by a numeric sequence
|
|
97
|
+
# add NUMERIC_ONLY after the word for this function
|
|
98
|
+
#This case is mostly for the english "No." which can either be a sentence of its own, or
|
|
99
|
+
#if followed by a number, a non-breaking prefix
|
|
100
|
+
Nr #NUMERIC_ONLY#
|
package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.ml
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
#Anything in this file, followed by a period (and an upper-case word), does NOT indicate an end-of-sentence marker.
|
|
2
|
+
|
|
3
|
+
#common exceptions
|
|
4
|
+
# Dr
|
|
5
|
+
ഡോ
|
|
6
|
+
# Mr
|
|
7
|
+
ശ്രീ
|
|
8
|
+
|
|
9
|
+
#others
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
#phonetics
|
|
13
|
+
# A
|
|
14
|
+
എ
|
|
15
|
+
# B
|
|
16
|
+
ബി
|
|
17
|
+
# C
|
|
18
|
+
സി
|
|
19
|
+
# D
|
|
20
|
+
ഡി
|
|
21
|
+
# E
|
|
22
|
+
ഇ
|
|
23
|
+
# F
|
|
24
|
+
എഫ്
|
|
25
|
+
# G
|
|
26
|
+
ജി
|
|
27
|
+
# H
|
|
28
|
+
എച്ച്
|
|
29
|
+
# I
|
|
30
|
+
ഐ
|
|
31
|
+
# J
|
|
32
|
+
ജെ
|
|
33
|
+
# K
|
|
34
|
+
കെ
|
|
35
|
+
# L
|
|
36
|
+
എൽ
|
|
37
|
+
# M
|
|
38
|
+
എം
|
|
39
|
+
# N
|
|
40
|
+
എൻ
|
|
41
|
+
# O
|
|
42
|
+
ഒ
|
|
43
|
+
# P
|
|
44
|
+
പി
|
|
45
|
+
# Q
|
|
46
|
+
ക്യൂ
|
|
47
|
+
# R
|
|
48
|
+
ആർ
|
|
49
|
+
# S
|
|
50
|
+
എസ്
|
|
51
|
+
# T
|
|
52
|
+
ടി
|
|
53
|
+
# U
|
|
54
|
+
യു
|
|
55
|
+
# V
|
|
56
|
+
വി
|
|
57
|
+
# W
|
|
58
|
+
ഡബ്ല്യു
|
|
59
|
+
# X
|
|
60
|
+
എക്സ്
|
|
61
|
+
# Y
|
|
62
|
+
വൈ
|
|
63
|
+
# Z
|
|
64
|
+
സെഡ്
|
|
65
|
+
|
|
66
|
+
#consonants
|
|
67
|
+
|
package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.mni
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
#Anything in this file, followed by a period (and an upper-case word), does NOT indicate an end-of-sentence marker.
|
|
2
|
+
|
|
3
|
+
#common exceptions
|
|
4
|
+
# Dr
|
|
5
|
+
দা
|
|
6
|
+
|
|
7
|
+
#others
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
#phonetics
|
|
11
|
+
# A
|
|
12
|
+
এ
|
|
13
|
+
# B
|
|
14
|
+
বি
|
|
15
|
+
# C
|
|
16
|
+
সি
|
|
17
|
+
# D
|
|
18
|
+
ডি
|
|
19
|
+
# E
|
|
20
|
+
ই
|
|
21
|
+
# F
|
|
22
|
+
এফ
|
|
23
|
+
# G
|
|
24
|
+
জি
|
|
25
|
+
# H
|
|
26
|
+
এইচ
|
|
27
|
+
# I
|
|
28
|
+
আম
|
|
29
|
+
# J
|
|
30
|
+
জে
|
|
31
|
+
# K
|
|
32
|
+
কে
|
|
33
|
+
# L
|
|
34
|
+
এল
|
|
35
|
+
# M
|
|
36
|
+
এম
|
|
37
|
+
# N
|
|
38
|
+
এন
|
|
39
|
+
# O
|
|
40
|
+
হে
|
|
41
|
+
# P
|
|
42
|
+
পি
|
|
43
|
+
# Q
|
|
44
|
+
কিউ
|
|
45
|
+
# R
|
|
46
|
+
আর
|
|
47
|
+
# S
|
|
48
|
+
এস
|
|
49
|
+
# T
|
|
50
|
+
টি
|
|
51
|
+
# U
|
|
52
|
+
ইউ
|
|
53
|
+
# V
|
|
54
|
+
ভি
|
|
55
|
+
# W
|
|
56
|
+
ডব্লু
|
|
57
|
+
# X
|
|
58
|
+
এক্স
|
|
59
|
+
# Y
|
|
60
|
+
ওয়াই
|
|
61
|
+
# Z
|
|
62
|
+
জেড
|
|
63
|
+
|
|
64
|
+
#consonants
|
|
65
|
+
|
package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.mr
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
#Anything in this file, followed by a period (and an upper-case word), does NOT indicate an end-of-sentence marker.
|
|
2
|
+
|
|
3
|
+
#common exceptions
|
|
4
|
+
# Rs
|
|
5
|
+
रु
|
|
6
|
+
# Dr
|
|
7
|
+
डॉ
|
|
8
|
+
# Dr
|
|
9
|
+
डा
|
|
10
|
+
# Mr
|
|
11
|
+
श्री
|
|
12
|
+
|
|
13
|
+
#others
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
#phonetics
|
|
17
|
+
# A
|
|
18
|
+
ए
|
|
19
|
+
ऐ
|
|
20
|
+
# B
|
|
21
|
+
बी
|
|
22
|
+
# C
|
|
23
|
+
सी
|
|
24
|
+
# D
|
|
25
|
+
डी
|
|
26
|
+
# E
|
|
27
|
+
ई
|
|
28
|
+
# F
|
|
29
|
+
ऐफ
|
|
30
|
+
एफ
|
|
31
|
+
# G
|
|
32
|
+
जी
|
|
33
|
+
# H
|
|
34
|
+
ऐच
|
|
35
|
+
एच
|
|
36
|
+
# I
|
|
37
|
+
आइ
|
|
38
|
+
# J
|
|
39
|
+
जे
|
|
40
|
+
# K
|
|
41
|
+
के
|
|
42
|
+
# L
|
|
43
|
+
ऐल
|
|
44
|
+
एल
|
|
45
|
+
# M
|
|
46
|
+
ऐम
|
|
47
|
+
एम
|
|
48
|
+
# N
|
|
49
|
+
ऐन
|
|
50
|
+
एन
|
|
51
|
+
# O
|
|
52
|
+
ओ
|
|
53
|
+
# P
|
|
54
|
+
पी
|
|
55
|
+
# Q
|
|
56
|
+
क्यू
|
|
57
|
+
# R
|
|
58
|
+
आर
|
|
59
|
+
# S
|
|
60
|
+
ऐस
|
|
61
|
+
एस
|
|
62
|
+
# T
|
|
63
|
+
टी
|
|
64
|
+
# U
|
|
65
|
+
यू
|
|
66
|
+
# V
|
|
67
|
+
वी
|
|
68
|
+
# W
|
|
69
|
+
डब्ल्यू
|
|
70
|
+
# X
|
|
71
|
+
ऐक्स
|
|
72
|
+
एक्स
|
|
73
|
+
# Y
|
|
74
|
+
वाय
|
|
75
|
+
वाई
|
|
76
|
+
# Z
|
|
77
|
+
ज़ैड
|
|
78
|
+
|
|
79
|
+
#consonants
|
|
80
|
+
क
|
|
81
|
+
ख
|
|
82
|
+
ग
|
|
83
|
+
घ
|
|
84
|
+
ङ
|
|
85
|
+
च
|
|
86
|
+
छ
|
|
87
|
+
ज
|
|
88
|
+
झ
|
|
89
|
+
ञ
|
|
90
|
+
ट
|
|
91
|
+
ठ
|
|
92
|
+
ड
|
|
93
|
+
ढ
|
|
94
|
+
ण
|
|
95
|
+
त
|
|
96
|
+
थ
|
|
97
|
+
द
|
|
98
|
+
ध
|
|
99
|
+
न
|
|
100
|
+
प
|
|
101
|
+
फ
|
|
102
|
+
ब
|
|
103
|
+
भ
|
|
104
|
+
म
|
|
105
|
+
य
|
|
106
|
+
र
|
|
107
|
+
ल
|
|
108
|
+
व
|
|
109
|
+
श
|
|
110
|
+
ष
|
|
111
|
+
स
|
|
112
|
+
ह
|
|
113
|
+
|
package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.nl
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
#Anything in this file, followed by a period (and an upper-case word), does NOT indicate an end-of-sentence marker.
|
|
2
|
+
#Special cases are included for prefixes that ONLY appear before 0-9 numbers.
|
|
3
|
+
#Sources: http://nl.wikipedia.org/wiki/Lijst_van_afkortingen
|
|
4
|
+
# http://nl.wikipedia.org/wiki/Aanspreekvorm
|
|
5
|
+
# http://nl.wikipedia.org/wiki/Titulatuur_in_het_Nederlands_hoger_onderwijs
|
|
6
|
+
#any single upper case letter followed by a period is not a sentence ender (excluding I occasionally, but we leave it in)
|
|
7
|
+
#usually upper case letters are initials in a name
|
|
8
|
+
A
|
|
9
|
+
B
|
|
10
|
+
C
|
|
11
|
+
D
|
|
12
|
+
E
|
|
13
|
+
F
|
|
14
|
+
G
|
|
15
|
+
H
|
|
16
|
+
I
|
|
17
|
+
J
|
|
18
|
+
K
|
|
19
|
+
L
|
|
20
|
+
M
|
|
21
|
+
N
|
|
22
|
+
O
|
|
23
|
+
P
|
|
24
|
+
Q
|
|
25
|
+
R
|
|
26
|
+
S
|
|
27
|
+
T
|
|
28
|
+
U
|
|
29
|
+
V
|
|
30
|
+
W
|
|
31
|
+
X
|
|
32
|
+
Y
|
|
33
|
+
Z
|
|
34
|
+
|
|
35
|
+
#List of titles. These are often followed by upper-case names, but do not indicate sentence breaks
|
|
36
|
+
bacc
|
|
37
|
+
bc
|
|
38
|
+
bgen
|
|
39
|
+
c.i
|
|
40
|
+
dhr
|
|
41
|
+
dr
|
|
42
|
+
dr.h.c
|
|
43
|
+
drs
|
|
44
|
+
drs
|
|
45
|
+
ds
|
|
46
|
+
eint
|
|
47
|
+
fa
|
|
48
|
+
Fa
|
|
49
|
+
fam
|
|
50
|
+
gen
|
|
51
|
+
genm
|
|
52
|
+
ing
|
|
53
|
+
ir
|
|
54
|
+
jhr
|
|
55
|
+
jkvr
|
|
56
|
+
jr
|
|
57
|
+
kand
|
|
58
|
+
kol
|
|
59
|
+
lgen
|
|
60
|
+
lkol
|
|
61
|
+
Lt
|
|
62
|
+
maj
|
|
63
|
+
Mej
|
|
64
|
+
mevr
|
|
65
|
+
Mme
|
|
66
|
+
mr
|
|
67
|
+
mr
|
|
68
|
+
Mw
|
|
69
|
+
o.b.s
|
|
70
|
+
plv
|
|
71
|
+
prof
|
|
72
|
+
ritm
|
|
73
|
+
tint
|
|
74
|
+
Vz
|
|
75
|
+
Z.D
|
|
76
|
+
Z.D.H
|
|
77
|
+
Z.E
|
|
78
|
+
Z.Em
|
|
79
|
+
Z.H
|
|
80
|
+
Z.K.H
|
|
81
|
+
Z.K.M
|
|
82
|
+
Z.M
|
|
83
|
+
z.v
|
|
84
|
+
|
|
85
|
+
#misc - odd period-ending items that NEVER indicate breaks (p.m. does NOT fall into this category - it sometimes ends a sentence)
|
|
86
|
+
#we seem to have a lot of these in dutch i.e.: i.p.v - in plaats van (in stead of) never ends a sentence
|
|
87
|
+
a.g.v
|
|
88
|
+
bijv
|
|
89
|
+
bijz
|
|
90
|
+
bv
|
|
91
|
+
d.w.z
|
|
92
|
+
e.c
|
|
93
|
+
e.g
|
|
94
|
+
e.k
|
|
95
|
+
ev
|
|
96
|
+
i.p.v
|
|
97
|
+
i.s.m
|
|
98
|
+
i.t.t
|
|
99
|
+
i.v.m
|
|
100
|
+
m.a.w
|
|
101
|
+
m.b.t
|
|
102
|
+
m.b.v
|
|
103
|
+
m.h.o
|
|
104
|
+
m.i
|
|
105
|
+
m.i.v
|
|
106
|
+
v.w.t
|
|
107
|
+
|
|
108
|
+
#Numbers only. These should only induce breaks when followed by a numeric sequence
|
|
109
|
+
# add NUMERIC_ONLY after the word for this function
|
|
110
|
+
#This case is mostly for the english "No." which can either be a sentence of its own, or
|
|
111
|
+
#if followed by a number, a non-breaking prefix
|
|
112
|
+
Nr #NUMERIC_ONLY#
|
|
113
|
+
Nrs
|
|
114
|
+
nrs
|
|
115
|
+
nr #NUMERIC_ONLY#
|
package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.or
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
#Anything in this file, followed by a period (and an upper-case word), does NOT indicate an end-of-sentence marker.
|
|
2
|
+
|
|
3
|
+
#common exceptions
|
|
4
|
+
# Mr
|
|
5
|
+
ରୀ
|
|
6
|
+
|
|
7
|
+
#others
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
#phonetics
|
|
11
|
+
# A
|
|
12
|
+
|
|
13
|
+
# B
|
|
14
|
+
|
|
15
|
+
# C
|
|
16
|
+
|
|
17
|
+
# D
|
|
18
|
+
|
|
19
|
+
# E
|
|
20
|
+
|
|
21
|
+
# F
|
|
22
|
+
|
|
23
|
+
# G
|
|
24
|
+
|
|
25
|
+
# H
|
|
26
|
+
|
|
27
|
+
# I
|
|
28
|
+
|
|
29
|
+
# J
|
|
30
|
+
|
|
31
|
+
# K
|
|
32
|
+
|
|
33
|
+
# L
|
|
34
|
+
|
|
35
|
+
# M
|
|
36
|
+
|
|
37
|
+
# N
|
|
38
|
+
|
|
39
|
+
# O
|
|
40
|
+
|
|
41
|
+
# P
|
|
42
|
+
|
|
43
|
+
# Q
|
|
44
|
+
|
|
45
|
+
# R
|
|
46
|
+
|
|
47
|
+
# S
|
|
48
|
+
|
|
49
|
+
# T
|
|
50
|
+
|
|
51
|
+
# U
|
|
52
|
+
|
|
53
|
+
# V
|
|
54
|
+
|
|
55
|
+
# W
|
|
56
|
+
|
|
57
|
+
# X
|
|
58
|
+
|
|
59
|
+
# Y
|
|
60
|
+
|
|
61
|
+
# Z
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
#consonants
|
|
65
|
+
କ
|
|
66
|
+
ଖ
|
|
67
|
+
ଗ
|
|
68
|
+
ଘ
|
|
69
|
+
ଙ
|
|
70
|
+
ଚ
|
|
71
|
+
ଛ
|
|
72
|
+
ଜ
|
|
73
|
+
ଝ
|
|
74
|
+
ଞ
|
|
75
|
+
ଟ
|
|
76
|
+
ଠ
|
|
77
|
+
ଡ
|
|
78
|
+
ଢ
|
|
79
|
+
ଣ
|
|
80
|
+
ତ
|
|
81
|
+
ଥ
|
|
82
|
+
ଦ
|
|
83
|
+
ଧ
|
|
84
|
+
ନ
|
|
85
|
+
ପ
|
|
86
|
+
ଫ
|
|
87
|
+
ବ
|
|
88
|
+
ଵ
|
|
89
|
+
ଭ
|
|
90
|
+
ମ
|
|
91
|
+
ଯ
|
|
92
|
+
ୟ
|
|
93
|
+
ର
|
|
94
|
+
ଲ
|
|
95
|
+
ଳ
|
|
96
|
+
ୱ
|
|
97
|
+
ଶ
|
|
98
|
+
ଷ
|
|
99
|
+
ସ
|
|
100
|
+
ହ
|
|
101
|
+
|
package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.pa
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
#Anything in this file, followed by a period (and an upper-case word), does NOT indicate an end-of-sentence marker.
|
|
2
|
+
|
|
3
|
+
#common exceptions
|
|
4
|
+
# Dr
|
|
5
|
+
ਡਾ
|
|
6
|
+
# Dr
|
|
7
|
+
ਪ੍ਰੋ
|
|
8
|
+
# Mr
|
|
9
|
+
ਸ੍ਰੀ
|
|
10
|
+
|
|
11
|
+
#others
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
#phonetics
|
|
15
|
+
# A
|
|
16
|
+
ਏ
|
|
17
|
+
# B
|
|
18
|
+
ਬੀ
|
|
19
|
+
# C
|
|
20
|
+
ਸੀ
|
|
21
|
+
# D
|
|
22
|
+
ਡੀ
|
|
23
|
+
# E
|
|
24
|
+
ਈ
|
|
25
|
+
# F
|
|
26
|
+
ਐੱਫ
|
|
27
|
+
# G
|
|
28
|
+
ਜੀ
|
|
29
|
+
# H
|
|
30
|
+
ਐਚ
|
|
31
|
+
# I
|
|
32
|
+
ਆਈ
|
|
33
|
+
# J
|
|
34
|
+
ਜੇ
|
|
35
|
+
# K
|
|
36
|
+
ਕੇ
|
|
37
|
+
# L
|
|
38
|
+
ਐਲ
|
|
39
|
+
# M
|
|
40
|
+
ਐੱਮ
|
|
41
|
+
# N
|
|
42
|
+
ਐੱਨ
|
|
43
|
+
# O
|
|
44
|
+
ਓ
|
|
45
|
+
# P
|
|
46
|
+
ਪੀ
|
|
47
|
+
# Q
|
|
48
|
+
ਕੀਓ
|
|
49
|
+
# R
|
|
50
|
+
ਆਰ
|
|
51
|
+
# S
|
|
52
|
+
ਐੱਸ
|
|
53
|
+
ਸ
|
|
54
|
+
# T
|
|
55
|
+
ਟੀ
|
|
56
|
+
# U
|
|
57
|
+
ਯੂ
|
|
58
|
+
# V
|
|
59
|
+
ਵੀ
|
|
60
|
+
# W
|
|
61
|
+
ਡਬਲਿਊ
|
|
62
|
+
# X
|
|
63
|
+
ਐਕ੍ਸ
|
|
64
|
+
# Y
|
|
65
|
+
ਵਾਈ
|
|
66
|
+
# Z
|
|
67
|
+
ਜ਼ੈਡ
|
|
68
|
+
|
|
69
|
+
#consonants
|
|
70
|
+
ਕ
|
|
71
|
+
ਖ
|
|
72
|
+
ਗ
|
|
73
|
+
ਘ
|
|
74
|
+
ਙ
|
|
75
|
+
ਚ
|
|
76
|
+
ਛ
|
|
77
|
+
ਜ
|
|
78
|
+
ਝ
|
|
79
|
+
ਞ
|
|
80
|
+
ਟ
|
|
81
|
+
ਠ
|
|
82
|
+
ਡ
|
|
83
|
+
ਢ
|
|
84
|
+
ਣ
|
|
85
|
+
ਤ
|
|
86
|
+
ਥ
|
|
87
|
+
ਦ
|
|
88
|
+
ਧ
|
|
89
|
+
ਨ
|
|
90
|
+
ਪ
|
|
91
|
+
ਫ
|
|
92
|
+
ਬ
|
|
93
|
+
ਭ
|
|
94
|
+
ਮ
|
|
95
|
+
ਯ
|
|
96
|
+
ਰ
|
|
97
|
+
ਲ
|
|
98
|
+
ਵ
|
|
99
|
+
ੜ
|
|
100
|
+
ਸ
|
|
101
|
+
ਹ
|
|
102
|
+
|