@qvac/translation-nmtcpp 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +470 -0
- package/binding.js +1 -0
- package/index.d.ts +82 -0
- package/index.js +188 -0
- package/lib/error.js +65 -0
- package/marian.js +186 -0
- package/package.json +69 -0
- package/prebuilds/android-arm/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/android-arm64/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/android-ia32/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/android-x64/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/darwin-arm64/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/darwin-arm64/qvac__translation-nmtcpp.bare.exports +3622 -0
- package/prebuilds/darwin-x64/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/darwin-x64/qvac__translation-nmtcpp.bare.exports +3731 -0
- package/prebuilds/ios-arm64/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/ios-arm64/qvac__translation-nmtcpp.bare.exports +3603 -0
- package/prebuilds/ios-arm64-simulator/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/ios-arm64-simulator/qvac__translation-nmtcpp.bare.exports +3603 -0
- package/prebuilds/ios-x64-simulator/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/ios-x64-simulator/qvac__translation-nmtcpp.bare.exports +3720 -0
- package/prebuilds/linux-x64/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/win32-x64/qvac__translation-nmtcpp.bare +0 -0
- package/prebuilds/win32-x64/qvac__translation-nmtcpp.bare.exports +0 -0
- package/third-party/indic-processor-deps/indicnlp/INDIC_NLP_LICENCE +9 -0
- package/third-party/indic-processor-deps/indicnlp/index.js +11 -0
- package/third-party/indic-processor-deps/indicnlp/indic_detokenize.js +141 -0
- package/third-party/indic-processor-deps/indicnlp/indic_normalize.js +1213 -0
- package/third-party/indic-processor-deps/indicnlp/indic_tokenize.js +123 -0
- package/third-party/indic-processor-deps/indicnlp/langinfo.js +609 -0
- package/third-party/indic-processor-deps/indicnlp/sinhala_transliterator.js +197 -0
- package/third-party/indic-processor-deps/indicnlp/unicode_transliterator.js +120 -0
- package/third-party/indic-processor-deps/sacremoses/SACREMOSES_LICENCE +21 -0
- package/third-party/indic-processor-deps/sacremoses/cjk.js +202 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/README.txt +8 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.as +65 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.bn +65 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.ca +75 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.cs +390 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.de +325 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.el +1568 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.en +123 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.es +118 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.et +138 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.fi +138 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.fr +153 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.ga +48 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.gu +105 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.hi +113 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.hu +103 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.is +251 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.it +180 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.kn +70 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.lt +698 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.lv +100 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.ml +67 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.mni +65 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.mr +113 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.nl +115 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.or +101 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.pa +102 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.pl +283 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.pt +210 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.ro +38 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.ru +293 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.sk +474 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.sl +78 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.sv +97 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.ta +71 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.tdt +210 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.te +70 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.yue +53 -0
- package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.zh +53 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/CJK.txt +23246 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/CJKSymbols.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Close_Punctuation.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Currency_Symbol.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Han.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Hangul.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Hangul_Syllables.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Hiragana.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsAlnum-unichars-au.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsAlnum.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsAlpha-unichars-au.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsAlpha.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsLower.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsN.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsPf.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsPi.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsSc.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsSo.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/IsUpper.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Katakana.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Line_Separator.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Lowercase_Letter.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Number.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Open_Punctuation.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Punctuation.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Separator.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Symbol.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Titlecase_Letter.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/data/perluniprops/Uppercase_Letter.txt +1 -0
- package/third-party/indic-processor-deps/sacremoses/index.js +8 -0
- package/third-party/indic-processor-deps/sacremoses/indic.js +76 -0
- package/third-party/indic-processor-deps/sacremoses/normalizer.js +264 -0
- package/third-party/indic-processor-deps/sacremoses/pernuliprops.js +287 -0
- package/third-party/indic-processor-deps/sacremoses/tokenizer.js +1217 -0
- package/third-party/indic-processor.js +565 -0
package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.sl
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
dr
|
|
2
|
+
Dr
|
|
3
|
+
itd
|
|
4
|
+
itn
|
|
5
|
+
št #NUMERIC_ONLY#
|
|
6
|
+
Št #NUMERIC_ONLY#
|
|
7
|
+
d
|
|
8
|
+
jan
|
|
9
|
+
Jan
|
|
10
|
+
feb
|
|
11
|
+
Feb
|
|
12
|
+
mar
|
|
13
|
+
Mar
|
|
14
|
+
apr
|
|
15
|
+
Apr
|
|
16
|
+
jun
|
|
17
|
+
Jun
|
|
18
|
+
jul
|
|
19
|
+
Jul
|
|
20
|
+
avg
|
|
21
|
+
Avg
|
|
22
|
+
sept
|
|
23
|
+
Sept
|
|
24
|
+
sep
|
|
25
|
+
Sep
|
|
26
|
+
okt
|
|
27
|
+
Okt
|
|
28
|
+
nov
|
|
29
|
+
Nov
|
|
30
|
+
dec
|
|
31
|
+
Dec
|
|
32
|
+
tj
|
|
33
|
+
Tj
|
|
34
|
+
npr
|
|
35
|
+
Npr
|
|
36
|
+
sl
|
|
37
|
+
Sl
|
|
38
|
+
op
|
|
39
|
+
Op
|
|
40
|
+
gl
|
|
41
|
+
Gl
|
|
42
|
+
oz
|
|
43
|
+
Oz
|
|
44
|
+
prev
|
|
45
|
+
dipl
|
|
46
|
+
ing
|
|
47
|
+
prim
|
|
48
|
+
Prim
|
|
49
|
+
cf
|
|
50
|
+
Cf
|
|
51
|
+
gl
|
|
52
|
+
Gl
|
|
53
|
+
A
|
|
54
|
+
B
|
|
55
|
+
C
|
|
56
|
+
D
|
|
57
|
+
E
|
|
58
|
+
F
|
|
59
|
+
G
|
|
60
|
+
H
|
|
61
|
+
I
|
|
62
|
+
J
|
|
63
|
+
K
|
|
64
|
+
L
|
|
65
|
+
M
|
|
66
|
+
N
|
|
67
|
+
O
|
|
68
|
+
P
|
|
69
|
+
Q
|
|
70
|
+
R
|
|
71
|
+
S
|
|
72
|
+
T
|
|
73
|
+
U
|
|
74
|
+
V
|
|
75
|
+
W
|
|
76
|
+
X
|
|
77
|
+
Y
|
|
78
|
+
Z
|
package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.sv
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
#single upper case letter are usually initials
|
|
2
|
+
A
|
|
3
|
+
B
|
|
4
|
+
C
|
|
5
|
+
D
|
|
6
|
+
E
|
|
7
|
+
F
|
|
8
|
+
G
|
|
9
|
+
H
|
|
10
|
+
I
|
|
11
|
+
J
|
|
12
|
+
K
|
|
13
|
+
L
|
|
14
|
+
M
|
|
15
|
+
N
|
|
16
|
+
O
|
|
17
|
+
P
|
|
18
|
+
Q
|
|
19
|
+
R
|
|
20
|
+
S
|
|
21
|
+
T
|
|
22
|
+
U
|
|
23
|
+
V
|
|
24
|
+
W
|
|
25
|
+
X
|
|
26
|
+
Y
|
|
27
|
+
Z
|
|
28
|
+
Å
|
|
29
|
+
Ä
|
|
30
|
+
Ö
|
|
31
|
+
#misc abbreviations
|
|
32
|
+
#If all words in text are in small case, then tex, mao, tom, maj, may be confused with names, and iaf, etc with named entities.
|
|
33
|
+
AB
|
|
34
|
+
VG
|
|
35
|
+
dvs
|
|
36
|
+
d.v.s
|
|
37
|
+
d. v. s
|
|
38
|
+
etc
|
|
39
|
+
from
|
|
40
|
+
fr.o.m
|
|
41
|
+
fr. o. m
|
|
42
|
+
iaf
|
|
43
|
+
i.a.f
|
|
44
|
+
i. a. f
|
|
45
|
+
jfr
|
|
46
|
+
kl
|
|
47
|
+
kr
|
|
48
|
+
mao
|
|
49
|
+
m.a.o
|
|
50
|
+
m. a. o
|
|
51
|
+
mfl
|
|
52
|
+
m.fl
|
|
53
|
+
m. fl
|
|
54
|
+
mm
|
|
55
|
+
m.m
|
|
56
|
+
m. m.
|
|
57
|
+
osv
|
|
58
|
+
o.s.v
|
|
59
|
+
o. s. v
|
|
60
|
+
pga
|
|
61
|
+
p.g.a
|
|
62
|
+
p. g. a
|
|
63
|
+
tex
|
|
64
|
+
t.ex
|
|
65
|
+
t. ex
|
|
66
|
+
#tom. is risky, as tom is a word, and can be at end of sentence. One recent text has 9 tom., and 52 tom not at end of sentence.
|
|
67
|
+
tom
|
|
68
|
+
t.o.m
|
|
69
|
+
t. o. m
|
|
70
|
+
vs
|
|
71
|
+
adv
|
|
72
|
+
jur
|
|
73
|
+
kand
|
|
74
|
+
mag
|
|
75
|
+
fil
|
|
76
|
+
lic
|
|
77
|
+
prop
|
|
78
|
+
d
|
|
79
|
+
f
|
|
80
|
+
s
|
|
81
|
+
mha
|
|
82
|
+
m.h.a
|
|
83
|
+
m. h. a
|
|
84
|
+
vol
|
|
85
|
+
#months
|
|
86
|
+
jan
|
|
87
|
+
feb
|
|
88
|
+
mar
|
|
89
|
+
apr
|
|
90
|
+
#maj is a full word
|
|
91
|
+
jun
|
|
92
|
+
jul
|
|
93
|
+
aug
|
|
94
|
+
sep
|
|
95
|
+
okt
|
|
96
|
+
nov
|
|
97
|
+
dec
|
package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.ta
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
#Anything in this file, followed by a period (and an upper-case word), does NOT indicate an end-of-sentence marker.
|
|
2
|
+
|
|
3
|
+
#common exceptions
|
|
4
|
+
# Rs
|
|
5
|
+
ர
|
|
6
|
+
# Rs
|
|
7
|
+
ூ
|
|
8
|
+
# Mr
|
|
9
|
+
திரு
|
|
10
|
+
|
|
11
|
+
#others
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
#phonetics
|
|
15
|
+
# A
|
|
16
|
+
ஏ
|
|
17
|
+
# B
|
|
18
|
+
பீ
|
|
19
|
+
# C
|
|
20
|
+
சீ
|
|
21
|
+
# D
|
|
22
|
+
டீ
|
|
23
|
+
# E
|
|
24
|
+
ஈ
|
|
25
|
+
# F
|
|
26
|
+
எஃப்
|
|
27
|
+
# G
|
|
28
|
+
ஜீ
|
|
29
|
+
# H
|
|
30
|
+
எச்
|
|
31
|
+
ஹெச்
|
|
32
|
+
# I
|
|
33
|
+
ஐ
|
|
34
|
+
# J
|
|
35
|
+
ஜே
|
|
36
|
+
ஜை
|
|
37
|
+
# K
|
|
38
|
+
கே
|
|
39
|
+
# L
|
|
40
|
+
எல்
|
|
41
|
+
# M
|
|
42
|
+
எம்
|
|
43
|
+
# N
|
|
44
|
+
என்
|
|
45
|
+
# O
|
|
46
|
+
ஓ
|
|
47
|
+
# P
|
|
48
|
+
ப்பீ
|
|
49
|
+
# Q
|
|
50
|
+
கியூ
|
|
51
|
+
# R
|
|
52
|
+
ஆர்
|
|
53
|
+
# S
|
|
54
|
+
எஸ்
|
|
55
|
+
# T
|
|
56
|
+
ட்டீ
|
|
57
|
+
# U
|
|
58
|
+
யூ
|
|
59
|
+
# V
|
|
60
|
+
வீ
|
|
61
|
+
# W
|
|
62
|
+
டபிள்-யூ
|
|
63
|
+
# X
|
|
64
|
+
எக்ஸ்
|
|
65
|
+
# Y
|
|
66
|
+
வை
|
|
67
|
+
# Z
|
|
68
|
+
செட்
|
|
69
|
+
|
|
70
|
+
#consonants
|
|
71
|
+
|
package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.tdt
ADDED
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
#File adapted for TDT from PT by Raphael Merx. Last update: 10.11.2009.
|
|
2
|
+
#Anything in this file, followed by a period (and an upper-case word), does NOT indicate an end-of-sentence marker.
|
|
3
|
+
#Special cases are included for prefixes that ONLY appear before 0-9 numbers.
|
|
4
|
+
|
|
5
|
+
#any single upper case letter followed by a period is not a sentence ender (excluding I occasionally, but we leave it in)
|
|
6
|
+
#usually upper case letters are initials in a name
|
|
7
|
+
A
|
|
8
|
+
B
|
|
9
|
+
C
|
|
10
|
+
D
|
|
11
|
+
E
|
|
12
|
+
F
|
|
13
|
+
G
|
|
14
|
+
H
|
|
15
|
+
I
|
|
16
|
+
J
|
|
17
|
+
K
|
|
18
|
+
L
|
|
19
|
+
M
|
|
20
|
+
N
|
|
21
|
+
O
|
|
22
|
+
P
|
|
23
|
+
Q
|
|
24
|
+
R
|
|
25
|
+
S
|
|
26
|
+
T
|
|
27
|
+
U
|
|
28
|
+
V
|
|
29
|
+
W
|
|
30
|
+
X
|
|
31
|
+
Y
|
|
32
|
+
Z
|
|
33
|
+
a
|
|
34
|
+
b
|
|
35
|
+
c
|
|
36
|
+
d
|
|
37
|
+
e
|
|
38
|
+
f
|
|
39
|
+
g
|
|
40
|
+
h
|
|
41
|
+
i
|
|
42
|
+
j
|
|
43
|
+
k
|
|
44
|
+
l
|
|
45
|
+
m
|
|
46
|
+
n
|
|
47
|
+
o
|
|
48
|
+
p
|
|
49
|
+
q
|
|
50
|
+
r
|
|
51
|
+
s
|
|
52
|
+
t
|
|
53
|
+
u
|
|
54
|
+
v
|
|
55
|
+
w
|
|
56
|
+
x
|
|
57
|
+
y
|
|
58
|
+
z
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
#Roman Numerals. A dot after one of these is not a sentence break in Portuguese.
|
|
62
|
+
I
|
|
63
|
+
II
|
|
64
|
+
III
|
|
65
|
+
IV
|
|
66
|
+
V
|
|
67
|
+
VI
|
|
68
|
+
VII
|
|
69
|
+
VIII
|
|
70
|
+
IX
|
|
71
|
+
X
|
|
72
|
+
XI
|
|
73
|
+
XII
|
|
74
|
+
XIII
|
|
75
|
+
XIV
|
|
76
|
+
XV
|
|
77
|
+
XVI
|
|
78
|
+
XVII
|
|
79
|
+
XVIII
|
|
80
|
+
XIX
|
|
81
|
+
XX
|
|
82
|
+
i
|
|
83
|
+
ii
|
|
84
|
+
iii
|
|
85
|
+
iv
|
|
86
|
+
v
|
|
87
|
+
vi
|
|
88
|
+
vii
|
|
89
|
+
viii
|
|
90
|
+
ix
|
|
91
|
+
x
|
|
92
|
+
xi
|
|
93
|
+
xii
|
|
94
|
+
xiii
|
|
95
|
+
xiv
|
|
96
|
+
xv
|
|
97
|
+
xvi
|
|
98
|
+
xvii
|
|
99
|
+
xviii
|
|
100
|
+
xix
|
|
101
|
+
xx
|
|
102
|
+
|
|
103
|
+
#List of titles. These are often followed by upper-case names, but do not indicate sentence breaks
|
|
104
|
+
Adj
|
|
105
|
+
Adm
|
|
106
|
+
Adv
|
|
107
|
+
Art
|
|
108
|
+
Ca
|
|
109
|
+
Capt
|
|
110
|
+
Cmdr
|
|
111
|
+
Col
|
|
112
|
+
Comdr
|
|
113
|
+
Con
|
|
114
|
+
Corp
|
|
115
|
+
Cpl
|
|
116
|
+
DR
|
|
117
|
+
DRA
|
|
118
|
+
Dr
|
|
119
|
+
Dra
|
|
120
|
+
Dras
|
|
121
|
+
Drs
|
|
122
|
+
Eng
|
|
123
|
+
Enga
|
|
124
|
+
Engas
|
|
125
|
+
Engos
|
|
126
|
+
Ex
|
|
127
|
+
Exo
|
|
128
|
+
Exmo
|
|
129
|
+
Fig
|
|
130
|
+
Gen
|
|
131
|
+
Hosp
|
|
132
|
+
Insp
|
|
133
|
+
Lda
|
|
134
|
+
MM
|
|
135
|
+
MR
|
|
136
|
+
MRS
|
|
137
|
+
MS
|
|
138
|
+
Maj
|
|
139
|
+
Mrs
|
|
140
|
+
Ms
|
|
141
|
+
Msgr
|
|
142
|
+
Op
|
|
143
|
+
Ord
|
|
144
|
+
Pfc
|
|
145
|
+
Ph
|
|
146
|
+
Prof
|
|
147
|
+
Pvt
|
|
148
|
+
Rep
|
|
149
|
+
Reps
|
|
150
|
+
Res
|
|
151
|
+
Rev
|
|
152
|
+
Rt
|
|
153
|
+
Sen
|
|
154
|
+
Sens
|
|
155
|
+
Sfc
|
|
156
|
+
Sgt
|
|
157
|
+
Sr
|
|
158
|
+
Sra
|
|
159
|
+
Sras
|
|
160
|
+
Srs
|
|
161
|
+
Sto
|
|
162
|
+
Supt
|
|
163
|
+
Surg
|
|
164
|
+
adj
|
|
165
|
+
adm
|
|
166
|
+
adv
|
|
167
|
+
art
|
|
168
|
+
cit
|
|
169
|
+
col
|
|
170
|
+
con
|
|
171
|
+
corp
|
|
172
|
+
cpl
|
|
173
|
+
dr
|
|
174
|
+
dra
|
|
175
|
+
dras
|
|
176
|
+
drs
|
|
177
|
+
eng
|
|
178
|
+
enga
|
|
179
|
+
engas
|
|
180
|
+
engos
|
|
181
|
+
ex
|
|
182
|
+
exo
|
|
183
|
+
exmo
|
|
184
|
+
fig
|
|
185
|
+
op
|
|
186
|
+
prof
|
|
187
|
+
sr
|
|
188
|
+
sra
|
|
189
|
+
sras
|
|
190
|
+
srs
|
|
191
|
+
sto
|
|
192
|
+
|
|
193
|
+
#misc - odd period-ending items that NEVER indicate breaks (p.m. does NOT fall into this category - it sometimes ends a sentence)
|
|
194
|
+
v
|
|
195
|
+
vs
|
|
196
|
+
i.e
|
|
197
|
+
rev
|
|
198
|
+
e.g
|
|
199
|
+
|
|
200
|
+
#Numbers only. These should only induce breaks when followed by a numeric sequence
|
|
201
|
+
# add NUMERIC_ONLY after the word for this function
|
|
202
|
+
#This case is mostly for the english "No." which can either be a sentence of its own, or
|
|
203
|
+
#if followed by a number, a non-breaking prefix
|
|
204
|
+
No #NUMERIC_ONLY#
|
|
205
|
+
Nos
|
|
206
|
+
Art #NUMERIC_ONLY#
|
|
207
|
+
Nr
|
|
208
|
+
p #NUMERIC_ONLY#
|
|
209
|
+
pp #NUMERIC_ONLY#
|
|
210
|
+
|
package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.te
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
#Anything in this file, followed by a period (and an upper-case word), does NOT indicate an end-of-sentence marker.
|
|
2
|
+
|
|
3
|
+
#common exceptions
|
|
4
|
+
# Rs
|
|
5
|
+
ర
|
|
6
|
+
# Rs
|
|
7
|
+
ూ
|
|
8
|
+
# Mr
|
|
9
|
+
శ్రీ
|
|
10
|
+
|
|
11
|
+
#others
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
#phonetics
|
|
15
|
+
# A
|
|
16
|
+
ఎ
|
|
17
|
+
# B
|
|
18
|
+
బి
|
|
19
|
+
# C
|
|
20
|
+
సి
|
|
21
|
+
# D
|
|
22
|
+
డి
|
|
23
|
+
# E
|
|
24
|
+
ఇ
|
|
25
|
+
# F
|
|
26
|
+
ఎఫ్
|
|
27
|
+
# G
|
|
28
|
+
జి
|
|
29
|
+
# H
|
|
30
|
+
హెచ్
|
|
31
|
+
# I
|
|
32
|
+
ఐ
|
|
33
|
+
# J
|
|
34
|
+
జె
|
|
35
|
+
# K
|
|
36
|
+
కె
|
|
37
|
+
# L
|
|
38
|
+
ఎల్
|
|
39
|
+
# M
|
|
40
|
+
ఎం
|
|
41
|
+
ఎమ్
|
|
42
|
+
# N
|
|
43
|
+
ఎన్
|
|
44
|
+
# O
|
|
45
|
+
ఓ
|
|
46
|
+
# P
|
|
47
|
+
పి
|
|
48
|
+
# Q
|
|
49
|
+
క్యూ
|
|
50
|
+
# R
|
|
51
|
+
ఆర్
|
|
52
|
+
# S
|
|
53
|
+
ఎస్
|
|
54
|
+
# T
|
|
55
|
+
టి
|
|
56
|
+
# U
|
|
57
|
+
యు
|
|
58
|
+
# V
|
|
59
|
+
వి
|
|
60
|
+
# W
|
|
61
|
+
డబ్ల్యూ
|
|
62
|
+
# X
|
|
63
|
+
ఎక్స్
|
|
64
|
+
# Y
|
|
65
|
+
వై
|
|
66
|
+
# Z
|
|
67
|
+
జెడ్
|
|
68
|
+
|
|
69
|
+
#consonants
|
|
70
|
+
|
package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.yue
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Cantonese (Chinese)
|
|
3
|
+
#
|
|
4
|
+
# Anything in this file, followed by a period,
|
|
5
|
+
# does NOT indicate an end-of-sentence marker.
|
|
6
|
+
#
|
|
7
|
+
# English/Euro-language given-name initials (appearing in
|
|
8
|
+
# news, periodicals, etc.)
|
|
9
|
+
A
|
|
10
|
+
Ā
|
|
11
|
+
B
|
|
12
|
+
C
|
|
13
|
+
Č
|
|
14
|
+
D
|
|
15
|
+
E
|
|
16
|
+
Ē
|
|
17
|
+
F
|
|
18
|
+
G
|
|
19
|
+
Ģ
|
|
20
|
+
H
|
|
21
|
+
I
|
|
22
|
+
Ī
|
|
23
|
+
J
|
|
24
|
+
K
|
|
25
|
+
Ķ
|
|
26
|
+
L
|
|
27
|
+
Ļ
|
|
28
|
+
M
|
|
29
|
+
N
|
|
30
|
+
Ņ
|
|
31
|
+
O
|
|
32
|
+
P
|
|
33
|
+
Q
|
|
34
|
+
R
|
|
35
|
+
S
|
|
36
|
+
Š
|
|
37
|
+
T
|
|
38
|
+
U
|
|
39
|
+
Ū
|
|
40
|
+
V
|
|
41
|
+
W
|
|
42
|
+
X
|
|
43
|
+
Y
|
|
44
|
+
Z
|
|
45
|
+
Ž
|
|
46
|
+
|
|
47
|
+
# Numbers only. These should only induce breaks when followed by
|
|
48
|
+
# a numeric sequence.
|
|
49
|
+
# Add NUMERIC_ONLY after the word for this function. This case is
|
|
50
|
+
# mostly for the english "No." which can either be a sentence of its
|
|
51
|
+
# own, or if followed by a number, a non-breaking prefix.
|
|
52
|
+
No #NUMERIC_ONLY#
|
|
53
|
+
Nr #NUMERIC_ONLY#
|
package/third-party/indic-processor-deps/sacremoses/data/nonbreaking_prefixes/nonbreaking_prefix.zh
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Mandarin (Chinese)
|
|
3
|
+
#
|
|
4
|
+
# Anything in this file, followed by a period,
|
|
5
|
+
# does NOT indicate an end-of-sentence marker.
|
|
6
|
+
#
|
|
7
|
+
# English/Euro-language given-name initials (appearing in
|
|
8
|
+
# news, periodicals, etc.)
|
|
9
|
+
A
|
|
10
|
+
Ā
|
|
11
|
+
B
|
|
12
|
+
C
|
|
13
|
+
Č
|
|
14
|
+
D
|
|
15
|
+
E
|
|
16
|
+
Ē
|
|
17
|
+
F
|
|
18
|
+
G
|
|
19
|
+
Ģ
|
|
20
|
+
H
|
|
21
|
+
I
|
|
22
|
+
Ī
|
|
23
|
+
J
|
|
24
|
+
K
|
|
25
|
+
Ķ
|
|
26
|
+
L
|
|
27
|
+
Ļ
|
|
28
|
+
M
|
|
29
|
+
N
|
|
30
|
+
Ņ
|
|
31
|
+
O
|
|
32
|
+
P
|
|
33
|
+
Q
|
|
34
|
+
R
|
|
35
|
+
S
|
|
36
|
+
Š
|
|
37
|
+
T
|
|
38
|
+
U
|
|
39
|
+
Ū
|
|
40
|
+
V
|
|
41
|
+
W
|
|
42
|
+
X
|
|
43
|
+
Y
|
|
44
|
+
Z
|
|
45
|
+
Ž
|
|
46
|
+
|
|
47
|
+
# Numbers only. These should only induce breaks when followed by
|
|
48
|
+
# a numeric sequence.
|
|
49
|
+
# Add NUMERIC_ONLY after the word for this function. This case is
|
|
50
|
+
# mostly for the english "No." which can either be a sentence of its
|
|
51
|
+
# own, or if followed by a number, a non-breaking prefix.
|
|
52
|
+
No #NUMERIC_ONLY#
|
|
53
|
+
Nr #NUMERIC_ONLY#
|