language_detection 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +19 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +85 -0
- data/Rakefile +11 -0
- data/ext/cld/Makefile +34 -0
- data/ext/cld/base/basictypes.h +348 -0
- data/ext/cld/base/build_config.h +124 -0
- data/ext/cld/base/casts.h +156 -0
- data/ext/cld/base/commandlineflags.h +443 -0
- data/ext/cld/base/crash.h +41 -0
- data/ext/cld/base/dynamic_annotations.h +358 -0
- data/ext/cld/base/global_strip_options.h +59 -0
- data/ext/cld/base/log_severity.h +46 -0
- data/ext/cld/base/logging.h +1403 -0
- data/ext/cld/base/macros.h +243 -0
- data/ext/cld/base/port.h +54 -0
- data/ext/cld/base/scoped_ptr.h +428 -0
- data/ext/cld/base/stl_decl.h +0 -0
- data/ext/cld/base/stl_decl_msvc.h +107 -0
- data/ext/cld/base/string_util.h +29 -0
- data/ext/cld/base/strtoint.h +93 -0
- data/ext/cld/base/template_util.h +96 -0
- data/ext/cld/base/type_traits.h +198 -0
- data/ext/cld/base/vlog_is_on.h +143 -0
- data/ext/cld/cld.so +0 -0
- data/ext/cld/encodings/compact_lang_det/cldutil.cc +905 -0
- data/ext/cld/encodings/compact_lang_det/cldutil.h +1205 -0
- data/ext/cld/encodings/compact_lang_det/cldutil_dbg.h +76 -0
- data/ext/cld/encodings/compact_lang_det/cldutil_dbg_empty.cc +76 -0
- data/ext/cld/encodings/compact_lang_det/compact_lang_det.cc +62 -0
- data/ext/cld/encodings/compact_lang_det/compact_lang_det.h +145 -0
- data/ext/cld/encodings/compact_lang_det/compact_lang_det_impl.cc +2574 -0
- data/ext/cld/encodings/compact_lang_det/compact_lang_det_impl.h +173 -0
- data/ext/cld/encodings/compact_lang_det/compact_lang_det_unittest_small.cc +406 -0
- data/ext/cld/encodings/compact_lang_det/compile.cmd +1 -0
- data/ext/cld/encodings/compact_lang_det/ext_lang_enc.cc +545 -0
- data/ext/cld/encodings/compact_lang_det/ext_lang_enc.h +119 -0
- data/ext/cld/encodings/compact_lang_det/generated/cld_generated_score_deltaoctachrome_0406.cc +380 -0
- data/ext/cld/encodings/compact_lang_det/generated/cld_generated_score_quadchrome_0406.cc +382 -0
- data/ext/cld/encodings/compact_lang_det/generated/compact_lang_det_generated_cjkbis_0.cc +49 -0
- data/ext/cld/encodings/compact_lang_det/generated/compact_lang_det_generated_ctjkvz.cc +7119 -0
- data/ext/cld/encodings/compact_lang_det/generated/compact_lang_det_generated_ctjkvz_0.cc +61 -0
- data/ext/cld/encodings/compact_lang_det/generated/compact_lang_det_generated_deltaoctachrome.cc +1263 -0
- data/ext/cld/encodings/compact_lang_det/generated/compact_lang_det_generated_longwords8_0.cc +53 -0
- data/ext/cld/encodings/compact_lang_det/generated/compact_lang_det_generated_meanscore.h +10 -0
- data/ext/cld/encodings/compact_lang_det/generated/compact_lang_det_generated_quads_0.cc +50 -0
- data/ext/cld/encodings/compact_lang_det/generated/compact_lang_det_generated_quadschrome.cc +70935 -0
- data/ext/cld/encodings/compact_lang_det/getonescriptspan.cc +570 -0
- data/ext/cld/encodings/compact_lang_det/getonescriptspan.h +131 -0
- data/ext/cld/encodings/compact_lang_det/letterscript_enum.cc +117 -0
- data/ext/cld/encodings/compact_lang_det/letterscript_enum.h +99 -0
- data/ext/cld/encodings/compact_lang_det/subsetsequence.cc +259 -0
- data/ext/cld/encodings/compact_lang_det/subsetsequence.h +44 -0
- data/ext/cld/encodings/compact_lang_det/subsetsequence_unittest.cc +99 -0
- data/ext/cld/encodings/compact_lang_det/tote.cc +299 -0
- data/ext/cld/encodings/compact_lang_det/tote.h +89 -0
- data/ext/cld/encodings/compact_lang_det/unittest_data.h +193 -0
- data/ext/cld/encodings/compact_lang_det/utf8propjustletter.h +1162 -0
- data/ext/cld/encodings/compact_lang_det/utf8propletterscriptnum.h +1222 -0
- data/ext/cld/encodings/compact_lang_det/utf8scannotjustletterspecial.h +1185 -0
- data/ext/cld/encodings/compact_lang_det/win/cld_basictypes.h +10 -0
- data/ext/cld/encodings/compact_lang_det/win/cld_commandlineflags.h +28 -0
- data/ext/cld/encodings/compact_lang_det/win/cld_google.h +18 -0
- data/ext/cld/encodings/compact_lang_det/win/cld_htmlutils.h +13 -0
- data/ext/cld/encodings/compact_lang_det/win/cld_htmlutils_google3.cc +32 -0
- data/ext/cld/encodings/compact_lang_det/win/cld_htmlutils_windows.cc +29 -0
- data/ext/cld/encodings/compact_lang_det/win/cld_logging.h +21 -0
- data/ext/cld/encodings/compact_lang_det/win/cld_macros.h +19 -0
- data/ext/cld/encodings/compact_lang_det/win/cld_strtoint.h +26 -0
- data/ext/cld/encodings/compact_lang_det/win/cld_unicodetext.cc +84 -0
- data/ext/cld/encodings/compact_lang_det/win/cld_unicodetext.h +40 -0
- data/ext/cld/encodings/compact_lang_det/win/cld_unilib.h +15 -0
- data/ext/cld/encodings/compact_lang_det/win/cld_unilib_google3.cc +18 -0
- data/ext/cld/encodings/compact_lang_det/win/cld_unilib_windows.cc +29 -0
- data/ext/cld/encodings/compact_lang_det/win/cld_utf.h +24 -0
- data/ext/cld/encodings/compact_lang_det/win/cld_utf8statetable.cc +224 -0
- data/ext/cld/encodings/compact_lang_det/win/cld_utf8statetable.h +141 -0
- data/ext/cld/encodings/compact_lang_det/win/cld_utf8utils.h +22 -0
- data/ext/cld/encodings/compact_lang_det/win/cld_utf8utils_google3.cc +18 -0
- data/ext/cld/encodings/compact_lang_det/win/cld_utf8utils_windows.cc +17 -0
- data/ext/cld/encodings/compact_lang_det/win/normalizedunicodetext.cc +172 -0
- data/ext/cld/encodings/compact_lang_det/win/normalizedunicodetext.h +67 -0
- data/ext/cld/encodings/internal/encodings.cc +12 -0
- data/ext/cld/encodings/lang_enc.h +254 -0
- data/ext/cld/encodings/proto/encodings.pb.h +169 -0
- data/ext/cld/encodings/public/encodings.h +301 -0
- data/ext/cld/extconf.rb +1 -0
- data/ext/cld/language_detection.cc +88 -0
- data/ext/cld/languages/internal/languages.cc +337 -0
- data/ext/cld/languages/proto/languages.pb.h +179 -0
- data/ext/cld/languages/public/languages.h +379 -0
- data/language_detection.gemspec +28 -0
- data/lib/language_detection/string.rb +1 -0
- data/lib/language_detection/version.rb +3 -0
- data/lib/language_detection.rb +54 -0
- data/test/_helper.rb +15 -0
- data/test/fixtures/languages.csv +80 -0
- data/test/language_detection_test.rb +88 -0
- metadata +250 -0
@@ -0,0 +1,382 @@
|
|
1
|
+
// Copyright (c) 2011 The Chromium Authors. All rights reserved.
|
2
|
+
// Use of this source code is governed by a BSD-style license that can be
|
3
|
+
// found in the LICENSE file.
|
4
|
+
|
5
|
+
#include "base/basictypes.h"
|
6
|
+
#include "encodings/compact_lang_det/ext_lang_enc.h"
|
7
|
+
|
8
|
+
// score_me text [ 8] ja not found in tables
|
9
|
+
// score_me text [ 9] ko not found in tables
|
10
|
+
// score_me text [ 16] zh not found in tables
|
11
|
+
// score_me text [ 18] el not found in tables
|
12
|
+
// score_me text [ 26] un not found in tables
|
13
|
+
// score_me text [ 39] la not found in tables
|
14
|
+
// score_me text [ 41] ml not found in tables
|
15
|
+
// score_me text [ 43] ne not found in tables
|
16
|
+
// score_me text [ 44] te not found in tables
|
17
|
+
// score_me text [ 46] ta not found in tables
|
18
|
+
// score_me text [ 48] jw not found in tables
|
19
|
+
// score_me text [ 49] oc not found in tables
|
20
|
+
// score_me text [ 51] bh not found in tables
|
21
|
+
// score_me text [ 52] gu not found in tables
|
22
|
+
// score_me text [ 53] th not found in tables
|
23
|
+
// score_me text [ 56] eo not found in tables
|
24
|
+
// score_me text [ 58] ia not found in tables
|
25
|
+
// score_me text [ 59] kn not found in tables
|
26
|
+
// score_me text [ 60] pa not found in tables
|
27
|
+
// score_me text [ 61] gd not found in tables
|
28
|
+
// score_me text [ 64] mr not found in tables
|
29
|
+
// score_me text [ 67] fy not found in tables
|
30
|
+
// score_me text [ 69] zhT not found in tables
|
31
|
+
// score_me text [ 70] fo not found in tables
|
32
|
+
// score_me text [ 71] su not found in tables
|
33
|
+
// score_me text [ 72] uz not found in tables
|
34
|
+
// score_me text [ 73] am not found in tables
|
35
|
+
// score_me text [ 75] ka not found in tables
|
36
|
+
// score_me text [ 76] ti not found in tables
|
37
|
+
// score_me text [ 78] bs not found in tables
|
38
|
+
// score_me text [ 79] si not found in tables
|
39
|
+
// score_me text [ 80] nn not found in tables
|
40
|
+
// score_me text [ 83] xh not found in tables
|
41
|
+
// score_me text [ 84] zu not found in tables
|
42
|
+
// score_me text [ 85] gn not found in tables
|
43
|
+
// score_me text [ 86] st not found in tables
|
44
|
+
// score_me text [ 87] tk not found in tables
|
45
|
+
// score_me text [ 88] ky not found in tables
|
46
|
+
// score_me text [ 89] br not found in tables
|
47
|
+
// score_me text [ 90] tw not found in tables
|
48
|
+
// score_me text [ 93] so not found in tables
|
49
|
+
// score_me text [ 94] ug not found in tables
|
50
|
+
// score_me text [ 95] ku not found in tables
|
51
|
+
// score_me text [ 96] mn not found in tables
|
52
|
+
// score_me text [ 97] hy not found in tables
|
53
|
+
// score_me text [ 98] lo not found in tables
|
54
|
+
// score_me text [ 99] sd not found in tables
|
55
|
+
// score_me text [100] rm not found in tables
|
56
|
+
// score_me text [102] lb not found in tables
|
57
|
+
// score_me text [103] my not found in tables
|
58
|
+
// score_me text [104] km not found in tables
|
59
|
+
// score_me text [105] bo not found in tables
|
60
|
+
// score_me text [107] chr not found in tables
|
61
|
+
// score_me text [109] sit-NP not found in tables
|
62
|
+
// score_me text [110] or not found in tables
|
63
|
+
// score_me text [111] as not found in tables
|
64
|
+
// score_me text [112] co not found in tables
|
65
|
+
// score_me text [113] ie not found in tables
|
66
|
+
// score_me text [114] kk not found in tables
|
67
|
+
// score_me text [115] ln not found in tables
|
68
|
+
// score_me text [116] mo not found in tables
|
69
|
+
// score_me text [117] ps not found in tables
|
70
|
+
// score_me text [118] qu not found in tables
|
71
|
+
// score_me text [119] sn not found in tables
|
72
|
+
// score_me text [120] tg not found in tables
|
73
|
+
// score_me text [121] tt not found in tables
|
74
|
+
// score_me text [122] to not found in tables
|
75
|
+
// score_me text [123] yo not found in tables
|
76
|
+
// score_me text [128] mi not found in tables
|
77
|
+
// score_me text [129] wo not found in tables
|
78
|
+
// score_me text [130] ab not found in tables
|
79
|
+
// score_me text [131] aa not found in tables
|
80
|
+
// score_me text [132] ay not found in tables
|
81
|
+
// score_me text [133] ba not found in tables
|
82
|
+
// score_me text [134] bi not found in tables
|
83
|
+
// score_me text [135] dz not found in tables
|
84
|
+
// score_me text [136] fj not found in tables
|
85
|
+
// score_me text [137] kl not found in tables
|
86
|
+
// score_me text [138] ha not found in tables
|
87
|
+
// score_me text [140] ik not found in tables
|
88
|
+
// score_me text [141] iu not found in tables
|
89
|
+
// score_me text [142] ks not found in tables
|
90
|
+
// score_me text [143] rw not found in tables
|
91
|
+
// score_me text [144] mg not found in tables
|
92
|
+
// score_me text [145] na not found in tables
|
93
|
+
// score_me text [146] om not found in tables
|
94
|
+
// score_me text [147] rn not found in tables
|
95
|
+
// score_me text [148] sm not found in tables
|
96
|
+
// score_me text [149] sg not found in tables
|
97
|
+
// score_me text [150] sa not found in tables
|
98
|
+
// score_me text [151] ss not found in tables
|
99
|
+
// score_me text [152] ts not found in tables
|
100
|
+
// score_me text [153] tn not found in tables
|
101
|
+
// score_me text [154] vo not found in tables
|
102
|
+
// score_me text [155] za not found in tables
|
103
|
+
// score_me text [156] kha not found in tables
|
104
|
+
// score_me text [157] sco not found in tables
|
105
|
+
// score_me text [158] lg not found in tables
|
106
|
+
// score_me text [159] gv not found in tables
|
107
|
+
// score_me text [160] srM not found in tables
|
108
|
+
// score_me text [161] ak not found in tables
|
109
|
+
// score_me text [162] ig not found in tables
|
110
|
+
// score_me text [163] mfe not found in tables
|
111
|
+
// score_me text [164] haw not found in tables
|
112
|
+
// score_me text [165] nso not found in tables
|
113
|
+
// score_me text [166] nr not found in tables
|
114
|
+
// score_me text [167] ve not found in tables
|
115
|
+
// score_me text [168] ny not found in tables
|
116
|
+
// score_me text [169] crs not found in tables
|
117
|
+
// score_me text [185] zzb not found in tables
|
118
|
+
// score_me text [186] zzp not found in tables
|
119
|
+
// score_me text [187] zzh not found in tables
|
120
|
+
// score_me text [188] tlh not found in tables
|
121
|
+
// score_me text [189] zze not found in tables
|
122
|
+
|
123
|
+
// No score_me text for [ 25] xxx
|
124
|
+
|
125
|
+
// Average score per 1024 bytes
|
126
|
+
// Renamed kAvgQuadScore -> kMeanScore to match older CLD code.
|
127
|
+
extern const short kMeanScore[244 * 4] = {
|
128
|
+
// Latn Cyrl Arab Other script
|
129
|
+
1234, 0, 0, 0, // [ 0] ENGLISH en
|
130
|
+
721, 0, 0, 0, // [ 1] DANISH da
|
131
|
+
1037, 0, 0, 0, // [ 2] DUTCH nl
|
132
|
+
1337, 0, 0, 0, // [ 3] FINNISH fi
|
133
|
+
973, 0, 0, 0, // [ 4] FRENCH fr
|
134
|
+
1211, 0, 0, 0, // [ 5] GERMAN de
|
135
|
+
0, 0, 0, 727, // [ 6] HEBREW iw
|
136
|
+
866, 0, 0, 0, // [ 7] ITALIAN it
|
137
|
+
0, 0, 0, 0, // [ 8] Japanese ja ==Zero==
|
138
|
+
0, 0, 0, 0, // [ 9] Korean ko ==Zero==
|
139
|
+
833, 0, 0, 0, // [ 10] NORWEGIAN no
|
140
|
+
1280, 0, 0, 0, // [ 11] POLISH pl
|
141
|
+
718, 0, 0, 0, // [ 12] PORTUGUESE pt
|
142
|
+
0, 767, 0, 0, // [ 13] RUSSIAN ru
|
143
|
+
642, 0, 0, 0, // [ 14] SPANISH es
|
144
|
+
826, 0, 0, 0, // [ 15] SWEDISH sv
|
145
|
+
0, 0, 0, 0, // [ 16] Chinese zh ==Zero==
|
146
|
+
982, 0, 0, 0, // [ 17] CZECH cs
|
147
|
+
0, 0, 0, 0, // [ 18] GREEK el ==Zero==
|
148
|
+
1214, 0, 0, 0, // [ 19] ICELANDIC is
|
149
|
+
1194, 0, 0, 0, // [ 20] LATVIAN lv
|
150
|
+
1220, 0, 0, 0, // [ 21] LITHUANIAN lt
|
151
|
+
951, 0, 0, 0, // [ 22] ROMANIAN ro
|
152
|
+
1335, 0, 0, 0, // [ 23] HUNGARIAN hu
|
153
|
+
1145, 0, 0, 0, // [ 24] ESTONIAN et
|
154
|
+
0, 0, 0, 0, // [ 25] Ignore xxx ==Zero==
|
155
|
+
0, 0, 0, 0, // [ 26] Unknown un ==Zero==
|
156
|
+
0, 691, 0, 0, // [ 27] BULGARIAN bg
|
157
|
+
774, 0, 0, 0, // [ 28] CROATIAN hr
|
158
|
+
1048, 686, 0, 0, // [ 29] SERBIAN sr
|
159
|
+
1503, 0, 0, 0, // [ 30] IRISH ga
|
160
|
+
464, 0, 0, 0, // [ 31] GALICIAN gl
|
161
|
+
|
162
|
+
1570, 0, 0, 0, // [ 32] TAGALOG tl
|
163
|
+
1390, 0, 0, 0, // [ 33] TURKISH tr
|
164
|
+
0, 677, 0, 0, // [ 34] UKRAINIAN uk
|
165
|
+
0, 0, 0, 601, // [ 35] HINDI hi
|
166
|
+
0, 695, 0, 0, // [ 36] MACEDONIAN mk
|
167
|
+
0, 0, 0, 563, // [ 37] BENGALI bn
|
168
|
+
1307, 0, 0, 0, // [ 38] INDONESIAN id
|
169
|
+
0, 0, 0, 0, // [ 39] LATIN la ==Zero==
|
170
|
+
1402, 0, 0, 0, // [ 40] MALAY ms
|
171
|
+
0, 0, 0, 0, // [ 41] MALAYALAM ml ==Zero==
|
172
|
+
1536, 0, 0, 0, // [ 42] WELSH cy
|
173
|
+
0, 0, 0, 0, // [ 43] NEPALI ne ==Zero==
|
174
|
+
0, 0, 0, 0, // [ 44] TELUGU te ==Zero==
|
175
|
+
1146, 0, 0, 0, // [ 45] ALBANIAN sq
|
176
|
+
0, 0, 0, 0, // [ 46] TAMIL ta ==Zero==
|
177
|
+
0, 834, 0, 0, // [ 47] BELARUSIAN be
|
178
|
+
0, 0, 0, 0, // [ 48] JAVANESE jw ==Zero==
|
179
|
+
0, 0, 0, 0, // [ 49] OCCITAN oc ==Zero==
|
180
|
+
0, 0, 964, 0, // [ 50] URDU ur
|
181
|
+
0, 0, 0, 0, // [ 51] BIHARI bh ==Zero==
|
182
|
+
0, 0, 0, 0, // [ 52] GUJARATI gu ==Zero==
|
183
|
+
0, 0, 0, 0, // [ 53] THAI th ==Zero==
|
184
|
+
0, 0, 735, 0, // [ 54] ARABIC ar
|
185
|
+
667, 0, 0, 0, // [ 55] CATALAN ca
|
186
|
+
0, 0, 0, 0, // [ 56] ESPERANTO eo ==Zero==
|
187
|
+
1442, 0, 0, 0, // [ 57] BASQUE eu
|
188
|
+
0, 0, 0, 0, // [ 58] INTERLINGUA ia ==Zero==
|
189
|
+
0, 0, 0, 0, // [ 59] KANNADA kn ==Zero==
|
190
|
+
0, 0, 0, 0, // [ 60] PUNJABI pa ==Zero==
|
191
|
+
0, 0, 0, 0, // [ 61] SCOTS_GAELIC gd ==Zero==
|
192
|
+
1448, 0, 0, 0, // [ 62] SWAHILI sw
|
193
|
+
907, 0, 0, 0, // [ 63] SLOVENIAN sl
|
194
|
+
|
195
|
+
0, 0, 0, 0, // [ 64] MARATHI mr ==Zero==
|
196
|
+
1027, 0, 0, 0, // [ 65] MALTESE mt
|
197
|
+
1097, 0, 0, 0, // [ 66] VIETNAMESE vi
|
198
|
+
0, 0, 0, 0, // [ 67] FRISIAN fy ==Zero==
|
199
|
+
1007, 0, 0, 0, // [ 68] SLOVAK sk
|
200
|
+
0, 0, 0, 0, // [ 69] ChineseT zhT ==Zero==
|
201
|
+
0, 0, 0, 0, // [ 70] FAROESE fo ==Zero==
|
202
|
+
0, 0, 0, 0, // [ 71] SUNDANESE su ==Zero==
|
203
|
+
0, 0, 0, 0, // [ 72] UZBEK uz ==Zero==
|
204
|
+
0, 0, 0, 0, // [ 73] AMHARIC am ==Zero==
|
205
|
+
1371, 0, 0, 0, // [ 74] AZERBAIJANI az
|
206
|
+
0, 0, 0, 0, // [ 75] GEORGIAN ka ==Zero==
|
207
|
+
0, 0, 0, 0, // [ 76] TIGRINYA ti ==Zero==
|
208
|
+
0, 0, 855, 0, // [ 77] PERSIAN fa
|
209
|
+
0, 0, 0, 0, // [ 78] BOSNIAN bs ==Zero==
|
210
|
+
0, 0, 0, 0, // [ 79] SINHALESE si ==Zero==
|
211
|
+
0, 0, 0, 0, // [ 80] NORWEGIAN_N nn ==Zero==
|
212
|
+
0, 0, 0, 0, // [ 81] PORTUGUESE_P pt-PT ==Zero==
|
213
|
+
0, 0, 0, 0, // [ 82] PORTUGUESE_B pt-BR ==Zero==
|
214
|
+
0, 0, 0, 0, // [ 83] XHOSA xh ==Zero==
|
215
|
+
0, 0, 0, 0, // [ 84] ZULU zu ==Zero==
|
216
|
+
0, 0, 0, 0, // [ 85] GUARANI gn ==Zero==
|
217
|
+
0, 0, 0, 0, // [ 86] SESOTHO st ==Zero==
|
218
|
+
0, 0, 0, 0, // [ 87] TURKMEN tk ==Zero==
|
219
|
+
0, 0, 0, 0, // [ 88] KYRGYZ ky ==Zero==
|
220
|
+
0, 0, 0, 0, // [ 89] BRETON br ==Zero==
|
221
|
+
0, 0, 0, 0, // [ 90] TWI tw ==Zero==
|
222
|
+
0, 0, 0, 807, // [ 91] YIDDISH yi
|
223
|
+
0, 0, 0, 0, // [ 92] SERBO_CROATIAN sh ==Zero==
|
224
|
+
0, 0, 0, 0, // [ 93] SOMALI so ==Zero==
|
225
|
+
0, 0, 0, 0, // [ 94] UIGHUR ug ==Zero==
|
226
|
+
0, 0, 0, 0, // [ 95] KURDISH ku ==Zero==
|
227
|
+
|
228
|
+
0, 0, 0, 0, // [ 96] MONGOLIAN mn ==Zero==
|
229
|
+
0, 0, 0, 0, // [ 97] ARMENIAN hy ==Zero==
|
230
|
+
0, 0, 0, 0, // [ 98] LAOTHIAN lo ==Zero==
|
231
|
+
0, 0, 0, 0, // [ 99] SINDHI sd ==Zero==
|
232
|
+
0, 0, 0, 0, // [100] RHAETO_ROMANCE rm ==Zero==
|
233
|
+
885, 0, 0, 0, // [101] AFRIKAANS af
|
234
|
+
0, 0, 0, 0, // [102] LUXEMBOURGISH lb ==Zero==
|
235
|
+
0, 0, 0, 0, // [103] BURMESE my ==Zero==
|
236
|
+
0, 0, 0, 0, // [104] KHMER km ==Zero==
|
237
|
+
0, 0, 0, 0, // [105] TIBETAN bo ==Zero==
|
238
|
+
0, 0, 0, 0, // [106] DHIVEHI dv ==Zero==
|
239
|
+
0, 0, 0, 0, // [107] CHEROKEE chr ==Zero==
|
240
|
+
0, 0, 0, 0, // [108] SYRIAC syr ==Zero==
|
241
|
+
0, 0, 0, 0, // [109] LIMBU sit-NP ==Zero==
|
242
|
+
0, 0, 0, 0, // [110] ORIYA or ==Zero==
|
243
|
+
0, 0, 0, 0, // [111] ASSAMESE as ==Zero==
|
244
|
+
0, 0, 0, 0, // [112] CORSICAN co ==Zero==
|
245
|
+
0, 0, 0, 0, // [113] INTERLINGUE ie ==Zero==
|
246
|
+
0, 0, 0, 0, // [114] KAZAKH kk ==Zero==
|
247
|
+
0, 0, 0, 0, // [115] LINGALA ln ==Zero==
|
248
|
+
0, 0, 0, 0, // [116] MOLDAVIAN mo ==Zero==
|
249
|
+
0, 0, 0, 0, // [117] PASHTO ps ==Zero==
|
250
|
+
0, 0, 0, 0, // [118] QUECHUA qu ==Zero==
|
251
|
+
0, 0, 0, 0, // [119] SHONA sn ==Zero==
|
252
|
+
0, 0, 0, 0, // [120] TAJIK tg ==Zero==
|
253
|
+
0, 0, 0, 0, // [121] TATAR tt ==Zero==
|
254
|
+
0, 0, 0, 0, // [122] TONGA to ==Zero==
|
255
|
+
0, 0, 0, 0, // [123] YORUBA yo ==Zero==
|
256
|
+
0, 0, 0, 0, // [124] CREOLES_AND_PIDGINS_ENGLISH_BASED cpe ==Zero==
|
257
|
+
0, 0, 0, 0, // [125] CREOLES_AND_PIDGINS_FRENCH_BASED cpf ==Zero==
|
258
|
+
0, 0, 0, 0, // [126] CREOLES_AND_PIDGINS_PORTUGUESE_BASED cpp ==Zero==
|
259
|
+
0, 0, 0, 0, // [127] CREOLES_AND_PIDGINS_OTHER crp ==Zero==
|
260
|
+
|
261
|
+
0, 0, 0, 0, // [128] MAORI mi ==Zero==
|
262
|
+
0, 0, 0, 0, // [129] WOLOF wo ==Zero==
|
263
|
+
0, 0, 0, 0, // [130] ABKHAZIAN ab ==Zero==
|
264
|
+
0, 0, 0, 0, // [131] AFAR aa ==Zero==
|
265
|
+
0, 0, 0, 0, // [132] AYMARA ay ==Zero==
|
266
|
+
0, 0, 0, 0, // [133] BASHKIR ba ==Zero==
|
267
|
+
0, 0, 0, 0, // [134] BISLAMA bi ==Zero==
|
268
|
+
0, 0, 0, 0, // [135] DZONGKHA dz ==Zero==
|
269
|
+
0, 0, 0, 0, // [136] FIJIAN fj ==Zero==
|
270
|
+
0, 0, 0, 0, // [137] GREENLANDIC kl ==Zero==
|
271
|
+
0, 0, 0, 0, // [138] HAUSA ha ==Zero==
|
272
|
+
1151, 0, 0, 0, // [139] HAITIAN_CREOLE ht
|
273
|
+
0, 0, 0, 0, // [140] INUPIAK ik ==Zero==
|
274
|
+
0, 0, 0, 0, // [141] INUKTITUT iu ==Zero==
|
275
|
+
0, 0, 0, 0, // [142] KASHMIRI ks ==Zero==
|
276
|
+
0, 0, 0, 0, // [143] KINYARWANDA rw ==Zero==
|
277
|
+
0, 0, 0, 0, // [144] MALAGASY mg ==Zero==
|
278
|
+
0, 0, 0, 0, // [145] NAURU na ==Zero==
|
279
|
+
0, 0, 0, 0, // [146] OROMO om ==Zero==
|
280
|
+
0, 0, 0, 0, // [147] RUNDI rn ==Zero==
|
281
|
+
0, 0, 0, 0, // [148] SAMOAN sm ==Zero==
|
282
|
+
0, 0, 0, 0, // [149] SANGO sg ==Zero==
|
283
|
+
0, 0, 0, 0, // [150] SANSKRIT sa ==Zero==
|
284
|
+
0, 0, 0, 0, // [151] SISWANT ss ==Zero==
|
285
|
+
0, 0, 0, 0, // [152] TSONGA ts ==Zero==
|
286
|
+
0, 0, 0, 0, // [153] TSWANA tn ==Zero==
|
287
|
+
0, 0, 0, 0, // [154] VOLAPUK vo ==Zero==
|
288
|
+
0, 0, 0, 0, // [155] ZHUANG za ==Zero==
|
289
|
+
0, 0, 0, 0, // [156] KHASI kha ==Zero==
|
290
|
+
0, 0, 0, 0, // [157] SCOTS sco ==Zero==
|
291
|
+
0, 0, 0, 0, // [158] GANDA lg ==Zero==
|
292
|
+
0, 0, 0, 0, // [159] MANX gv ==Zero==
|
293
|
+
|
294
|
+
0, 0, 0, 0, // [160] MONTENEGRIN srM ==Zero==
|
295
|
+
0, 0, 0, 0, // [161] AKAN ak ==Zero==
|
296
|
+
0, 0, 0, 0, // [162] IGBO ig ==Zero==
|
297
|
+
0, 0, 0, 0, // [163] MAURITIAN_CREOLE mfe ==Zero==
|
298
|
+
0, 0, 0, 0, // [164] HAWAIIAN haw ==Zero==
|
299
|
+
0, 0, 0, 0, // [165] PEDI nso ==Zero==
|
300
|
+
0, 0, 0, 0, // [166] NDEBELE nr ==Zero==
|
301
|
+
0, 0, 0, 0, // [167] VENDA ve ==Zero==
|
302
|
+
0, 0, 0, 0, // [168] CHICHEWA ny ==Zero==
|
303
|
+
0, 0, 0, 0, // [169] SESELWA crs ==Zero==
|
304
|
+
0, 0, 0, 0, // [170] invalid_language ?? ==Zero==
|
305
|
+
0, 0, 0, 0, // [171] invalid_language ?? ==Zero==
|
306
|
+
0, 0, 0, 0, // [172] invalid_language ?? ==Zero==
|
307
|
+
0, 0, 0, 0, // [173] invalid_language ?? ==Zero==
|
308
|
+
0, 0, 0, 0, // [174] invalid_language ?? ==Zero==
|
309
|
+
0, 0, 0, 0, // [175] invalid_language ?? ==Zero==
|
310
|
+
0, 0, 0, 0, // [176] invalid_language ?? ==Zero==
|
311
|
+
0, 0, 0, 0, // [177] invalid_language ?? ==Zero==
|
312
|
+
0, 0, 0, 0, // [178] invalid_language ?? ==Zero==
|
313
|
+
0, 0, 0, 0, // [179] invalid_language ?? ==Zero==
|
314
|
+
0, 0, 0, 0, // [180] invalid_language ?? ==Zero==
|
315
|
+
0, 0, 0, 0, // [181] invalid_language ?? ==Zero==
|
316
|
+
0, 0, 0, 0, // [182] invalid_language ?? ==Zero==
|
317
|
+
0, 0, 0, 0, // [183] invalid_language ?? ==Zero==
|
318
|
+
0, 0, 0, 0, // [184] invalid_language ?? ==Zero==
|
319
|
+
0, 0, 0, 0, // [185] X_BORK_BORK_BORK zzb ==Zero==
|
320
|
+
0, 0, 0, 0, // [186] X_PIG_LATIN zzp ==Zero==
|
321
|
+
0, 0, 0, 0, // [187] X_HACKER zzh ==Zero==
|
322
|
+
0, 0, 0, 0, // [188] X_KLINGON tlh ==Zero==
|
323
|
+
0, 0, 0, 0, // [189] X_ELMER_FUDD zze ==Zero==
|
324
|
+
0, 0, 0, 0, // [190] X_OGHAM xx-Ogam ==Zero==
|
325
|
+
0, 0, 0, 0, // [191] X_RUNIC xx-Runr ==Zero==
|
326
|
+
|
327
|
+
0, 0, 0, 0, // [192] X_YI xx-Yiii ==Zero==
|
328
|
+
0, 0, 0, 0, // [193] X_OLD_ITALIC xx-Ital ==Zero==
|
329
|
+
0, 0, 0, 0, // [194] X_GOTHIC xx-Goth ==Zero==
|
330
|
+
0, 0, 0, 0, // [195] X_DESERET xx-Dsrt ==Zero==
|
331
|
+
0, 0, 0, 0, // [196] X_HANUNOO xx-Hano ==Zero==
|
332
|
+
0, 0, 0, 0, // [197] X_BUHID xx-Buhd ==Zero==
|
333
|
+
0, 0, 0, 0, // [198] X_TAGBANWA xx-Tagb ==Zero==
|
334
|
+
0, 0, 0, 0, // [199] X_TAI_LE xx-Tale ==Zero==
|
335
|
+
0, 0, 0, 0, // [200] X_LINEAR_B xx-Linb ==Zero==
|
336
|
+
0, 0, 0, 0, // [201] X_UGARITIC xx-Ugar ==Zero==
|
337
|
+
0, 0, 0, 0, // [202] X_SHAVIAN xx-Shaw ==Zero==
|
338
|
+
0, 0, 0, 0, // [203] X_OSMANYA xx-Osma ==Zero==
|
339
|
+
0, 0, 0, 0, // [204] X_CYPRIOT xx-Cprt ==Zero==
|
340
|
+
0, 0, 0, 0, // [205] X_BUGINESE xx-Bugi ==Zero==
|
341
|
+
0, 0, 0, 0, // [206] X_COPTIC xx-Copt ==Zero==
|
342
|
+
0, 0, 0, 0, // [207] X_NEW_TAI_LUE xx-Talu ==Zero==
|
343
|
+
0, 0, 0, 0, // [208] X_GLAGOLITIC xx-Glag ==Zero==
|
344
|
+
0, 0, 0, 0, // [209] X_TIFINAGH xx-Tfng ==Zero==
|
345
|
+
0, 0, 0, 0, // [210] X_SYLOTI_NAGRI xx-Sylo ==Zero==
|
346
|
+
0, 0, 0, 0, // [211] X_OLD_PERSIAN xx-Xpeo ==Zero==
|
347
|
+
0, 0, 0, 0, // [212] X_KHAROSHTHI xx-Khar ==Zero==
|
348
|
+
0, 0, 0, 0, // [213] X_BALINESE xx-Bali ==Zero==
|
349
|
+
0, 0, 0, 0, // [214] X_CUNEIFORM xx-Xsux ==Zero==
|
350
|
+
0, 0, 0, 0, // [215] X_PHOENICIAN xx-Phnx ==Zero==
|
351
|
+
0, 0, 0, 0, // [216] X_PHAGS_PA xx-Phag ==Zero==
|
352
|
+
0, 0, 0, 0, // [217] X_NKO xx-Nkoo ==Zero==
|
353
|
+
0, 0, 0, 0, // [218] X_SUDANESE xx-Sund ==Zero==
|
354
|
+
0, 0, 0, 0, // [219] X_LEPCHA xx-Lepc ==Zero==
|
355
|
+
0, 0, 0, 0, // [220] X_OL_CHIKI xx-Olck ==Zero==
|
356
|
+
0, 0, 0, 0, // [221] X_VAI xx-Vaii ==Zero==
|
357
|
+
0, 0, 0, 0, // [222] X_SAURASHTRA xx-Saur ==Zero==
|
358
|
+
0, 0, 0, 0, // [223] X_KAYAH_LI xx-Kali ==Zero==
|
359
|
+
|
360
|
+
0, 0, 0, 0, // [224] X_REJANG xx-Rjng ==Zero==
|
361
|
+
0, 0, 0, 0, // [225] X_LYCIAN xx-Lyci ==Zero==
|
362
|
+
0, 0, 0, 0, // [226] X_CARIAN xx-Cari ==Zero==
|
363
|
+
0, 0, 0, 0, // [227] X_LYDIAN xx-Lydi ==Zero==
|
364
|
+
0, 0, 0, 0, // [228] X_CHAM xx-Cham ==Zero==
|
365
|
+
0, 0, 0, 0, // [229] invalid_language ?? ==Zero==
|
366
|
+
0, 0, 0, 0, // [230] invalid_language ?? ==Zero==
|
367
|
+
0, 0, 0, 0, // [231] invalid_language ?? ==Zero==
|
368
|
+
0, 0, 0, 0, // [232] invalid_language ?? ==Zero==
|
369
|
+
0, 0, 0, 0, // [233] invalid_language ?? ==Zero==
|
370
|
+
0, 0, 0, 0, // [234] invalid_language ?? ==Zero==
|
371
|
+
0, 0, 0, 0, // [235] invalid_language ?? ==Zero==
|
372
|
+
0, 0, 0, 0, // [236] invalid_language ?? ==Zero==
|
373
|
+
0, 0, 0, 0, // [237] invalid_language ?? ==Zero==
|
374
|
+
0, 0, 0, 0, // [238] invalid_language ?? ==Zero==
|
375
|
+
0, 0, 0, 0, // [239] invalid_language ?? ==Zero==
|
376
|
+
0, 0, 0, 0, // [240] invalid_language ?? ==Zero==
|
377
|
+
0, 0, 0, 0, // [241] invalid_language ?? ==Zero==
|
378
|
+
0, 0, 0, 0, // [242] invalid_language ?? ==Zero==
|
379
|
+
0, 0, 0, 0, // [243] invalid_language ?? ==Zero==
|
380
|
+
};
|
381
|
+
|
382
|
+
COMPILE_ASSERT(EXT_NUM_LANGUAGES >= 209, k_ext_num_languages_changed);
|
@@ -0,0 +1,49 @@
|
|
1
|
+
// Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
|
2
|
+
// Use of this source code is governed by a BSD-style license that can be
|
3
|
+
// found in the LICENSE file.
|
4
|
+
//
|
5
|
+
// Created by postproc-shortwords 1.7 on 2009-01-29 16:13:04
|
6
|
+
// From input file /tmp/langdet_v25_12cjk_sort.utf8
|
7
|
+
// See compact_lang_det.cc for usage
|
8
|
+
//
|
9
|
+
#include "encodings/compact_lang_det/cldutil.h"
|
10
|
+
|
11
|
+
// Suppressed:
|
12
|
+
// az-Arab az-Cyrl ku-Latn tg-Arab za-Hani zzb-Latn zze-Latn zzh-Latn ru-Latn
|
13
|
+
|
14
|
+
// Remapped:
|
15
|
+
// xxx-Latn=>ut-Latn sh-Latn=>hr-Latn sh-Cyrl=>sr-Cyrl
|
16
|
+
|
17
|
+
|
18
|
+
static const int kCjkBiTableBuildDate = 20090129; // yyyymmdd
|
19
|
+
static const int kCjkBiTableSize = 1; // Bucket count
|
20
|
+
static const int kCjkBiTableKeyMask = 0xffffffff; // Mask hash key
|
21
|
+
|
22
|
+
COMPILE_ASSERT(MONTENEGRIN == 160, k_montenegrin_changed);
|
23
|
+
COMPILE_ASSERT(EXT_NUM_LANGUAGES == 209, k_ext_num_languages_changed);
|
24
|
+
|
25
|
+
// Empty table
|
26
|
+
static const cld::IndirectProbBucket4 kCjkBiTable[kCjkBiTableSize] = {
|
27
|
+
// key[4], words[4] in UTF-8
|
28
|
+
// value[4]
|
29
|
+
{ {0x00000000,0x00000000,0x00000000,0x00000000}}, // [000] c
|
30
|
+
};
|
31
|
+
|
32
|
+
static const uint32 kCjkBiTableInd[1] = {
|
33
|
+
// [0000]
|
34
|
+
0x00000000, };
|
35
|
+
|
36
|
+
COMPILE_ASSERT(1 < (1 << 16), k_indirectbits_too_small);
|
37
|
+
|
38
|
+
|
39
|
+
extern const cld::CLDTableSummary kCjkBiTable_obj = {
|
40
|
+
kCjkBiTable,
|
41
|
+
kCjkBiTableInd,
|
42
|
+
kCjkBiTableSize,
|
43
|
+
arraysize(kCjkBiTableInd),
|
44
|
+
kCjkBiTableKeyMask,
|
45
|
+
kCjkBiTableBuildDate,
|
46
|
+
};
|
47
|
+
|
48
|
+
// End of generated tables
|
49
|
+
|