language_detection 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (100) hide show
  1. data/.gitignore +19 -0
  2. data/Gemfile +4 -0
  3. data/LICENSE.txt +22 -0
  4. data/README.md +85 -0
  5. data/Rakefile +11 -0
  6. data/ext/cld/Makefile +34 -0
  7. data/ext/cld/base/basictypes.h +348 -0
  8. data/ext/cld/base/build_config.h +124 -0
  9. data/ext/cld/base/casts.h +156 -0
  10. data/ext/cld/base/commandlineflags.h +443 -0
  11. data/ext/cld/base/crash.h +41 -0
  12. data/ext/cld/base/dynamic_annotations.h +358 -0
  13. data/ext/cld/base/global_strip_options.h +59 -0
  14. data/ext/cld/base/log_severity.h +46 -0
  15. data/ext/cld/base/logging.h +1403 -0
  16. data/ext/cld/base/macros.h +243 -0
  17. data/ext/cld/base/port.h +54 -0
  18. data/ext/cld/base/scoped_ptr.h +428 -0
  19. data/ext/cld/base/stl_decl.h +0 -0
  20. data/ext/cld/base/stl_decl_msvc.h +107 -0
  21. data/ext/cld/base/string_util.h +29 -0
  22. data/ext/cld/base/strtoint.h +93 -0
  23. data/ext/cld/base/template_util.h +96 -0
  24. data/ext/cld/base/type_traits.h +198 -0
  25. data/ext/cld/base/vlog_is_on.h +143 -0
  26. data/ext/cld/cld.so +0 -0
  27. data/ext/cld/encodings/compact_lang_det/cldutil.cc +905 -0
  28. data/ext/cld/encodings/compact_lang_det/cldutil.h +1205 -0
  29. data/ext/cld/encodings/compact_lang_det/cldutil_dbg.h +76 -0
  30. data/ext/cld/encodings/compact_lang_det/cldutil_dbg_empty.cc +76 -0
  31. data/ext/cld/encodings/compact_lang_det/compact_lang_det.cc +62 -0
  32. data/ext/cld/encodings/compact_lang_det/compact_lang_det.h +145 -0
  33. data/ext/cld/encodings/compact_lang_det/compact_lang_det_impl.cc +2574 -0
  34. data/ext/cld/encodings/compact_lang_det/compact_lang_det_impl.h +173 -0
  35. data/ext/cld/encodings/compact_lang_det/compact_lang_det_unittest_small.cc +406 -0
  36. data/ext/cld/encodings/compact_lang_det/compile.cmd +1 -0
  37. data/ext/cld/encodings/compact_lang_det/ext_lang_enc.cc +545 -0
  38. data/ext/cld/encodings/compact_lang_det/ext_lang_enc.h +119 -0
  39. data/ext/cld/encodings/compact_lang_det/generated/cld_generated_score_deltaoctachrome_0406.cc +380 -0
  40. data/ext/cld/encodings/compact_lang_det/generated/cld_generated_score_quadchrome_0406.cc +382 -0
  41. data/ext/cld/encodings/compact_lang_det/generated/compact_lang_det_generated_cjkbis_0.cc +49 -0
  42. data/ext/cld/encodings/compact_lang_det/generated/compact_lang_det_generated_ctjkvz.cc +7119 -0
  43. data/ext/cld/encodings/compact_lang_det/generated/compact_lang_det_generated_ctjkvz_0.cc +61 -0
  44. data/ext/cld/encodings/compact_lang_det/generated/compact_lang_det_generated_deltaoctachrome.cc +1263 -0
  45. data/ext/cld/encodings/compact_lang_det/generated/compact_lang_det_generated_longwords8_0.cc +53 -0
  46. data/ext/cld/encodings/compact_lang_det/generated/compact_lang_det_generated_meanscore.h +10 -0
  47. data/ext/cld/encodings/compact_lang_det/generated/compact_lang_det_generated_quads_0.cc +50 -0
  48. data/ext/cld/encodings/compact_lang_det/generated/compact_lang_det_generated_quadschrome.cc +70935 -0
  49. data/ext/cld/encodings/compact_lang_det/getonescriptspan.cc +570 -0
  50. data/ext/cld/encodings/compact_lang_det/getonescriptspan.h +131 -0
  51. data/ext/cld/encodings/compact_lang_det/letterscript_enum.cc +117 -0
  52. data/ext/cld/encodings/compact_lang_det/letterscript_enum.h +99 -0
  53. data/ext/cld/encodings/compact_lang_det/subsetsequence.cc +259 -0
  54. data/ext/cld/encodings/compact_lang_det/subsetsequence.h +44 -0
  55. data/ext/cld/encodings/compact_lang_det/subsetsequence_unittest.cc +99 -0
  56. data/ext/cld/encodings/compact_lang_det/tote.cc +299 -0
  57. data/ext/cld/encodings/compact_lang_det/tote.h +89 -0
  58. data/ext/cld/encodings/compact_lang_det/unittest_data.h +193 -0
  59. data/ext/cld/encodings/compact_lang_det/utf8propjustletter.h +1162 -0
  60. data/ext/cld/encodings/compact_lang_det/utf8propletterscriptnum.h +1222 -0
  61. data/ext/cld/encodings/compact_lang_det/utf8scannotjustletterspecial.h +1185 -0
  62. data/ext/cld/encodings/compact_lang_det/win/cld_basictypes.h +10 -0
  63. data/ext/cld/encodings/compact_lang_det/win/cld_commandlineflags.h +28 -0
  64. data/ext/cld/encodings/compact_lang_det/win/cld_google.h +18 -0
  65. data/ext/cld/encodings/compact_lang_det/win/cld_htmlutils.h +13 -0
  66. data/ext/cld/encodings/compact_lang_det/win/cld_htmlutils_google3.cc +32 -0
  67. data/ext/cld/encodings/compact_lang_det/win/cld_htmlutils_windows.cc +29 -0
  68. data/ext/cld/encodings/compact_lang_det/win/cld_logging.h +21 -0
  69. data/ext/cld/encodings/compact_lang_det/win/cld_macros.h +19 -0
  70. data/ext/cld/encodings/compact_lang_det/win/cld_strtoint.h +26 -0
  71. data/ext/cld/encodings/compact_lang_det/win/cld_unicodetext.cc +84 -0
  72. data/ext/cld/encodings/compact_lang_det/win/cld_unicodetext.h +40 -0
  73. data/ext/cld/encodings/compact_lang_det/win/cld_unilib.h +15 -0
  74. data/ext/cld/encodings/compact_lang_det/win/cld_unilib_google3.cc +18 -0
  75. data/ext/cld/encodings/compact_lang_det/win/cld_unilib_windows.cc +29 -0
  76. data/ext/cld/encodings/compact_lang_det/win/cld_utf.h +24 -0
  77. data/ext/cld/encodings/compact_lang_det/win/cld_utf8statetable.cc +224 -0
  78. data/ext/cld/encodings/compact_lang_det/win/cld_utf8statetable.h +141 -0
  79. data/ext/cld/encodings/compact_lang_det/win/cld_utf8utils.h +22 -0
  80. data/ext/cld/encodings/compact_lang_det/win/cld_utf8utils_google3.cc +18 -0
  81. data/ext/cld/encodings/compact_lang_det/win/cld_utf8utils_windows.cc +17 -0
  82. data/ext/cld/encodings/compact_lang_det/win/normalizedunicodetext.cc +172 -0
  83. data/ext/cld/encodings/compact_lang_det/win/normalizedunicodetext.h +67 -0
  84. data/ext/cld/encodings/internal/encodings.cc +12 -0
  85. data/ext/cld/encodings/lang_enc.h +254 -0
  86. data/ext/cld/encodings/proto/encodings.pb.h +169 -0
  87. data/ext/cld/encodings/public/encodings.h +301 -0
  88. data/ext/cld/extconf.rb +1 -0
  89. data/ext/cld/language_detection.cc +88 -0
  90. data/ext/cld/languages/internal/languages.cc +337 -0
  91. data/ext/cld/languages/proto/languages.pb.h +179 -0
  92. data/ext/cld/languages/public/languages.h +379 -0
  93. data/language_detection.gemspec +28 -0
  94. data/lib/language_detection/string.rb +1 -0
  95. data/lib/language_detection/version.rb +3 -0
  96. data/lib/language_detection.rb +54 -0
  97. data/test/_helper.rb +15 -0
  98. data/test/fixtures/languages.csv +80 -0
  99. data/test/language_detection_test.rb +88 -0
  100. metadata +250 -0
@@ -0,0 +1,382 @@
1
+ // Copyright (c) 2011 The Chromium Authors. All rights reserved.
2
+ // Use of this source code is governed by a BSD-style license that can be
3
+ // found in the LICENSE file.
4
+
5
+ #include "base/basictypes.h"
6
+ #include "encodings/compact_lang_det/ext_lang_enc.h"
7
+
8
+ // score_me text [ 8] ja not found in tables
9
+ // score_me text [ 9] ko not found in tables
10
+ // score_me text [ 16] zh not found in tables
11
+ // score_me text [ 18] el not found in tables
12
+ // score_me text [ 26] un not found in tables
13
+ // score_me text [ 39] la not found in tables
14
+ // score_me text [ 41] ml not found in tables
15
+ // score_me text [ 43] ne not found in tables
16
+ // score_me text [ 44] te not found in tables
17
+ // score_me text [ 46] ta not found in tables
18
+ // score_me text [ 48] jw not found in tables
19
+ // score_me text [ 49] oc not found in tables
20
+ // score_me text [ 51] bh not found in tables
21
+ // score_me text [ 52] gu not found in tables
22
+ // score_me text [ 53] th not found in tables
23
+ // score_me text [ 56] eo not found in tables
24
+ // score_me text [ 58] ia not found in tables
25
+ // score_me text [ 59] kn not found in tables
26
+ // score_me text [ 60] pa not found in tables
27
+ // score_me text [ 61] gd not found in tables
28
+ // score_me text [ 64] mr not found in tables
29
+ // score_me text [ 67] fy not found in tables
30
+ // score_me text [ 69] zhT not found in tables
31
+ // score_me text [ 70] fo not found in tables
32
+ // score_me text [ 71] su not found in tables
33
+ // score_me text [ 72] uz not found in tables
34
+ // score_me text [ 73] am not found in tables
35
+ // score_me text [ 75] ka not found in tables
36
+ // score_me text [ 76] ti not found in tables
37
+ // score_me text [ 78] bs not found in tables
38
+ // score_me text [ 79] si not found in tables
39
+ // score_me text [ 80] nn not found in tables
40
+ // score_me text [ 83] xh not found in tables
41
+ // score_me text [ 84] zu not found in tables
42
+ // score_me text [ 85] gn not found in tables
43
+ // score_me text [ 86] st not found in tables
44
+ // score_me text [ 87] tk not found in tables
45
+ // score_me text [ 88] ky not found in tables
46
+ // score_me text [ 89] br not found in tables
47
+ // score_me text [ 90] tw not found in tables
48
+ // score_me text [ 93] so not found in tables
49
+ // score_me text [ 94] ug not found in tables
50
+ // score_me text [ 95] ku not found in tables
51
+ // score_me text [ 96] mn not found in tables
52
+ // score_me text [ 97] hy not found in tables
53
+ // score_me text [ 98] lo not found in tables
54
+ // score_me text [ 99] sd not found in tables
55
+ // score_me text [100] rm not found in tables
56
+ // score_me text [102] lb not found in tables
57
+ // score_me text [103] my not found in tables
58
+ // score_me text [104] km not found in tables
59
+ // score_me text [105] bo not found in tables
60
+ // score_me text [107] chr not found in tables
61
+ // score_me text [109] sit-NP not found in tables
62
+ // score_me text [110] or not found in tables
63
+ // score_me text [111] as not found in tables
64
+ // score_me text [112] co not found in tables
65
+ // score_me text [113] ie not found in tables
66
+ // score_me text [114] kk not found in tables
67
+ // score_me text [115] ln not found in tables
68
+ // score_me text [116] mo not found in tables
69
+ // score_me text [117] ps not found in tables
70
+ // score_me text [118] qu not found in tables
71
+ // score_me text [119] sn not found in tables
72
+ // score_me text [120] tg not found in tables
73
+ // score_me text [121] tt not found in tables
74
+ // score_me text [122] to not found in tables
75
+ // score_me text [123] yo not found in tables
76
+ // score_me text [128] mi not found in tables
77
+ // score_me text [129] wo not found in tables
78
+ // score_me text [130] ab not found in tables
79
+ // score_me text [131] aa not found in tables
80
+ // score_me text [132] ay not found in tables
81
+ // score_me text [133] ba not found in tables
82
+ // score_me text [134] bi not found in tables
83
+ // score_me text [135] dz not found in tables
84
+ // score_me text [136] fj not found in tables
85
+ // score_me text [137] kl not found in tables
86
+ // score_me text [138] ha not found in tables
87
+ // score_me text [140] ik not found in tables
88
+ // score_me text [141] iu not found in tables
89
+ // score_me text [142] ks not found in tables
90
+ // score_me text [143] rw not found in tables
91
+ // score_me text [144] mg not found in tables
92
+ // score_me text [145] na not found in tables
93
+ // score_me text [146] om not found in tables
94
+ // score_me text [147] rn not found in tables
95
+ // score_me text [148] sm not found in tables
96
+ // score_me text [149] sg not found in tables
97
+ // score_me text [150] sa not found in tables
98
+ // score_me text [151] ss not found in tables
99
+ // score_me text [152] ts not found in tables
100
+ // score_me text [153] tn not found in tables
101
+ // score_me text [154] vo not found in tables
102
+ // score_me text [155] za not found in tables
103
+ // score_me text [156] kha not found in tables
104
+ // score_me text [157] sco not found in tables
105
+ // score_me text [158] lg not found in tables
106
+ // score_me text [159] gv not found in tables
107
+ // score_me text [160] srM not found in tables
108
+ // score_me text [161] ak not found in tables
109
+ // score_me text [162] ig not found in tables
110
+ // score_me text [163] mfe not found in tables
111
+ // score_me text [164] haw not found in tables
112
+ // score_me text [165] nso not found in tables
113
+ // score_me text [166] nr not found in tables
114
+ // score_me text [167] ve not found in tables
115
+ // score_me text [168] ny not found in tables
116
+ // score_me text [169] crs not found in tables
117
+ // score_me text [185] zzb not found in tables
118
+ // score_me text [186] zzp not found in tables
119
+ // score_me text [187] zzh not found in tables
120
+ // score_me text [188] tlh not found in tables
121
+ // score_me text [189] zze not found in tables
122
+
123
+ // No score_me text for [ 25] xxx
124
+
125
+ // Average score per 1024 bytes
126
+ // Renamed kAvgQuadScore -> kMeanScore to match older CLD code.
127
+ extern const short kMeanScore[244 * 4] = {
128
+ // Latn Cyrl Arab Other script
129
+ 1234, 0, 0, 0, // [ 0] ENGLISH en
130
+ 721, 0, 0, 0, // [ 1] DANISH da
131
+ 1037, 0, 0, 0, // [ 2] DUTCH nl
132
+ 1337, 0, 0, 0, // [ 3] FINNISH fi
133
+ 973, 0, 0, 0, // [ 4] FRENCH fr
134
+ 1211, 0, 0, 0, // [ 5] GERMAN de
135
+ 0, 0, 0, 727, // [ 6] HEBREW iw
136
+ 866, 0, 0, 0, // [ 7] ITALIAN it
137
+ 0, 0, 0, 0, // [ 8] Japanese ja ==Zero==
138
+ 0, 0, 0, 0, // [ 9] Korean ko ==Zero==
139
+ 833, 0, 0, 0, // [ 10] NORWEGIAN no
140
+ 1280, 0, 0, 0, // [ 11] POLISH pl
141
+ 718, 0, 0, 0, // [ 12] PORTUGUESE pt
142
+ 0, 767, 0, 0, // [ 13] RUSSIAN ru
143
+ 642, 0, 0, 0, // [ 14] SPANISH es
144
+ 826, 0, 0, 0, // [ 15] SWEDISH sv
145
+ 0, 0, 0, 0, // [ 16] Chinese zh ==Zero==
146
+ 982, 0, 0, 0, // [ 17] CZECH cs
147
+ 0, 0, 0, 0, // [ 18] GREEK el ==Zero==
148
+ 1214, 0, 0, 0, // [ 19] ICELANDIC is
149
+ 1194, 0, 0, 0, // [ 20] LATVIAN lv
150
+ 1220, 0, 0, 0, // [ 21] LITHUANIAN lt
151
+ 951, 0, 0, 0, // [ 22] ROMANIAN ro
152
+ 1335, 0, 0, 0, // [ 23] HUNGARIAN hu
153
+ 1145, 0, 0, 0, // [ 24] ESTONIAN et
154
+ 0, 0, 0, 0, // [ 25] Ignore xxx ==Zero==
155
+ 0, 0, 0, 0, // [ 26] Unknown un ==Zero==
156
+ 0, 691, 0, 0, // [ 27] BULGARIAN bg
157
+ 774, 0, 0, 0, // [ 28] CROATIAN hr
158
+ 1048, 686, 0, 0, // [ 29] SERBIAN sr
159
+ 1503, 0, 0, 0, // [ 30] IRISH ga
160
+ 464, 0, 0, 0, // [ 31] GALICIAN gl
161
+
162
+ 1570, 0, 0, 0, // [ 32] TAGALOG tl
163
+ 1390, 0, 0, 0, // [ 33] TURKISH tr
164
+ 0, 677, 0, 0, // [ 34] UKRAINIAN uk
165
+ 0, 0, 0, 601, // [ 35] HINDI hi
166
+ 0, 695, 0, 0, // [ 36] MACEDONIAN mk
167
+ 0, 0, 0, 563, // [ 37] BENGALI bn
168
+ 1307, 0, 0, 0, // [ 38] INDONESIAN id
169
+ 0, 0, 0, 0, // [ 39] LATIN la ==Zero==
170
+ 1402, 0, 0, 0, // [ 40] MALAY ms
171
+ 0, 0, 0, 0, // [ 41] MALAYALAM ml ==Zero==
172
+ 1536, 0, 0, 0, // [ 42] WELSH cy
173
+ 0, 0, 0, 0, // [ 43] NEPALI ne ==Zero==
174
+ 0, 0, 0, 0, // [ 44] TELUGU te ==Zero==
175
+ 1146, 0, 0, 0, // [ 45] ALBANIAN sq
176
+ 0, 0, 0, 0, // [ 46] TAMIL ta ==Zero==
177
+ 0, 834, 0, 0, // [ 47] BELARUSIAN be
178
+ 0, 0, 0, 0, // [ 48] JAVANESE jw ==Zero==
179
+ 0, 0, 0, 0, // [ 49] OCCITAN oc ==Zero==
180
+ 0, 0, 964, 0, // [ 50] URDU ur
181
+ 0, 0, 0, 0, // [ 51] BIHARI bh ==Zero==
182
+ 0, 0, 0, 0, // [ 52] GUJARATI gu ==Zero==
183
+ 0, 0, 0, 0, // [ 53] THAI th ==Zero==
184
+ 0, 0, 735, 0, // [ 54] ARABIC ar
185
+ 667, 0, 0, 0, // [ 55] CATALAN ca
186
+ 0, 0, 0, 0, // [ 56] ESPERANTO eo ==Zero==
187
+ 1442, 0, 0, 0, // [ 57] BASQUE eu
188
+ 0, 0, 0, 0, // [ 58] INTERLINGUA ia ==Zero==
189
+ 0, 0, 0, 0, // [ 59] KANNADA kn ==Zero==
190
+ 0, 0, 0, 0, // [ 60] PUNJABI pa ==Zero==
191
+ 0, 0, 0, 0, // [ 61] SCOTS_GAELIC gd ==Zero==
192
+ 1448, 0, 0, 0, // [ 62] SWAHILI sw
193
+ 907, 0, 0, 0, // [ 63] SLOVENIAN sl
194
+
195
+ 0, 0, 0, 0, // [ 64] MARATHI mr ==Zero==
196
+ 1027, 0, 0, 0, // [ 65] MALTESE mt
197
+ 1097, 0, 0, 0, // [ 66] VIETNAMESE vi
198
+ 0, 0, 0, 0, // [ 67] FRISIAN fy ==Zero==
199
+ 1007, 0, 0, 0, // [ 68] SLOVAK sk
200
+ 0, 0, 0, 0, // [ 69] ChineseT zhT ==Zero==
201
+ 0, 0, 0, 0, // [ 70] FAROESE fo ==Zero==
202
+ 0, 0, 0, 0, // [ 71] SUNDANESE su ==Zero==
203
+ 0, 0, 0, 0, // [ 72] UZBEK uz ==Zero==
204
+ 0, 0, 0, 0, // [ 73] AMHARIC am ==Zero==
205
+ 1371, 0, 0, 0, // [ 74] AZERBAIJANI az
206
+ 0, 0, 0, 0, // [ 75] GEORGIAN ka ==Zero==
207
+ 0, 0, 0, 0, // [ 76] TIGRINYA ti ==Zero==
208
+ 0, 0, 855, 0, // [ 77] PERSIAN fa
209
+ 0, 0, 0, 0, // [ 78] BOSNIAN bs ==Zero==
210
+ 0, 0, 0, 0, // [ 79] SINHALESE si ==Zero==
211
+ 0, 0, 0, 0, // [ 80] NORWEGIAN_N nn ==Zero==
212
+ 0, 0, 0, 0, // [ 81] PORTUGUESE_P pt-PT ==Zero==
213
+ 0, 0, 0, 0, // [ 82] PORTUGUESE_B pt-BR ==Zero==
214
+ 0, 0, 0, 0, // [ 83] XHOSA xh ==Zero==
215
+ 0, 0, 0, 0, // [ 84] ZULU zu ==Zero==
216
+ 0, 0, 0, 0, // [ 85] GUARANI gn ==Zero==
217
+ 0, 0, 0, 0, // [ 86] SESOTHO st ==Zero==
218
+ 0, 0, 0, 0, // [ 87] TURKMEN tk ==Zero==
219
+ 0, 0, 0, 0, // [ 88] KYRGYZ ky ==Zero==
220
+ 0, 0, 0, 0, // [ 89] BRETON br ==Zero==
221
+ 0, 0, 0, 0, // [ 90] TWI tw ==Zero==
222
+ 0, 0, 0, 807, // [ 91] YIDDISH yi
223
+ 0, 0, 0, 0, // [ 92] SERBO_CROATIAN sh ==Zero==
224
+ 0, 0, 0, 0, // [ 93] SOMALI so ==Zero==
225
+ 0, 0, 0, 0, // [ 94] UIGHUR ug ==Zero==
226
+ 0, 0, 0, 0, // [ 95] KURDISH ku ==Zero==
227
+
228
+ 0, 0, 0, 0, // [ 96] MONGOLIAN mn ==Zero==
229
+ 0, 0, 0, 0, // [ 97] ARMENIAN hy ==Zero==
230
+ 0, 0, 0, 0, // [ 98] LAOTHIAN lo ==Zero==
231
+ 0, 0, 0, 0, // [ 99] SINDHI sd ==Zero==
232
+ 0, 0, 0, 0, // [100] RHAETO_ROMANCE rm ==Zero==
233
+ 885, 0, 0, 0, // [101] AFRIKAANS af
234
+ 0, 0, 0, 0, // [102] LUXEMBOURGISH lb ==Zero==
235
+ 0, 0, 0, 0, // [103] BURMESE my ==Zero==
236
+ 0, 0, 0, 0, // [104] KHMER km ==Zero==
237
+ 0, 0, 0, 0, // [105] TIBETAN bo ==Zero==
238
+ 0, 0, 0, 0, // [106] DHIVEHI dv ==Zero==
239
+ 0, 0, 0, 0, // [107] CHEROKEE chr ==Zero==
240
+ 0, 0, 0, 0, // [108] SYRIAC syr ==Zero==
241
+ 0, 0, 0, 0, // [109] LIMBU sit-NP ==Zero==
242
+ 0, 0, 0, 0, // [110] ORIYA or ==Zero==
243
+ 0, 0, 0, 0, // [111] ASSAMESE as ==Zero==
244
+ 0, 0, 0, 0, // [112] CORSICAN co ==Zero==
245
+ 0, 0, 0, 0, // [113] INTERLINGUE ie ==Zero==
246
+ 0, 0, 0, 0, // [114] KAZAKH kk ==Zero==
247
+ 0, 0, 0, 0, // [115] LINGALA ln ==Zero==
248
+ 0, 0, 0, 0, // [116] MOLDAVIAN mo ==Zero==
249
+ 0, 0, 0, 0, // [117] PASHTO ps ==Zero==
250
+ 0, 0, 0, 0, // [118] QUECHUA qu ==Zero==
251
+ 0, 0, 0, 0, // [119] SHONA sn ==Zero==
252
+ 0, 0, 0, 0, // [120] TAJIK tg ==Zero==
253
+ 0, 0, 0, 0, // [121] TATAR tt ==Zero==
254
+ 0, 0, 0, 0, // [122] TONGA to ==Zero==
255
+ 0, 0, 0, 0, // [123] YORUBA yo ==Zero==
256
+ 0, 0, 0, 0, // [124] CREOLES_AND_PIDGINS_ENGLISH_BASED cpe ==Zero==
257
+ 0, 0, 0, 0, // [125] CREOLES_AND_PIDGINS_FRENCH_BASED cpf ==Zero==
258
+ 0, 0, 0, 0, // [126] CREOLES_AND_PIDGINS_PORTUGUESE_BASED cpp ==Zero==
259
+ 0, 0, 0, 0, // [127] CREOLES_AND_PIDGINS_OTHER crp ==Zero==
260
+
261
+ 0, 0, 0, 0, // [128] MAORI mi ==Zero==
262
+ 0, 0, 0, 0, // [129] WOLOF wo ==Zero==
263
+ 0, 0, 0, 0, // [130] ABKHAZIAN ab ==Zero==
264
+ 0, 0, 0, 0, // [131] AFAR aa ==Zero==
265
+ 0, 0, 0, 0, // [132] AYMARA ay ==Zero==
266
+ 0, 0, 0, 0, // [133] BASHKIR ba ==Zero==
267
+ 0, 0, 0, 0, // [134] BISLAMA bi ==Zero==
268
+ 0, 0, 0, 0, // [135] DZONGKHA dz ==Zero==
269
+ 0, 0, 0, 0, // [136] FIJIAN fj ==Zero==
270
+ 0, 0, 0, 0, // [137] GREENLANDIC kl ==Zero==
271
+ 0, 0, 0, 0, // [138] HAUSA ha ==Zero==
272
+ 1151, 0, 0, 0, // [139] HAITIAN_CREOLE ht
273
+ 0, 0, 0, 0, // [140] INUPIAK ik ==Zero==
274
+ 0, 0, 0, 0, // [141] INUKTITUT iu ==Zero==
275
+ 0, 0, 0, 0, // [142] KASHMIRI ks ==Zero==
276
+ 0, 0, 0, 0, // [143] KINYARWANDA rw ==Zero==
277
+ 0, 0, 0, 0, // [144] MALAGASY mg ==Zero==
278
+ 0, 0, 0, 0, // [145] NAURU na ==Zero==
279
+ 0, 0, 0, 0, // [146] OROMO om ==Zero==
280
+ 0, 0, 0, 0, // [147] RUNDI rn ==Zero==
281
+ 0, 0, 0, 0, // [148] SAMOAN sm ==Zero==
282
+ 0, 0, 0, 0, // [149] SANGO sg ==Zero==
283
+ 0, 0, 0, 0, // [150] SANSKRIT sa ==Zero==
284
+ 0, 0, 0, 0, // [151] SISWANT ss ==Zero==
285
+ 0, 0, 0, 0, // [152] TSONGA ts ==Zero==
286
+ 0, 0, 0, 0, // [153] TSWANA tn ==Zero==
287
+ 0, 0, 0, 0, // [154] VOLAPUK vo ==Zero==
288
+ 0, 0, 0, 0, // [155] ZHUANG za ==Zero==
289
+ 0, 0, 0, 0, // [156] KHASI kha ==Zero==
290
+ 0, 0, 0, 0, // [157] SCOTS sco ==Zero==
291
+ 0, 0, 0, 0, // [158] GANDA lg ==Zero==
292
+ 0, 0, 0, 0, // [159] MANX gv ==Zero==
293
+
294
+ 0, 0, 0, 0, // [160] MONTENEGRIN srM ==Zero==
295
+ 0, 0, 0, 0, // [161] AKAN ak ==Zero==
296
+ 0, 0, 0, 0, // [162] IGBO ig ==Zero==
297
+ 0, 0, 0, 0, // [163] MAURITIAN_CREOLE mfe ==Zero==
298
+ 0, 0, 0, 0, // [164] HAWAIIAN haw ==Zero==
299
+ 0, 0, 0, 0, // [165] PEDI nso ==Zero==
300
+ 0, 0, 0, 0, // [166] NDEBELE nr ==Zero==
301
+ 0, 0, 0, 0, // [167] VENDA ve ==Zero==
302
+ 0, 0, 0, 0, // [168] CHICHEWA ny ==Zero==
303
+ 0, 0, 0, 0, // [169] SESELWA crs ==Zero==
304
+ 0, 0, 0, 0, // [170] invalid_language ?? ==Zero==
305
+ 0, 0, 0, 0, // [171] invalid_language ?? ==Zero==
306
+ 0, 0, 0, 0, // [172] invalid_language ?? ==Zero==
307
+ 0, 0, 0, 0, // [173] invalid_language ?? ==Zero==
308
+ 0, 0, 0, 0, // [174] invalid_language ?? ==Zero==
309
+ 0, 0, 0, 0, // [175] invalid_language ?? ==Zero==
310
+ 0, 0, 0, 0, // [176] invalid_language ?? ==Zero==
311
+ 0, 0, 0, 0, // [177] invalid_language ?? ==Zero==
312
+ 0, 0, 0, 0, // [178] invalid_language ?? ==Zero==
313
+ 0, 0, 0, 0, // [179] invalid_language ?? ==Zero==
314
+ 0, 0, 0, 0, // [180] invalid_language ?? ==Zero==
315
+ 0, 0, 0, 0, // [181] invalid_language ?? ==Zero==
316
+ 0, 0, 0, 0, // [182] invalid_language ?? ==Zero==
317
+ 0, 0, 0, 0, // [183] invalid_language ?? ==Zero==
318
+ 0, 0, 0, 0, // [184] invalid_language ?? ==Zero==
319
+ 0, 0, 0, 0, // [185] X_BORK_BORK_BORK zzb ==Zero==
320
+ 0, 0, 0, 0, // [186] X_PIG_LATIN zzp ==Zero==
321
+ 0, 0, 0, 0, // [187] X_HACKER zzh ==Zero==
322
+ 0, 0, 0, 0, // [188] X_KLINGON tlh ==Zero==
323
+ 0, 0, 0, 0, // [189] X_ELMER_FUDD zze ==Zero==
324
+ 0, 0, 0, 0, // [190] X_OGHAM xx-Ogam ==Zero==
325
+ 0, 0, 0, 0, // [191] X_RUNIC xx-Runr ==Zero==
326
+
327
+ 0, 0, 0, 0, // [192] X_YI xx-Yiii ==Zero==
328
+ 0, 0, 0, 0, // [193] X_OLD_ITALIC xx-Ital ==Zero==
329
+ 0, 0, 0, 0, // [194] X_GOTHIC xx-Goth ==Zero==
330
+ 0, 0, 0, 0, // [195] X_DESERET xx-Dsrt ==Zero==
331
+ 0, 0, 0, 0, // [196] X_HANUNOO xx-Hano ==Zero==
332
+ 0, 0, 0, 0, // [197] X_BUHID xx-Buhd ==Zero==
333
+ 0, 0, 0, 0, // [198] X_TAGBANWA xx-Tagb ==Zero==
334
+ 0, 0, 0, 0, // [199] X_TAI_LE xx-Tale ==Zero==
335
+ 0, 0, 0, 0, // [200] X_LINEAR_B xx-Linb ==Zero==
336
+ 0, 0, 0, 0, // [201] X_UGARITIC xx-Ugar ==Zero==
337
+ 0, 0, 0, 0, // [202] X_SHAVIAN xx-Shaw ==Zero==
338
+ 0, 0, 0, 0, // [203] X_OSMANYA xx-Osma ==Zero==
339
+ 0, 0, 0, 0, // [204] X_CYPRIOT xx-Cprt ==Zero==
340
+ 0, 0, 0, 0, // [205] X_BUGINESE xx-Bugi ==Zero==
341
+ 0, 0, 0, 0, // [206] X_COPTIC xx-Copt ==Zero==
342
+ 0, 0, 0, 0, // [207] X_NEW_TAI_LUE xx-Talu ==Zero==
343
+ 0, 0, 0, 0, // [208] X_GLAGOLITIC xx-Glag ==Zero==
344
+ 0, 0, 0, 0, // [209] X_TIFINAGH xx-Tfng ==Zero==
345
+ 0, 0, 0, 0, // [210] X_SYLOTI_NAGRI xx-Sylo ==Zero==
346
+ 0, 0, 0, 0, // [211] X_OLD_PERSIAN xx-Xpeo ==Zero==
347
+ 0, 0, 0, 0, // [212] X_KHAROSHTHI xx-Khar ==Zero==
348
+ 0, 0, 0, 0, // [213] X_BALINESE xx-Bali ==Zero==
349
+ 0, 0, 0, 0, // [214] X_CUNEIFORM xx-Xsux ==Zero==
350
+ 0, 0, 0, 0, // [215] X_PHOENICIAN xx-Phnx ==Zero==
351
+ 0, 0, 0, 0, // [216] X_PHAGS_PA xx-Phag ==Zero==
352
+ 0, 0, 0, 0, // [217] X_NKO xx-Nkoo ==Zero==
353
+ 0, 0, 0, 0, // [218] X_SUDANESE xx-Sund ==Zero==
354
+ 0, 0, 0, 0, // [219] X_LEPCHA xx-Lepc ==Zero==
355
+ 0, 0, 0, 0, // [220] X_OL_CHIKI xx-Olck ==Zero==
356
+ 0, 0, 0, 0, // [221] X_VAI xx-Vaii ==Zero==
357
+ 0, 0, 0, 0, // [222] X_SAURASHTRA xx-Saur ==Zero==
358
+ 0, 0, 0, 0, // [223] X_KAYAH_LI xx-Kali ==Zero==
359
+
360
+ 0, 0, 0, 0, // [224] X_REJANG xx-Rjng ==Zero==
361
+ 0, 0, 0, 0, // [225] X_LYCIAN xx-Lyci ==Zero==
362
+ 0, 0, 0, 0, // [226] X_CARIAN xx-Cari ==Zero==
363
+ 0, 0, 0, 0, // [227] X_LYDIAN xx-Lydi ==Zero==
364
+ 0, 0, 0, 0, // [228] X_CHAM xx-Cham ==Zero==
365
+ 0, 0, 0, 0, // [229] invalid_language ?? ==Zero==
366
+ 0, 0, 0, 0, // [230] invalid_language ?? ==Zero==
367
+ 0, 0, 0, 0, // [231] invalid_language ?? ==Zero==
368
+ 0, 0, 0, 0, // [232] invalid_language ?? ==Zero==
369
+ 0, 0, 0, 0, // [233] invalid_language ?? ==Zero==
370
+ 0, 0, 0, 0, // [234] invalid_language ?? ==Zero==
371
+ 0, 0, 0, 0, // [235] invalid_language ?? ==Zero==
372
+ 0, 0, 0, 0, // [236] invalid_language ?? ==Zero==
373
+ 0, 0, 0, 0, // [237] invalid_language ?? ==Zero==
374
+ 0, 0, 0, 0, // [238] invalid_language ?? ==Zero==
375
+ 0, 0, 0, 0, // [239] invalid_language ?? ==Zero==
376
+ 0, 0, 0, 0, // [240] invalid_language ?? ==Zero==
377
+ 0, 0, 0, 0, // [241] invalid_language ?? ==Zero==
378
+ 0, 0, 0, 0, // [242] invalid_language ?? ==Zero==
379
+ 0, 0, 0, 0, // [243] invalid_language ?? ==Zero==
380
+ };
381
+
382
+ COMPILE_ASSERT(EXT_NUM_LANGUAGES >= 209, k_ext_num_languages_changed);
@@ -0,0 +1,49 @@
1
+ // Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
2
+ // Use of this source code is governed by a BSD-style license that can be
3
+ // found in the LICENSE file.
4
+ //
5
+ // Created by postproc-shortwords 1.7 on 2009-01-29 16:13:04
6
+ // From input file /tmp/langdet_v25_12cjk_sort.utf8
7
+ // See compact_lang_det.cc for usage
8
+ //
9
+ #include "encodings/compact_lang_det/cldutil.h"
10
+
11
+ // Suppressed:
12
+ // az-Arab az-Cyrl ku-Latn tg-Arab za-Hani zzb-Latn zze-Latn zzh-Latn ru-Latn
13
+
14
+ // Remapped:
15
+ // xxx-Latn=>ut-Latn sh-Latn=>hr-Latn sh-Cyrl=>sr-Cyrl
16
+
17
+
18
+ static const int kCjkBiTableBuildDate = 20090129; // yyyymmdd
19
+ static const int kCjkBiTableSize = 1; // Bucket count
20
+ static const int kCjkBiTableKeyMask = 0xffffffff; // Mask hash key
21
+
22
+ COMPILE_ASSERT(MONTENEGRIN == 160, k_montenegrin_changed);
23
+ COMPILE_ASSERT(EXT_NUM_LANGUAGES == 209, k_ext_num_languages_changed);
24
+
25
+ // Empty table
26
+ static const cld::IndirectProbBucket4 kCjkBiTable[kCjkBiTableSize] = {
27
+ // key[4], words[4] in UTF-8
28
+ // value[4]
29
+ { {0x00000000,0x00000000,0x00000000,0x00000000}}, // [000] c
30
+ };
31
+
32
+ static const uint32 kCjkBiTableInd[1] = {
33
+ // [0000]
34
+ 0x00000000, };
35
+
36
+ COMPILE_ASSERT(1 < (1 << 16), k_indirectbits_too_small);
37
+
38
+
39
+ extern const cld::CLDTableSummary kCjkBiTable_obj = {
40
+ kCjkBiTable,
41
+ kCjkBiTableInd,
42
+ kCjkBiTableSize,
43
+ arraysize(kCjkBiTableInd),
44
+ kCjkBiTableKeyMask,
45
+ kCjkBiTableBuildDate,
46
+ };
47
+
48
+ // End of generated tables
49
+