constile_gate 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +226 -0
- data/constile_gate.gemspec +21 -0
- data/lib/constile_gate/agents.rb +50 -0
- data/lib/constile_gate/charset.rb +427 -0
- data/lib/constile_gate/charset_tables.rb +284 -0
- data/lib/constile_gate/gate.rb +612 -0
- data/lib/constile_gate/http.rb +235 -0
- data/lib/constile_gate/js.rb +99 -0
- data/lib/constile_gate/page_meta.rb +83 -0
- data/lib/constile_gate/rack.rb +371 -0
- data/lib/constile_gate/railtie.rb +17 -0
- data/lib/constile_gate/url.rb +208 -0
- data/lib/constile_gate/version.rb +5 -0
- data/lib/constile_gate.rb +6 -0
- metadata +62 -0
|
@@ -0,0 +1,427 @@
|
|
|
1
|
+
require "constile_gate/js"
|
|
2
|
+
require "constile_gate/page_meta"
|
|
3
|
+
require "constile_gate/charset_tables"
|
|
4
|
+
|
|
5
|
+
module ConstileGate
|
|
6
|
+
# Reads a page in the charset its Content-Type names, as the reference's decodePage
|
|
7
|
+
# does with TextDecoder: the WHATWG Encoding Standard's labels and decoders, and
|
|
8
|
+
# UTF-8 when the page names no charset or one TextDecoder refuses.
|
|
9
|
+
module Charset
|
|
10
|
+
# The reference's /charset\s*=\s*"?([\w.:-]+)/i.
|
|
11
|
+
PARAM = /#{PageMeta.ci("charset")}[#{JS::WS}]*=[#{JS::WS}]*"?([A-Za-z0-9_.:\-]+)/.freeze
|
|
12
|
+
|
|
13
|
+
# Every label in the Encoding Standard, by the encoding it names.
|
|
14
|
+
LABELS = {
|
|
15
|
+
"utf-8" => %w[unicode-1-1-utf-8 unicode11utf8 unicode20utf8 utf8 utf-8 x-unicode20utf8],
|
|
16
|
+
"ibm866" => %w[866 cp866 csibm866 ibm866],
|
|
17
|
+
"iso-8859-2" => %w[csisolatin2 iso-8859-2 iso-ir-101 iso8859-2 iso88592 iso_8859-2 iso_8859-2:1987 l2 latin2],
|
|
18
|
+
"iso-8859-3" => %w[csisolatin3 iso-8859-3 iso-ir-109 iso8859-3 iso88593 iso_8859-3 iso_8859-3:1988 l3 latin3],
|
|
19
|
+
"iso-8859-4" => %w[csisolatin4 iso-8859-4 iso-ir-110 iso8859-4 iso88594 iso_8859-4 iso_8859-4:1988 l4 latin4],
|
|
20
|
+
"iso-8859-5" => %w[csisolatincyrillic cyrillic iso-8859-5 iso-ir-144 iso8859-5 iso88595 iso_8859-5 iso_8859-5:1988],
|
|
21
|
+
"iso-8859-6" => %w[
|
|
22
|
+
arabic asmo-708 csiso88596e csiso88596i csisolatinarabic ecma-114 iso-8859-6 iso-8859-6-e
|
|
23
|
+
iso-8859-6-i iso-ir-127 iso8859-6 iso88596 iso_8859-6 iso_8859-6:1987
|
|
24
|
+
],
|
|
25
|
+
"iso-8859-7" => %w[
|
|
26
|
+
csisolatingreek ecma-118 elot_928 greek greek8 iso-8859-7 iso-ir-126 iso8859-7 iso88597
|
|
27
|
+
iso_8859-7 iso_8859-7:1987 sun_eu_greek
|
|
28
|
+
],
|
|
29
|
+
"iso-8859-8" => %w[
|
|
30
|
+
csiso88598e csisolatinhebrew hebrew iso-8859-8 iso-8859-8-e iso-ir-138 iso8859-8 iso88598
|
|
31
|
+
iso_8859-8 iso_8859-8:1988 visual
|
|
32
|
+
],
|
|
33
|
+
"iso-8859-8-i" => %w[csiso88598i iso-8859-8-i logical],
|
|
34
|
+
"iso-8859-10" => %w[csisolatin6 iso-8859-10 iso-ir-157 iso8859-10 iso885910 l6 latin6],
|
|
35
|
+
"iso-8859-13" => %w[iso-8859-13 iso8859-13 iso885913],
|
|
36
|
+
"iso-8859-14" => %w[iso-8859-14 iso8859-14 iso885914],
|
|
37
|
+
"iso-8859-15" => %w[csisolatin9 iso-8859-15 iso8859-15 iso885915 iso_8859-15 l9],
|
|
38
|
+
"iso-8859-16" => %w[iso-8859-16],
|
|
39
|
+
"koi8-r" => %w[cskoi8r koi koi8 koi8-r koi8_r],
|
|
40
|
+
"koi8-u" => %w[koi8-ru koi8-u],
|
|
41
|
+
"macintosh" => %w[csmacintosh mac macintosh x-mac-roman],
|
|
42
|
+
"windows-874" => %w[dos-874 iso-8859-11 iso8859-11 iso885911 tis-620 windows-874],
|
|
43
|
+
"windows-1250" => %w[cp1250 windows-1250 x-cp1250],
|
|
44
|
+
"windows-1251" => %w[cp1251 windows-1251 x-cp1251],
|
|
45
|
+
"windows-1252" => %w[
|
|
46
|
+
ansi_x3.4-1968 ascii cp1252 cp819 csisolatin1 ibm819 iso-8859-1 iso-ir-100 iso8859-1 iso88591
|
|
47
|
+
iso_8859-1 iso_8859-1:1987 l1 latin1 us-ascii windows-1252 x-cp1252
|
|
48
|
+
],
|
|
49
|
+
"windows-1253" => %w[cp1253 windows-1253 x-cp1253],
|
|
50
|
+
"windows-1254" => %w[
|
|
51
|
+
cp1254 csisolatin5 iso-8859-9 iso-ir-148 iso8859-9 iso88599 iso_8859-9 iso_8859-9:1989 l5 latin5
|
|
52
|
+
windows-1254 x-cp1254
|
|
53
|
+
],
|
|
54
|
+
"windows-1255" => %w[cp1255 windows-1255 x-cp1255],
|
|
55
|
+
"windows-1256" => %w[cp1256 windows-1256 x-cp1256],
|
|
56
|
+
"windows-1257" => %w[cp1257 windows-1257 x-cp1257],
|
|
57
|
+
"windows-1258" => %w[cp1258 windows-1258 x-cp1258],
|
|
58
|
+
"x-mac-cyrillic" => %w[x-mac-cyrillic x-mac-ukrainian],
|
|
59
|
+
"gbk" => %w[chinese csgb2312 csiso58gb231280 gb2312 gb_2312 gb_2312-80 gbk iso-ir-58 x-gbk],
|
|
60
|
+
"gb18030" => %w[gb18030],
|
|
61
|
+
"big5" => %w[big5 big5-hkscs cn-big5 csbig5 x-x-big5],
|
|
62
|
+
"euc-jp" => %w[cseucpkdfmtjapanese euc-jp x-euc-jp],
|
|
63
|
+
"iso-2022-jp" => %w[csiso2022jp iso-2022-jp],
|
|
64
|
+
"shift_jis" => %w[csshiftjis ms932 ms_kanji shift-jis shift_jis sjis windows-31j x-sjis],
|
|
65
|
+
"euc-kr" => %w[
|
|
66
|
+
cseuckr csksc56011987 euc-kr iso-ir-149 korean ks_c_5601-1987 ks_c_5601-1989 ksc5601 ksc_5601
|
|
67
|
+
windows-949
|
|
68
|
+
],
|
|
69
|
+
"replacement" => %w[csiso2022kr hz-gb-2312 iso-2022-cn iso-2022-cn-ext iso-2022-kr replacement],
|
|
70
|
+
"utf-16be" => %w[unicodefffe utf-16be],
|
|
71
|
+
"utf-16le" => %w[csunicode iso-10646-ucs-2 ucs-2 unicode unicodefeff utf-16le utf-16],
|
|
72
|
+
"x-user-defined" => %w[x-user-defined],
|
|
73
|
+
}.freeze
|
|
74
|
+
ENCODING_BY_LABEL = LABELS.each_with_object({}) { |(name, labels), all| labels.each { |l| all[l] = name } }.freeze
|
|
75
|
+
|
|
76
|
+
# The Ruby encodings whose converters serve as the WHATWG indexes for the multibyte
|
|
77
|
+
# decoders below, corrected by OVERRIDES (charset_tables.rb) where workerd differs.
|
|
78
|
+
INDEX_ENCODINGS = {
|
|
79
|
+
jis0208: "EUC-JP", jis0212: "EUC-JP", shift_jis: "Windows-31J", euc_kr: "CP949", big5: "Big5-HKSCS", gb18030: "GB18030",
|
|
80
|
+
}.freeze
|
|
81
|
+
# Big5 pointers that decode to two code points.
|
|
82
|
+
BIG5_PAIRS = { 1133 => [0xCA, 0x304], 1135 => [0xCA, 0x30C], 1164 => [0xEA, 0x304], 1166 => [0xEA, 0x30C] }.freeze
|
|
83
|
+
REPLACEMENT = "\uFFFD".freeze
|
|
84
|
+
|
|
85
|
+
module_function
|
|
86
|
+
|
|
87
|
+
# The page's bytes as a UTF-8 String.
|
|
88
|
+
def decode(bytes, content_type)
|
|
89
|
+
m = PARAM.match(content_type.to_s)
|
|
90
|
+
name = m && ENCODING_BY_LABEL[m[1].downcase]
|
|
91
|
+
bytes = bytes.b
|
|
92
|
+
case name
|
|
93
|
+
when nil, "replacement", "utf-8" then utf8(bytes)
|
|
94
|
+
when "utf-16le" then utf16(bytes, "v")
|
|
95
|
+
when "utf-16be" then utf16(bytes, "n")
|
|
96
|
+
when "x-user-defined" then user_defined(bytes)
|
|
97
|
+
when "shift_jis" then shift_jis(bytes)
|
|
98
|
+
when "euc-jp" then euc_jp(bytes)
|
|
99
|
+
when "iso-2022-jp" then iso_2022_jp(bytes)
|
|
100
|
+
when "euc-kr" then euc_kr(bytes)
|
|
101
|
+
when "big5" then big5(bytes)
|
|
102
|
+
when "gbk", "gb18030" then gb18030(bytes)
|
|
103
|
+
else single_byte(bytes, SINGLE_BYTE.fetch(name))
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def utf8(bytes)
|
|
108
|
+
bytes = bytes.byteslice(3..-1) if bytes.start_with?("\xEF\xBB\xBF".b)
|
|
109
|
+
bytes.force_encoding(Encoding::UTF_8).scrub(REPLACEMENT)
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# The UTF-16 decoder: a byte order mark of the same order is dropped, and a lone
|
|
113
|
+
# surrogate or an odd last byte becomes U+FFFD.
|
|
114
|
+
def utf16(bytes, unit)
|
|
115
|
+
units = bytes.byteslice(0, bytes.bytesize - (bytes.bytesize % 2)).unpack("#{unit}*")
|
|
116
|
+
units.shift if units.first == 0xFEFF
|
|
117
|
+
out = String.new(encoding: Encoding::UTF_8)
|
|
118
|
+
high = nil
|
|
119
|
+
units.each do |u|
|
|
120
|
+
if high
|
|
121
|
+
if u.between?(0xDC00, 0xDFFF)
|
|
122
|
+
out << (0x10000 + ((high - 0xD800) << 10) + (u - 0xDC00)).chr(Encoding::UTF_8)
|
|
123
|
+
high = nil
|
|
124
|
+
next
|
|
125
|
+
end
|
|
126
|
+
out << REPLACEMENT
|
|
127
|
+
high = nil
|
|
128
|
+
end
|
|
129
|
+
if u.between?(0xD800, 0xDBFF)
|
|
130
|
+
high = u
|
|
131
|
+
elsif u.between?(0xDC00, 0xDFFF)
|
|
132
|
+
out << REPLACEMENT
|
|
133
|
+
else
|
|
134
|
+
out << u.chr(Encoding::UTF_8)
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
out << REPLACEMENT if high
|
|
138
|
+
out << REPLACEMENT if bytes.bytesize.odd?
|
|
139
|
+
out
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
def single_byte(bytes, table)
|
|
143
|
+
out = String.new(encoding: Encoding::UTF_8)
|
|
144
|
+
bytes.each_byte { |b| out << (b < 0x80 ? b.chr : table[b - 0x80]) }
|
|
145
|
+
out
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
# x-user-defined: ASCII, and each byte from 0x80 up as U+F780 onward.
|
|
149
|
+
def user_defined(bytes)
|
|
150
|
+
out = String.new(encoding: Encoding::UTF_8)
|
|
151
|
+
bytes.each_byte { |b| out << (b < 0x80 ? b.chr : (0xF780 + b - 0x80).chr(Encoding::UTF_8)) }
|
|
152
|
+
out
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# The character an index gives for a complete code (its bytes), or nil when it has none.
|
|
156
|
+
def lookup(index, code)
|
|
157
|
+
key = code.pack("C*")
|
|
158
|
+
overrides = OVERRIDES[index]
|
|
159
|
+
if overrides
|
|
160
|
+
hex = key.unpack1("H*")
|
|
161
|
+
return overrides[hex] if overrides.key?(hex)
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
memo = MEMO[index]
|
|
165
|
+
return memo[key] if memo.key?(key)
|
|
166
|
+
|
|
167
|
+
s = key.dup.force_encoding(INDEX_ENCODINGS.fetch(index)).encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: REPLACEMENT)
|
|
168
|
+
memo[key] = s.length == 1 && s != REPLACEMENT ? s.freeze : nil
|
|
169
|
+
end
|
|
170
|
+
MEMO = Hash.new { |h, index| h[index] = {} }
|
|
171
|
+
|
|
172
|
+
# The bytes of a page, read one at a time, with bytes put back in front.
|
|
173
|
+
class Queue
|
|
174
|
+
def initialize(bytes)
|
|
175
|
+
@bytes = bytes
|
|
176
|
+
@i = 0
|
|
177
|
+
@front = []
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
# The next byte, or nil at the end.
|
|
181
|
+
def read
|
|
182
|
+
return @front.shift unless @front.empty?
|
|
183
|
+
|
|
184
|
+
b = @bytes.getbyte(@i)
|
|
185
|
+
@i += 1 if b
|
|
186
|
+
b
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
def prepend(*bytes)
|
|
190
|
+
@front = bytes.compact + @front
|
|
191
|
+
end
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
def char(code_point)
|
|
195
|
+
code_point.chr(Encoding::UTF_8)
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
# The Encoding Standard's Shift_JIS decoder.
|
|
199
|
+
def shift_jis(bytes)
|
|
200
|
+
q = Queue.new(bytes)
|
|
201
|
+
out = String.new(encoding: Encoding::UTF_8)
|
|
202
|
+
lead = 0
|
|
203
|
+
while (b = q.read)
|
|
204
|
+
if lead != 0
|
|
205
|
+
l = lead
|
|
206
|
+
lead = 0
|
|
207
|
+
cp = nil
|
|
208
|
+
if b.between?(0x40, 0x7E) || b.between?(0x80, 0xFC)
|
|
209
|
+
pointer = (l - (l < 0xA0 ? 0x81 : 0xC1)) * 188 + b - (b < 0x7F ? 0x40 : 0x41)
|
|
210
|
+
cp = pointer.between?(8836, 10_715) ? char(0xE000 - 8836 + pointer) : lookup(:shift_jis, [l, b])
|
|
211
|
+
end
|
|
212
|
+
next out << cp if cp
|
|
213
|
+
|
|
214
|
+
q.prepend(b) if b < 0x80
|
|
215
|
+
out << REPLACEMENT
|
|
216
|
+
elsif b <= 0x80 then out << char(b)
|
|
217
|
+
elsif b.between?(0xA1, 0xDF) then out << char(0xFF61 - 0xA1 + b)
|
|
218
|
+
elsif b.between?(0x81, 0x9F) || b.between?(0xE0, 0xFC) then lead = b
|
|
219
|
+
else out << REPLACEMENT
|
|
220
|
+
end
|
|
221
|
+
end
|
|
222
|
+
out << REPLACEMENT if lead != 0
|
|
223
|
+
out
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
# The Encoding Standard's EUC-JP decoder.
|
|
227
|
+
def euc_jp(bytes)
|
|
228
|
+
q = Queue.new(bytes)
|
|
229
|
+
out = String.new(encoding: Encoding::UTF_8)
|
|
230
|
+
lead = 0
|
|
231
|
+
jis0212 = false
|
|
232
|
+
while (b = q.read)
|
|
233
|
+
if lead == 0x8E && b.between?(0xA1, 0xDF)
|
|
234
|
+
lead = 0
|
|
235
|
+
out << char(0xFF61 - 0xA1 + b)
|
|
236
|
+
elsif lead == 0x8F && b.between?(0xA1, 0xFE)
|
|
237
|
+
jis0212 = true
|
|
238
|
+
lead = b
|
|
239
|
+
elsif lead != 0
|
|
240
|
+
l = lead
|
|
241
|
+
lead = 0
|
|
242
|
+
cp = nil
|
|
243
|
+
cp = jis0212 ? lookup(:jis0212, [0x8F, l, b]) : lookup(:jis0208, [l, b]) if l.between?(0xA1, 0xFE) && b.between?(0xA1, 0xFE)
|
|
244
|
+
jis0212 = false
|
|
245
|
+
next out << cp if cp
|
|
246
|
+
|
|
247
|
+
q.prepend(b) if b < 0x80
|
|
248
|
+
out << REPLACEMENT
|
|
249
|
+
elsif b < 0x80 then out << char(b)
|
|
250
|
+
elsif b == 0x8E || b == 0x8F || b.between?(0xA1, 0xFE) then lead = b
|
|
251
|
+
else out << REPLACEMENT
|
|
252
|
+
end
|
|
253
|
+
end
|
|
254
|
+
out << REPLACEMENT if lead != 0
|
|
255
|
+
out
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
# The Encoding Standard's ISO-2022-JP decoder.
|
|
259
|
+
def iso_2022_jp(bytes)
|
|
260
|
+
q = Queue.new(bytes)
|
|
261
|
+
out = String.new(encoding: Encoding::UTF_8)
|
|
262
|
+
state = output_state = :ascii
|
|
263
|
+
lead = 0
|
|
264
|
+
output_flag = false
|
|
265
|
+
loop do
|
|
266
|
+
b = q.read
|
|
267
|
+
case state
|
|
268
|
+
when :ascii, :roman, :katakana, :lead
|
|
269
|
+
return out if b.nil?
|
|
270
|
+
|
|
271
|
+
if b == 0x1B
|
|
272
|
+
state = :escape_start
|
|
273
|
+
next
|
|
274
|
+
end
|
|
275
|
+
output_flag = false
|
|
276
|
+
case state
|
|
277
|
+
when :ascii
|
|
278
|
+
out << (b <= 0x7F && b != 0x0E && b != 0x0F ? char(b) : REPLACEMENT)
|
|
279
|
+
when :roman
|
|
280
|
+
out << (b == 0x5C ? char(0xA5) : b == 0x7E ? char(0x203E) : b <= 0x7F && b != 0x0E && b != 0x0F ? char(b) : REPLACEMENT)
|
|
281
|
+
when :katakana
|
|
282
|
+
out << (b.between?(0x21, 0x5F) ? char(0xFF61 - 0x21 + b) : REPLACEMENT)
|
|
283
|
+
when :lead
|
|
284
|
+
if b.between?(0x21, 0x7E)
|
|
285
|
+
lead = b
|
|
286
|
+
state = :trail
|
|
287
|
+
else
|
|
288
|
+
out << REPLACEMENT
|
|
289
|
+
end
|
|
290
|
+
end
|
|
291
|
+
when :trail
|
|
292
|
+
if b == 0x1B
|
|
293
|
+
state = :escape_start
|
|
294
|
+
out << REPLACEMENT
|
|
295
|
+
elsif b && b.between?(0x21, 0x7E)
|
|
296
|
+
state = :lead
|
|
297
|
+
out << (lookup(:jis0208, [lead | 0x80, b | 0x80]) || REPLACEMENT)
|
|
298
|
+
else
|
|
299
|
+
state = :lead
|
|
300
|
+
out << REPLACEMENT
|
|
301
|
+
end
|
|
302
|
+
when :escape_start
|
|
303
|
+
if b == 0x24 || b == 0x28
|
|
304
|
+
lead = b
|
|
305
|
+
state = :escape
|
|
306
|
+
else
|
|
307
|
+
q.prepend(b)
|
|
308
|
+
output_flag = false
|
|
309
|
+
state = output_state
|
|
310
|
+
out << REPLACEMENT
|
|
311
|
+
end
|
|
312
|
+
when :escape
|
|
313
|
+
l = lead
|
|
314
|
+
lead = 0
|
|
315
|
+
found = if l == 0x28 && b == 0x42 then :ascii
|
|
316
|
+
elsif l == 0x28 && b == 0x4A then :roman
|
|
317
|
+
elsif l == 0x28 && b == 0x49 then :katakana
|
|
318
|
+
elsif l == 0x24 && (b == 0x40 || b == 0x42) then :lead
|
|
319
|
+
end
|
|
320
|
+
if found
|
|
321
|
+
state = output_state = found
|
|
322
|
+
out << REPLACEMENT if output_flag
|
|
323
|
+
output_flag = true
|
|
324
|
+
else
|
|
325
|
+
q.prepend(l, b)
|
|
326
|
+
output_flag = false
|
|
327
|
+
state = output_state
|
|
328
|
+
out << REPLACEMENT
|
|
329
|
+
end
|
|
330
|
+
end
|
|
331
|
+
end
|
|
332
|
+
end
|
|
333
|
+
|
|
334
|
+
# The Encoding Standard's EUC-KR decoder.
|
|
335
|
+
def euc_kr(bytes)
|
|
336
|
+
q = Queue.new(bytes)
|
|
337
|
+
out = String.new(encoding: Encoding::UTF_8)
|
|
338
|
+
lead = 0
|
|
339
|
+
while (b = q.read)
|
|
340
|
+
if lead != 0
|
|
341
|
+
l = lead
|
|
342
|
+
lead = 0
|
|
343
|
+
cp = b.between?(0x41, 0xFE) ? lookup(:euc_kr, [l, b]) : nil
|
|
344
|
+
next out << cp if cp
|
|
345
|
+
|
|
346
|
+
q.prepend(b) if b < 0x80
|
|
347
|
+
out << REPLACEMENT
|
|
348
|
+
elsif b < 0x80 then out << char(b)
|
|
349
|
+
elsif b.between?(0x81, 0xFE) then lead = b
|
|
350
|
+
else out << REPLACEMENT
|
|
351
|
+
end
|
|
352
|
+
end
|
|
353
|
+
out << REPLACEMENT if lead != 0
|
|
354
|
+
out
|
|
355
|
+
end
|
|
356
|
+
|
|
357
|
+
# The Encoding Standard's Big5 decoder.
|
|
358
|
+
def big5(bytes)
|
|
359
|
+
q = Queue.new(bytes)
|
|
360
|
+
out = String.new(encoding: Encoding::UTF_8)
|
|
361
|
+
lead = 0
|
|
362
|
+
while (b = q.read)
|
|
363
|
+
if lead != 0
|
|
364
|
+
l = lead
|
|
365
|
+
lead = 0
|
|
366
|
+
cp = nil
|
|
367
|
+
if b.between?(0x40, 0x7E) || b.between?(0xA1, 0xFE)
|
|
368
|
+
pointer = (l - 0x81) * 157 + b - (b < 0x7F ? 0x40 : 0x62)
|
|
369
|
+
next BIG5_PAIRS[pointer].each { |c| out << char(c) } if BIG5_PAIRS.key?(pointer)
|
|
370
|
+
|
|
371
|
+
cp = lookup(:big5, [l, b])
|
|
372
|
+
end
|
|
373
|
+
next out << cp if cp
|
|
374
|
+
|
|
375
|
+
q.prepend(b) if b < 0x80
|
|
376
|
+
out << REPLACEMENT
|
|
377
|
+
elsif b < 0x80 then out << char(b)
|
|
378
|
+
elsif b.between?(0x81, 0xFE) then lead = b
|
|
379
|
+
else out << REPLACEMENT
|
|
380
|
+
end
|
|
381
|
+
end
|
|
382
|
+
out << REPLACEMENT if lead != 0
|
|
383
|
+
out
|
|
384
|
+
end
|
|
385
|
+
|
|
386
|
+
# The Encoding Standard's gb18030 decoder, which also decodes GBK.
|
|
387
|
+
def gb18030(bytes)
|
|
388
|
+
q = Queue.new(bytes)
|
|
389
|
+
out = String.new(encoding: Encoding::UTF_8)
|
|
390
|
+
first = second = third = 0
|
|
391
|
+
while (b = q.read)
|
|
392
|
+
if third != 0
|
|
393
|
+
unless b.between?(0x30, 0x39)
|
|
394
|
+
q.prepend(second, third, b)
|
|
395
|
+
first = second = third = 0
|
|
396
|
+
next out << REPLACEMENT
|
|
397
|
+
end
|
|
398
|
+
cp = lookup(:gb18030, [first, second, third, b])
|
|
399
|
+
first = second = third = 0
|
|
400
|
+
out << (cp || REPLACEMENT)
|
|
401
|
+
elsif second != 0
|
|
402
|
+
next third = b if b.between?(0x81, 0xFE)
|
|
403
|
+
|
|
404
|
+
q.prepend(second, b)
|
|
405
|
+
first = second = 0
|
|
406
|
+
out << REPLACEMENT
|
|
407
|
+
elsif first != 0
|
|
408
|
+
next second = b if b.between?(0x30, 0x39)
|
|
409
|
+
|
|
410
|
+
l = first
|
|
411
|
+
first = 0
|
|
412
|
+
cp = b.between?(0x40, 0x7E) || b.between?(0x80, 0xFE) ? lookup(:gb18030, [l, b]) : nil
|
|
413
|
+
next out << cp if cp
|
|
414
|
+
|
|
415
|
+
q.prepend(b) if b < 0x80
|
|
416
|
+
out << REPLACEMENT
|
|
417
|
+
elsif b < 0x80 then out << char(b)
|
|
418
|
+
elsif b == 0x80 then out << char(0x20AC)
|
|
419
|
+
elsif b.between?(0x81, 0xFE) then first = b
|
|
420
|
+
else out << REPLACEMENT
|
|
421
|
+
end
|
|
422
|
+
end
|
|
423
|
+
out << REPLACEMENT if first != 0 || second != 0 || third != 0
|
|
424
|
+
out
|
|
425
|
+
end
|
|
426
|
+
end
|
|
427
|
+
end
|