constile_gate 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,427 @@
1
+ require "constile_gate/js"
2
+ require "constile_gate/page_meta"
3
+ require "constile_gate/charset_tables"
4
+
5
+ module ConstileGate
6
+ # Reads a page in the charset its Content-Type names, as the reference's decodePage
7
+ # does with TextDecoder: the WHATWG Encoding Standard's labels and decoders, and
8
+ # UTF-8 when the page names no charset or one TextDecoder refuses.
9
+ module Charset
10
+ # The reference's /charset\s*=\s*"?([\w.:-]+)/i.
11
+ PARAM = /#{PageMeta.ci("charset")}[#{JS::WS}]*=[#{JS::WS}]*"?([A-Za-z0-9_.:\-]+)/.freeze
12
+
13
+ # Every label in the Encoding Standard, by the encoding it names.
14
+ LABELS = {
15
+ "utf-8" => %w[unicode-1-1-utf-8 unicode11utf8 unicode20utf8 utf8 utf-8 x-unicode20utf8],
16
+ "ibm866" => %w[866 cp866 csibm866 ibm866],
17
+ "iso-8859-2" => %w[csisolatin2 iso-8859-2 iso-ir-101 iso8859-2 iso88592 iso_8859-2 iso_8859-2:1987 l2 latin2],
18
+ "iso-8859-3" => %w[csisolatin3 iso-8859-3 iso-ir-109 iso8859-3 iso88593 iso_8859-3 iso_8859-3:1988 l3 latin3],
19
+ "iso-8859-4" => %w[csisolatin4 iso-8859-4 iso-ir-110 iso8859-4 iso88594 iso_8859-4 iso_8859-4:1988 l4 latin4],
20
+ "iso-8859-5" => %w[csisolatincyrillic cyrillic iso-8859-5 iso-ir-144 iso8859-5 iso88595 iso_8859-5 iso_8859-5:1988],
21
+ "iso-8859-6" => %w[
22
+ arabic asmo-708 csiso88596e csiso88596i csisolatinarabic ecma-114 iso-8859-6 iso-8859-6-e
23
+ iso-8859-6-i iso-ir-127 iso8859-6 iso88596 iso_8859-6 iso_8859-6:1987
24
+ ],
25
+ "iso-8859-7" => %w[
26
+ csisolatingreek ecma-118 elot_928 greek greek8 iso-8859-7 iso-ir-126 iso8859-7 iso88597
27
+ iso_8859-7 iso_8859-7:1987 sun_eu_greek
28
+ ],
29
+ "iso-8859-8" => %w[
30
+ csiso88598e csisolatinhebrew hebrew iso-8859-8 iso-8859-8-e iso-ir-138 iso8859-8 iso88598
31
+ iso_8859-8 iso_8859-8:1988 visual
32
+ ],
33
+ "iso-8859-8-i" => %w[csiso88598i iso-8859-8-i logical],
34
+ "iso-8859-10" => %w[csisolatin6 iso-8859-10 iso-ir-157 iso8859-10 iso885910 l6 latin6],
35
+ "iso-8859-13" => %w[iso-8859-13 iso8859-13 iso885913],
36
+ "iso-8859-14" => %w[iso-8859-14 iso8859-14 iso885914],
37
+ "iso-8859-15" => %w[csisolatin9 iso-8859-15 iso8859-15 iso885915 iso_8859-15 l9],
38
+ "iso-8859-16" => %w[iso-8859-16],
39
+ "koi8-r" => %w[cskoi8r koi koi8 koi8-r koi8_r],
40
+ "koi8-u" => %w[koi8-ru koi8-u],
41
+ "macintosh" => %w[csmacintosh mac macintosh x-mac-roman],
42
+ "windows-874" => %w[dos-874 iso-8859-11 iso8859-11 iso885911 tis-620 windows-874],
43
+ "windows-1250" => %w[cp1250 windows-1250 x-cp1250],
44
+ "windows-1251" => %w[cp1251 windows-1251 x-cp1251],
45
+ "windows-1252" => %w[
46
+ ansi_x3.4-1968 ascii cp1252 cp819 csisolatin1 ibm819 iso-8859-1 iso-ir-100 iso8859-1 iso88591
47
+ iso_8859-1 iso_8859-1:1987 l1 latin1 us-ascii windows-1252 x-cp1252
48
+ ],
49
+ "windows-1253" => %w[cp1253 windows-1253 x-cp1253],
50
+ "windows-1254" => %w[
51
+ cp1254 csisolatin5 iso-8859-9 iso-ir-148 iso8859-9 iso88599 iso_8859-9 iso_8859-9:1989 l5 latin5
52
+ windows-1254 x-cp1254
53
+ ],
54
+ "windows-1255" => %w[cp1255 windows-1255 x-cp1255],
55
+ "windows-1256" => %w[cp1256 windows-1256 x-cp1256],
56
+ "windows-1257" => %w[cp1257 windows-1257 x-cp1257],
57
+ "windows-1258" => %w[cp1258 windows-1258 x-cp1258],
58
+ "x-mac-cyrillic" => %w[x-mac-cyrillic x-mac-ukrainian],
59
+ "gbk" => %w[chinese csgb2312 csiso58gb231280 gb2312 gb_2312 gb_2312-80 gbk iso-ir-58 x-gbk],
60
+ "gb18030" => %w[gb18030],
61
+ "big5" => %w[big5 big5-hkscs cn-big5 csbig5 x-x-big5],
62
+ "euc-jp" => %w[cseucpkdfmtjapanese euc-jp x-euc-jp],
63
+ "iso-2022-jp" => %w[csiso2022jp iso-2022-jp],
64
+ "shift_jis" => %w[csshiftjis ms932 ms_kanji shift-jis shift_jis sjis windows-31j x-sjis],
65
+ "euc-kr" => %w[
66
+ cseuckr csksc56011987 euc-kr iso-ir-149 korean ks_c_5601-1987 ks_c_5601-1989 ksc5601 ksc_5601
67
+ windows-949
68
+ ],
69
+ "replacement" => %w[csiso2022kr hz-gb-2312 iso-2022-cn iso-2022-cn-ext iso-2022-kr replacement],
70
+ "utf-16be" => %w[unicodefffe utf-16be],
71
+ "utf-16le" => %w[csunicode iso-10646-ucs-2 ucs-2 unicode unicodefeff utf-16le utf-16],
72
+ "x-user-defined" => %w[x-user-defined],
73
+ }.freeze
74
+ ENCODING_BY_LABEL = LABELS.each_with_object({}) { |(name, labels), all| labels.each { |l| all[l] = name } }.freeze
75
+
76
+ # The Ruby encodings whose converters serve as the WHATWG indexes for the multibyte
77
+ # decoders below, corrected by OVERRIDES (charset_tables.rb) where workerd differs.
78
+ INDEX_ENCODINGS = {
79
+ jis0208: "EUC-JP", jis0212: "EUC-JP", shift_jis: "Windows-31J", euc_kr: "CP949", big5: "Big5-HKSCS", gb18030: "GB18030",
80
+ }.freeze
81
+ # Big5 pointers that decode to two code points.
82
+ BIG5_PAIRS = { 1133 => [0xCA, 0x304], 1135 => [0xCA, 0x30C], 1164 => [0xEA, 0x304], 1166 => [0xEA, 0x30C] }.freeze
83
+ REPLACEMENT = "\uFFFD".freeze
84
+
85
+ module_function
86
+
87
+ # The page's bytes as a UTF-8 String.
88
+ def decode(bytes, content_type)
89
+ m = PARAM.match(content_type.to_s)
90
+ name = m && ENCODING_BY_LABEL[m[1].downcase]
91
+ bytes = bytes.b
92
+ case name
93
+ when nil, "replacement", "utf-8" then utf8(bytes)
94
+ when "utf-16le" then utf16(bytes, "v")
95
+ when "utf-16be" then utf16(bytes, "n")
96
+ when "x-user-defined" then user_defined(bytes)
97
+ when "shift_jis" then shift_jis(bytes)
98
+ when "euc-jp" then euc_jp(bytes)
99
+ when "iso-2022-jp" then iso_2022_jp(bytes)
100
+ when "euc-kr" then euc_kr(bytes)
101
+ when "big5" then big5(bytes)
102
+ when "gbk", "gb18030" then gb18030(bytes)
103
+ else single_byte(bytes, SINGLE_BYTE.fetch(name))
104
+ end
105
+ end
106
+
107
+ def utf8(bytes)
108
+ bytes = bytes.byteslice(3..-1) if bytes.start_with?("\xEF\xBB\xBF".b)
109
+ bytes.force_encoding(Encoding::UTF_8).scrub(REPLACEMENT)
110
+ end
111
+
112
+ # The UTF-16 decoder: a byte order mark of the same order is dropped, and a lone
113
+ # surrogate or an odd last byte becomes U+FFFD.
114
+ def utf16(bytes, unit)
115
+ units = bytes.byteslice(0, bytes.bytesize - (bytes.bytesize % 2)).unpack("#{unit}*")
116
+ units.shift if units.first == 0xFEFF
117
+ out = String.new(encoding: Encoding::UTF_8)
118
+ high = nil
119
+ units.each do |u|
120
+ if high
121
+ if u.between?(0xDC00, 0xDFFF)
122
+ out << (0x10000 + ((high - 0xD800) << 10) + (u - 0xDC00)).chr(Encoding::UTF_8)
123
+ high = nil
124
+ next
125
+ end
126
+ out << REPLACEMENT
127
+ high = nil
128
+ end
129
+ if u.between?(0xD800, 0xDBFF)
130
+ high = u
131
+ elsif u.between?(0xDC00, 0xDFFF)
132
+ out << REPLACEMENT
133
+ else
134
+ out << u.chr(Encoding::UTF_8)
135
+ end
136
+ end
137
+ out << REPLACEMENT if high
138
+ out << REPLACEMENT if bytes.bytesize.odd?
139
+ out
140
+ end
141
+
142
+ def single_byte(bytes, table)
143
+ out = String.new(encoding: Encoding::UTF_8)
144
+ bytes.each_byte { |b| out << (b < 0x80 ? b.chr : table[b - 0x80]) }
145
+ out
146
+ end
147
+
148
+ # x-user-defined: ASCII, and each byte from 0x80 up as U+F780 onward.
149
+ def user_defined(bytes)
150
+ out = String.new(encoding: Encoding::UTF_8)
151
+ bytes.each_byte { |b| out << (b < 0x80 ? b.chr : (0xF780 + b - 0x80).chr(Encoding::UTF_8)) }
152
+ out
153
+ end
154
+
155
+ # The character an index gives for a complete code (its bytes), or nil when it has none.
156
+ def lookup(index, code)
157
+ key = code.pack("C*")
158
+ overrides = OVERRIDES[index]
159
+ if overrides
160
+ hex = key.unpack1("H*")
161
+ return overrides[hex] if overrides.key?(hex)
162
+ end
163
+
164
+ memo = MEMO[index]
165
+ return memo[key] if memo.key?(key)
166
+
167
+ s = key.dup.force_encoding(INDEX_ENCODINGS.fetch(index)).encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: REPLACEMENT)
168
+ memo[key] = s.length == 1 && s != REPLACEMENT ? s.freeze : nil
169
+ end
170
+ MEMO = Hash.new { |h, index| h[index] = {} }
171
+
172
+ # The bytes of a page, read one at a time, with bytes put back in front.
173
+ class Queue
174
+ def initialize(bytes)
175
+ @bytes = bytes
176
+ @i = 0
177
+ @front = []
178
+ end
179
+
180
+ # The next byte, or nil at the end.
181
+ def read
182
+ return @front.shift unless @front.empty?
183
+
184
+ b = @bytes.getbyte(@i)
185
+ @i += 1 if b
186
+ b
187
+ end
188
+
189
+ def prepend(*bytes)
190
+ @front = bytes.compact + @front
191
+ end
192
+ end
193
+
194
+ def char(code_point)
195
+ code_point.chr(Encoding::UTF_8)
196
+ end
197
+
198
+ # The Encoding Standard's Shift_JIS decoder.
199
+ def shift_jis(bytes)
200
+ q = Queue.new(bytes)
201
+ out = String.new(encoding: Encoding::UTF_8)
202
+ lead = 0
203
+ while (b = q.read)
204
+ if lead != 0
205
+ l = lead
206
+ lead = 0
207
+ cp = nil
208
+ if b.between?(0x40, 0x7E) || b.between?(0x80, 0xFC)
209
+ pointer = (l - (l < 0xA0 ? 0x81 : 0xC1)) * 188 + b - (b < 0x7F ? 0x40 : 0x41)
210
+ cp = pointer.between?(8836, 10_715) ? char(0xE000 - 8836 + pointer) : lookup(:shift_jis, [l, b])
211
+ end
212
+ next out << cp if cp
213
+
214
+ q.prepend(b) if b < 0x80
215
+ out << REPLACEMENT
216
+ elsif b <= 0x80 then out << char(b)
217
+ elsif b.between?(0xA1, 0xDF) then out << char(0xFF61 - 0xA1 + b)
218
+ elsif b.between?(0x81, 0x9F) || b.between?(0xE0, 0xFC) then lead = b
219
+ else out << REPLACEMENT
220
+ end
221
+ end
222
+ out << REPLACEMENT if lead != 0
223
+ out
224
+ end
225
+
226
+ # The Encoding Standard's EUC-JP decoder.
227
+ def euc_jp(bytes)
228
+ q = Queue.new(bytes)
229
+ out = String.new(encoding: Encoding::UTF_8)
230
+ lead = 0
231
+ jis0212 = false
232
+ while (b = q.read)
233
+ if lead == 0x8E && b.between?(0xA1, 0xDF)
234
+ lead = 0
235
+ out << char(0xFF61 - 0xA1 + b)
236
+ elsif lead == 0x8F && b.between?(0xA1, 0xFE)
237
+ jis0212 = true
238
+ lead = b
239
+ elsif lead != 0
240
+ l = lead
241
+ lead = 0
242
+ cp = nil
243
+ cp = jis0212 ? lookup(:jis0212, [0x8F, l, b]) : lookup(:jis0208, [l, b]) if l.between?(0xA1, 0xFE) && b.between?(0xA1, 0xFE)
244
+ jis0212 = false
245
+ next out << cp if cp
246
+
247
+ q.prepend(b) if b < 0x80
248
+ out << REPLACEMENT
249
+ elsif b < 0x80 then out << char(b)
250
+ elsif b == 0x8E || b == 0x8F || b.between?(0xA1, 0xFE) then lead = b
251
+ else out << REPLACEMENT
252
+ end
253
+ end
254
+ out << REPLACEMENT if lead != 0
255
+ out
256
+ end
257
+
258
+ # The Encoding Standard's ISO-2022-JP decoder.
259
+ def iso_2022_jp(bytes)
260
+ q = Queue.new(bytes)
261
+ out = String.new(encoding: Encoding::UTF_8)
262
+ state = output_state = :ascii
263
+ lead = 0
264
+ output_flag = false
265
+ loop do
266
+ b = q.read
267
+ case state
268
+ when :ascii, :roman, :katakana, :lead
269
+ return out if b.nil?
270
+
271
+ if b == 0x1B
272
+ state = :escape_start
273
+ next
274
+ end
275
+ output_flag = false
276
+ case state
277
+ when :ascii
278
+ out << (b <= 0x7F && b != 0x0E && b != 0x0F ? char(b) : REPLACEMENT)
279
+ when :roman
280
+ out << (b == 0x5C ? char(0xA5) : b == 0x7E ? char(0x203E) : b <= 0x7F && b != 0x0E && b != 0x0F ? char(b) : REPLACEMENT)
281
+ when :katakana
282
+ out << (b.between?(0x21, 0x5F) ? char(0xFF61 - 0x21 + b) : REPLACEMENT)
283
+ when :lead
284
+ if b.between?(0x21, 0x7E)
285
+ lead = b
286
+ state = :trail
287
+ else
288
+ out << REPLACEMENT
289
+ end
290
+ end
291
+ when :trail
292
+ if b == 0x1B
293
+ state = :escape_start
294
+ out << REPLACEMENT
295
+ elsif b && b.between?(0x21, 0x7E)
296
+ state = :lead
297
+ out << (lookup(:jis0208, [lead | 0x80, b | 0x80]) || REPLACEMENT)
298
+ else
299
+ state = :lead
300
+ out << REPLACEMENT
301
+ end
302
+ when :escape_start
303
+ if b == 0x24 || b == 0x28
304
+ lead = b
305
+ state = :escape
306
+ else
307
+ q.prepend(b)
308
+ output_flag = false
309
+ state = output_state
310
+ out << REPLACEMENT
311
+ end
312
+ when :escape
313
+ l = lead
314
+ lead = 0
315
+ found = if l == 0x28 && b == 0x42 then :ascii
316
+ elsif l == 0x28 && b == 0x4A then :roman
317
+ elsif l == 0x28 && b == 0x49 then :katakana
318
+ elsif l == 0x24 && (b == 0x40 || b == 0x42) then :lead
319
+ end
320
+ if found
321
+ state = output_state = found
322
+ out << REPLACEMENT if output_flag
323
+ output_flag = true
324
+ else
325
+ q.prepend(l, b)
326
+ output_flag = false
327
+ state = output_state
328
+ out << REPLACEMENT
329
+ end
330
+ end
331
+ end
332
+ end
333
+
334
+ # The Encoding Standard's EUC-KR decoder.
335
+ def euc_kr(bytes)
336
+ q = Queue.new(bytes)
337
+ out = String.new(encoding: Encoding::UTF_8)
338
+ lead = 0
339
+ while (b = q.read)
340
+ if lead != 0
341
+ l = lead
342
+ lead = 0
343
+ cp = b.between?(0x41, 0xFE) ? lookup(:euc_kr, [l, b]) : nil
344
+ next out << cp if cp
345
+
346
+ q.prepend(b) if b < 0x80
347
+ out << REPLACEMENT
348
+ elsif b < 0x80 then out << char(b)
349
+ elsif b.between?(0x81, 0xFE) then lead = b
350
+ else out << REPLACEMENT
351
+ end
352
+ end
353
+ out << REPLACEMENT if lead != 0
354
+ out
355
+ end
356
+
357
+ # The Encoding Standard's Big5 decoder.
358
+ def big5(bytes)
359
+ q = Queue.new(bytes)
360
+ out = String.new(encoding: Encoding::UTF_8)
361
+ lead = 0
362
+ while (b = q.read)
363
+ if lead != 0
364
+ l = lead
365
+ lead = 0
366
+ cp = nil
367
+ if b.between?(0x40, 0x7E) || b.between?(0xA1, 0xFE)
368
+ pointer = (l - 0x81) * 157 + b - (b < 0x7F ? 0x40 : 0x62)
369
+ next BIG5_PAIRS[pointer].each { |c| out << char(c) } if BIG5_PAIRS.key?(pointer)
370
+
371
+ cp = lookup(:big5, [l, b])
372
+ end
373
+ next out << cp if cp
374
+
375
+ q.prepend(b) if b < 0x80
376
+ out << REPLACEMENT
377
+ elsif b < 0x80 then out << char(b)
378
+ elsif b.between?(0x81, 0xFE) then lead = b
379
+ else out << REPLACEMENT
380
+ end
381
+ end
382
+ out << REPLACEMENT if lead != 0
383
+ out
384
+ end
385
+
386
+ # The Encoding Standard's gb18030 decoder, which also decodes GBK.
387
+ def gb18030(bytes)
388
+ q = Queue.new(bytes)
389
+ out = String.new(encoding: Encoding::UTF_8)
390
+ first = second = third = 0
391
+ while (b = q.read)
392
+ if third != 0
393
+ unless b.between?(0x30, 0x39)
394
+ q.prepend(second, third, b)
395
+ first = second = third = 0
396
+ next out << REPLACEMENT
397
+ end
398
+ cp = lookup(:gb18030, [first, second, third, b])
399
+ first = second = third = 0
400
+ out << (cp || REPLACEMENT)
401
+ elsif second != 0
402
+ next third = b if b.between?(0x81, 0xFE)
403
+
404
+ q.prepend(second, b)
405
+ first = second = 0
406
+ out << REPLACEMENT
407
+ elsif first != 0
408
+ next second = b if b.between?(0x30, 0x39)
409
+
410
+ l = first
411
+ first = 0
412
+ cp = b.between?(0x40, 0x7E) || b.between?(0x80, 0xFE) ? lookup(:gb18030, [l, b]) : nil
413
+ next out << cp if cp
414
+
415
+ q.prepend(b) if b < 0x80
416
+ out << REPLACEMENT
417
+ elsif b < 0x80 then out << char(b)
418
+ elsif b == 0x80 then out << char(0x20AC)
419
+ elsif b.between?(0x81, 0xFE) then first = b
420
+ else out << REPLACEMENT
421
+ end
422
+ end
423
+ out << REPLACEMENT if first != 0 || second != 0 || third != 0
424
+ out
425
+ end
426
+ end
427
+ end