fasthtmlparser 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,499 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FastHtmlParser
4
+ # @api private A parsed URL (net/url's URL); every string is binary.
5
+ class Url
6
+ attr_accessor :scheme, :opaque, :user, :host, :path, :raw_path, :force_query, :raw_query, :fragment, :raw_fragment
7
+
8
+ def initialize
9
+ @scheme = Chars::EMPTY
10
+ @opaque = Chars::EMPTY
11
+ @user = nil # the userinfo as written back (escaped), or nil
12
+ @host = Chars::EMPTY
13
+ @path = Chars::EMPTY # decoded
14
+ @raw_path = Chars::EMPTY # the path as written, when it differs from the default escaping of path
15
+ @force_query = false
16
+ @raw_query = Chars::EMPTY
17
+ @fragment = Chars::EMPTY # decoded
18
+ @raw_fragment = Chars::EMPTY
19
+ end
20
+ end
21
+
22
+ # @api private Links: href / src made absolute against the page URL, written as .NET's System.Uri writes them.
23
+ #
24
+ # Resolution follows RFC 3986 the way the Go port does it (net/url: Parse + ResolveReference), byte for byte.
25
+ module Links
26
+ class UrlError < StandardError; end
27
+
28
+ MODE_PATH = 0
29
+ MODE_HOST = 1
30
+ MODE_USER_PASSWORD = 2
31
+ MODE_FRAGMENT = 3
32
+
33
+ DEFAULT_PORTS = { "http" => "80", "https" => "443", "ftp" => "21", "ws" => "80", "wss" => "443" }.freeze
34
+ RESERVED_FOR_DISPLAY = "%/?#[]@:!$&'()*+,;="
35
+ UPPER_HEX = "0123456789ABCDEF"
36
+ HOST_ALLOWED = "!$&'()*+,;=:[]<>\""
37
+ VALID_ENCODED = "!$&'()*+,;=:@[]%"
38
+ USERINFO = /\A[a-zA-Z0-9\-._:~!$&'()*+,;=%@]*\z/n
39
+ CONTROL = /[\x00-\x1f\x7f]/n
40
+
41
+ # escape decisions by byte, per mode
42
+ SHOULD_ESCAPE = Array.new(4) do |mode|
43
+ Array.new(256) do |c|
44
+ ch = c.chr
45
+ if (c >= 0x61 && c <= 0x7a) || (c >= 0x41 && c <= 0x5a) || (c >= 0x30 && c <= 0x39)
46
+ false
47
+ elsif mode == MODE_HOST && HOST_ALLOWED.include?(ch)
48
+ false
49
+ elsif "-_.~".include?(ch)
50
+ false
51
+ elsif "$&+,/:;=?@".include?(ch) && mode != MODE_HOST
52
+ case mode
53
+ when MODE_PATH then ch == "?"
54
+ when MODE_USER_PASSWORD then ch == "@" || ch == "/" || ch == "?" || ch == ":"
55
+ else false
56
+ end
57
+ elsif mode == MODE_FRAGMENT && "!()*".include?(ch)
58
+ false
59
+ else
60
+ true
61
+ end
62
+ end.freeze
63
+ end.freeze
64
+
65
+ module_function
66
+
67
+ def should_escape?(c, mode)
68
+ SHOULD_ESCAPE[mode][c]
69
+ end
70
+
71
+ def hex?(c)
72
+ Chars.hex_byte?(c)
73
+ end
74
+
75
+ def unhex(c)
76
+ c <= 57 ? c - 48 : (c | 0x20) - 87
77
+ end
78
+
79
+ # net/url unescape; raises UrlError on a malformed URL.
80
+ def unescape(s, mode)
81
+ count = 0
82
+ n = s.bytesize
83
+ i = 0
84
+ while i < n
85
+ c = s.getbyte(i)
86
+ if c == 37
87
+ count += 1
88
+ raise UrlError, "invalid URL escape" if i + 2 >= n || !hex?(s.getbyte(i + 1)) || !hex?(s.getbyte(i + 2))
89
+ # in the host, %-encoding is only for non-ASCII bytes (and %25)
90
+ if mode == MODE_HOST && unhex(s.getbyte(i + 1)) < 8 && s.byteslice(i, 3) != "%25"
91
+ raise UrlError, "invalid URL escape"
92
+ end
93
+
94
+ i += 3
95
+ else
96
+ raise UrlError, "invalid character in host name" if mode == MODE_HOST && c < 0x80 && should_escape?(c, mode)
97
+
98
+ i += 1
99
+ end
100
+ end
101
+ return s if count == 0
102
+
103
+ out = String.new(capacity: n, encoding: Encoding::BINARY)
104
+ i = 0
105
+ while i < n
106
+ c = s.getbyte(i)
107
+ if c == 37
108
+ out << ((unhex(s.getbyte(i + 1)) << 4) | unhex(s.getbyte(i + 2)))
109
+ i += 3
110
+ else
111
+ out << c
112
+ i += 1
113
+ end
114
+ end
115
+ out
116
+ end
117
+
118
+ def escape(s, mode)
119
+ table = SHOULD_ESCAPE[mode]
120
+ out = nil
121
+ from = 0
122
+ n = s.bytesize
123
+ i = 0
124
+ while i < n
125
+ c = s.getbyte(i)
126
+ if table[c]
127
+ out ||= String.new(capacity: n + 16, encoding: Encoding::BINARY)
128
+ out << s.byteslice(from, i - from) << "%" << UPPER_HEX[c >> 4] << UPPER_HEX[c & 15]
129
+ from = i + 1
130
+ end
131
+ i += 1
132
+ end
133
+ return s if out.nil?
134
+
135
+ out << s.byteslice(from, n - from)
136
+ end
137
+
138
+ def valid_encoded?(s, mode)
139
+ s.each_byte do |c|
140
+ next if VALID_ENCODED.include?(c.chr)
141
+ return false if should_escape?(c, mode)
142
+ end
143
+ true
144
+ end
145
+
146
+ # ----- parsing -----
147
+
148
+ # [scheme, rest]
149
+ def get_scheme(raw)
150
+ n = raw.bytesize
151
+ i = 0
152
+ while i < n
153
+ c = raw.getbyte(i)
154
+ if Chars.alpha_byte?(c)
155
+ i += 1
156
+ next
157
+ end
158
+ if Chars.digit_byte?(c) || c == 43 || c == 45 || c == 46
159
+ return [Chars::EMPTY, raw] if i == 0
160
+
161
+ i += 1
162
+ next
163
+ end
164
+ if c == 58
165
+ raise UrlError, "missing protocol scheme" if i == 0
166
+
167
+ return [raw.byteslice(0, i), raw.byteslice(i + 1, n - i - 1)]
168
+ end
169
+ return [Chars::EMPTY, raw]
170
+ end
171
+ [Chars::EMPTY, raw]
172
+ end
173
+
174
+ def set_path(u, p)
175
+ u.path = unescape(p, MODE_PATH)
176
+ u.raw_path = escape(u.path, MODE_PATH) == p ? Chars::EMPTY : p
177
+ end
178
+
179
+ def escaped_path(u)
180
+ if !u.raw_path.empty? && valid_encoded?(u.raw_path, MODE_PATH)
181
+ begin
182
+ return u.raw_path if unescape(u.raw_path, MODE_PATH) == u.path
183
+ rescue UrlError
184
+ # fall through
185
+ end
186
+ end
187
+ return "*" if u.path == "*"
188
+
189
+ escape(u.path, MODE_PATH)
190
+ end
191
+
192
+ def escaped_fragment(u)
193
+ if !u.raw_fragment.empty? && valid_encoded?(u.raw_fragment, MODE_FRAGMENT)
194
+ begin
195
+ return u.raw_fragment if unescape(u.raw_fragment, MODE_FRAGMENT) == u.fragment
196
+ rescue UrlError
197
+ # fall through
198
+ end
199
+ end
200
+ escape(u.fragment, MODE_FRAGMENT)
201
+ end
202
+
203
+ def valid_optional_port?(port)
204
+ return true if port.empty?
205
+ return false if port.getbyte(0) != 58
206
+
207
+ port.byteslice(1, port.bytesize - 1).each_byte.all? { |c| c >= 48 && c <= 57 }
208
+ end
209
+
210
+ def parse_host(host)
211
+ if host.start_with?("[")
212
+ i = host.rindex("]")
213
+ raise UrlError, "missing ']' in host" if i.nil?
214
+ raise UrlError, "invalid port" unless valid_optional_port?(host.byteslice(i + 1, host.bytesize - i - 1))
215
+ else
216
+ i = host.rindex(":")
217
+ if i && !valid_optional_port?(host.byteslice(i, host.bytesize - i))
218
+ raise UrlError, "invalid port"
219
+ end
220
+ end
221
+ unescape(host, MODE_HOST)
222
+ end
223
+
224
+ def parse_authority(u, authority)
225
+ i = authority.rindex("@")
226
+ u.host = parse_host(i.nil? ? authority : authority.byteslice(i + 1, authority.bytesize - i - 1))
227
+ return if i.nil?
228
+
229
+ userinfo = authority.byteslice(0, i)
230
+ raise UrlError, "invalid userinfo" unless USERINFO.match?(userinfo)
231
+
232
+ colon = userinfo.index(":")
233
+ if colon.nil?
234
+ u.user = escape(unescape(userinfo, MODE_USER_PASSWORD), MODE_USER_PASSWORD)
235
+ else
236
+ name = unescape(userinfo.byteslice(0, colon), MODE_USER_PASSWORD)
237
+ password = unescape(userinfo.byteslice(colon + 1, userinfo.bytesize - colon - 1), MODE_USER_PASSWORD)
238
+ u.user = escape(name, MODE_USER_PASSWORD) + ":" + escape(password, MODE_USER_PASSWORD)
239
+ end
240
+ end
241
+
242
+ # net/url Parse.
243
+ def parse_url(raw_url)
244
+ hash = raw_url.index("#")
245
+ u = parse_no_fragment(hash.nil? ? raw_url : raw_url.byteslice(0, hash))
246
+ if hash
247
+ frag = raw_url.byteslice(hash + 1, raw_url.bytesize - hash - 1)
248
+ u.fragment = unescape(frag, MODE_FRAGMENT)
249
+ u.raw_fragment = escape(u.fragment, MODE_FRAGMENT) == frag ? Chars::EMPTY : frag
250
+ end
251
+ u
252
+ end
253
+
254
+ def parse_no_fragment(raw)
255
+ raise UrlError, "invalid control character in URL" if CONTROL.match?(raw)
256
+
257
+ u = Url.new
258
+ if raw == "*"
259
+ u.path = "*"
260
+ return u
261
+ end
262
+ scheme, rest = get_scheme(raw)
263
+ u.scheme = scheme.downcase(:ascii)
264
+
265
+ q = rest.index("?")
266
+ if q && q == rest.bytesize - 1
267
+ u.force_query = true
268
+ rest = rest.byteslice(0, q)
269
+ elsif q
270
+ u.raw_query = rest.byteslice(q + 1, rest.bytesize - q - 1)
271
+ rest = rest.byteslice(0, q)
272
+ end
273
+
274
+ unless rest.start_with?("/")
275
+ unless u.scheme.empty?
276
+ # rootless paths are opaque (RFC 3986)
277
+ u.opaque = rest
278
+ return u
279
+ end
280
+ # a relative-path reference: the first segment cannot contain ':'
281
+ slash = rest.index("/")
282
+ first = slash.nil? ? rest : rest.byteslice(0, slash)
283
+ raise UrlError, "first path segment in URL cannot contain colon" if first.include?(":")
284
+ end
285
+
286
+ if (!u.scheme.empty? || !rest.start_with?("///")) && rest.start_with?("//")
287
+ authority = rest.byteslice(2, rest.bytesize - 2)
288
+ rest = Chars::EMPTY
289
+ slash = authority.index("/")
290
+ if slash
291
+ rest = authority.byteslice(slash, authority.bytesize - slash)
292
+ authority = authority.byteslice(0, slash)
293
+ end
294
+ parse_authority(u, authority)
295
+ end
296
+ set_path(u, rest)
297
+ u
298
+ end
299
+
300
+ def resolve_path(base, ref)
301
+ full = if ref.empty?
302
+ base
303
+ elsif ref.getbyte(0) != 47
304
+ slash = base.rindex("/")
305
+ (slash.nil? ? +"".b : base.byteslice(0, slash + 1)) + ref
306
+ else
307
+ ref
308
+ end
309
+ return Chars::EMPTY if full.empty?
310
+
311
+ dst = +"/".b
312
+ first = true
313
+ remaining = full
314
+ found = true
315
+ elem = Chars::EMPTY
316
+ while found
317
+ slash = remaining.index("/")
318
+ found = !slash.nil?
319
+ elem = found ? remaining.byteslice(0, slash) : remaining
320
+ remaining = found ? remaining.byteslice(slash + 1, remaining.bytesize - slash - 1) : Chars::EMPTY
321
+ if elem == "."
322
+ first = false
323
+ next
324
+ end
325
+ if elem == ".."
326
+ str = dst.byteslice(1, dst.bytesize - 1)
327
+ index = str.rindex("/")
328
+ dst = +"/".b
329
+ if index.nil?
330
+ first = true
331
+ else
332
+ dst << str.byteslice(0, index)
333
+ end
334
+ else
335
+ dst << "/" unless first
336
+ dst << elem
337
+ first = false
338
+ end
339
+ end
340
+ dst << "/" if elem == "." || elem == ".."
341
+ dst = dst.byteslice(1, dst.bytesize - 1) if dst.bytesize > 1 && dst.getbyte(1) == 47
342
+ dst
343
+ end
344
+
345
+ # net/url ResolveReference.
346
+ def resolve_reference(base, ref)
347
+ url = ref.dup
348
+ url.scheme = base.scheme if ref.scheme.empty?
349
+ if !ref.scheme.empty? || !ref.host.empty? || !ref.user.nil?
350
+ set_path(url, resolve_path(escaped_path(ref), Chars::EMPTY))
351
+ return url
352
+ end
353
+ unless ref.opaque.empty?
354
+ url.user = nil
355
+ url.host = Chars::EMPTY
356
+ url.path = Chars::EMPTY
357
+ return url
358
+ end
359
+ if ref.path.empty? && !ref.force_query && ref.raw_query.empty?
360
+ url.raw_query = base.raw_query
361
+ if ref.fragment.empty?
362
+ url.fragment = base.fragment
363
+ url.raw_fragment = base.raw_fragment
364
+ end
365
+ end
366
+ if ref.path.empty? && !base.opaque.empty?
367
+ url.opaque = base.opaque
368
+ url.user = nil
369
+ url.host = Chars::EMPTY
370
+ url.path = Chars::EMPTY
371
+ return url
372
+ end
373
+ url.host = base.host
374
+ url.user = base.user
375
+ set_path(url, resolve_path(escaped_path(base), escaped_path(ref)))
376
+ url
377
+ end
378
+
379
+ # [host, port]
380
+ def split_host_port(host_port)
381
+ host = host_port
382
+ port = Chars::EMPTY
383
+ colon = host.rindex(":")
384
+ if colon && valid_optional_port?(host.byteslice(colon, host.bytesize - colon))
385
+ port = host.byteslice(colon + 1, host.bytesize - colon - 1)
386
+ host = host.byteslice(0, colon)
387
+ end
388
+ host = host.byteslice(1, host.bytesize - 2) if host.bytesize >= 2 && host.getbyte(0) == 91 && host.getbyte(-1) == 93
389
+ [host, port]
390
+ end
391
+
392
+ # Decodes %XX escapes except those of reserved and control characters; escaped bytes of non-ASCII text are decoded
393
+ # when they form valid UTF-8.
394
+ def unescape_for_display(s)
395
+ return s unless s.include?("%")
396
+
397
+ out = String.new(capacity: s.bytesize, encoding: Encoding::BINARY)
398
+ n = s.bytesize
399
+ i = 0
400
+ while i < n
401
+ if s.getbyte(i) != 37 || i + 2 >= n || !hex?(s.getbyte(i + 1)) || !hex?(s.getbyte(i + 2))
402
+ out << s.getbyte(i)
403
+ i += 1
404
+ next
405
+ end
406
+ c = (unhex(s.getbyte(i + 1)) << 4) | unhex(s.getbyte(i + 2))
407
+ if c < 0x80
408
+ if c < 0x20 || c == 0x7f || RESERVED_FOR_DISPLAY.include?(c.chr)
409
+ out << s.byteslice(i, 3)
410
+ else
411
+ out << c
412
+ end
413
+ i += 3
414
+ next
415
+ end
416
+ # a run of escaped bytes >= 0x80: decoded together when it is valid UTF-8
417
+ j = i
418
+ run = String.new(encoding: Encoding::BINARY)
419
+ while j + 2 < n && s.getbyte(j) == 37 && hex?(s.getbyte(j + 1)) && hex?(s.getbyte(j + 2))
420
+ v = (unhex(s.getbyte(j + 1)) << 4) | unhex(s.getbyte(j + 2))
421
+ break if v < 0x80
422
+
423
+ run << v
424
+ j += 3
425
+ end
426
+ out << (run.dup.force_encoding(Encoding::UTF_8).valid_encoding? ? run : s.byteslice(i, j - i))
427
+ i = j
428
+ end
429
+ out
430
+ end
431
+
432
+ # Writes a resolved URL as the .NET version does (System.Uri.ToString): the scheme and host in lower case, no
433
+ # default port, an empty path as "/", and percent-escapes decoded except those of the characters that delimit URL
434
+ # parts ("%2F" stays) and of control characters; non-ASCII text is written as is.
435
+ def display_string(u)
436
+ b = u.scheme.downcase(:ascii) + ":"
437
+ if u.opaque.empty?
438
+ if !u.host.empty? || !u.user.nil?
439
+ b << "//"
440
+ b << u.user << "@" unless u.user.nil?
441
+ host, port = split_host_port(u.host)
442
+ b << Chars.lower(Chars.scrub(host))
443
+ b << ":" << port if !port.empty? && port != DEFAULT_PORTS[u.scheme.downcase(:ascii)]
444
+ end
445
+ path = escaped_path(u)
446
+ path = "/" if path.empty? && !u.host.empty?
447
+ b << unescape_for_display(path)
448
+ else
449
+ b << u.opaque
450
+ end
451
+ b << "?" << unescape_for_display(u.raw_query) if !u.raw_query.empty? || u.force_query
452
+ b << "#" << unescape_for_display(escaped_fragment(u)) unless u.fragment.empty?
453
+ Chars.scrub(b)
454
+ end
455
+
456
+ # Parses a page URL with its own scheme and host; nil for anything else.
457
+ def absolute_url(s)
458
+ return nil if s.empty?
459
+
460
+ u = parse_url(s)
461
+ return nil if u.scheme.empty? || (u.host.empty? && u.opaque.empty? && !u.scheme.casecmp?("file"))
462
+
463
+ u
464
+ rescue UrlError
465
+ nil
466
+ end
467
+
468
+ # A link with its own scheme ("https://...", "mailto:...", "file:..."). "/path" and "//host/path" are relative.
469
+ def absolute_link?(link)
470
+ !parse_url(Chars.trim_space(link)).scheme.empty?
471
+ rescue UrlError
472
+ false
473
+ end
474
+
475
+ # The raw attribute value with its character references decoded and made absolute against the page.
476
+ # Percent-escapes are part of the URL and are kept. An empty link gives the page URL.
477
+ def correct_link(raw, page_url, base)
478
+ return page_url.dup if raw.empty?
479
+
480
+ link = Decode.needs_decoding?(raw) ? Decode.decode(raw, true) : raw
481
+ return link if base.nil? || absolute_link?(link)
482
+
483
+ begin
484
+ display_string(resolve_reference(base, parse_url(Chars.trim_space(link))))
485
+ rescue UrlError
486
+ link
487
+ end
488
+ end
489
+
490
+ # Everything after the first ':' of a link with the scheme (mailto, tel), or "".
491
+ def after_scheme(raw_href, scheme)
492
+ href = Chars.trim_space(Decode.decode(raw_href, true))
493
+ return +"".b if href.bytesize < scheme.bytesize || !href.byteslice(0, scheme.bytesize).casecmp?(scheme)
494
+
495
+ i = href.index(":")
496
+ i.nil? ? +"".b : href.byteslice(i + 1, href.bytesize - i - 1)
497
+ end
498
+ end
499
+ end