fasthtmlparser 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.md +79 -0
- data/README.md +196 -0
- data/fasthtmlparser.gemspec +26 -0
- data/lib/fasthtmlparser/attr.rb +243 -0
- data/lib/fasthtmlparser/attrs.rb +278 -0
- data/lib/fasthtmlparser/chars.rb +258 -0
- data/lib/fasthtmlparser/css.rb +1061 -0
- data/lib/fasthtmlparser/decode.rb +164 -0
- data/lib/fasthtmlparser/entities.rb +2251 -0
- data/lib/fasthtmlparser/html.rb +52 -0
- data/lib/fasthtmlparser/html_document.rb +294 -0
- data/lib/fasthtmlparser/html_node.rb +706 -0
- data/lib/fasthtmlparser/links.rb +499 -0
- data/lib/fasthtmlparser/parser.rb +893 -0
- data/lib/fasthtmlparser/query.rb +221 -0
- data/lib/fasthtmlparser/tags.rb +761 -0
- data/lib/fasthtmlparser/text.rb +280 -0
- data/lib/fasthtmlparser/version.rb +5 -0
- data/lib/fasthtmlparser/xpath.rb +1151 -0
- data/lib/fasthtmlparser/xpath_compiler.rb +627 -0
- data/lib/fasthtmlparser.rb +35 -0
- metadata +66 -0
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FastHtmlParser
|
|
4
|
+
# @api private HTML5 character reference decoder (WHATWG "character reference state"): all 2125 named references,
|
|
5
|
+
# the legacy names without ';', numeric references with or without ';', and the spec replacements (0, surrogates,
|
|
6
|
+
# > U+10FFFF -> U+FFFD; 0x80-0x9F -> Windows-1252).
|
|
7
|
+
module Decode
|
|
8
|
+
# Replacements for numeric references 0x80..0x9F; 0 = keep the code point as is.
|
|
9
|
+
WINDOWS_1252 = [
|
|
10
|
+
0x20ac, 0, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, 0x02c6, 0x2030, 0x0160, 0x2039, 0x0152, 0, 0x017d, 0,
|
|
11
|
+
0, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0x02dc, 0x2122, 0x0161, 0x203a, 0x0153, 0, 0x017e,
|
|
12
|
+
0x0178
|
|
13
|
+
].freeze
|
|
14
|
+
|
|
15
|
+
MAP = Entities::MAP
|
|
16
|
+
LEGACY = Entities::LEGACY
|
|
17
|
+
MAX_NAME = Entities::MAX_NAME
|
|
18
|
+
MAX_LEGACY_NAME = Entities::MAX_LEGACY_NAME
|
|
19
|
+
AMP = "&"
|
|
20
|
+
CR = "\r"
|
|
21
|
+
|
|
22
|
+
module_function
|
|
23
|
+
|
|
24
|
+
def alnum?(c)
|
|
25
|
+
c && ((c >= 48 && c <= 57) || ((c | 0x20) >= 97 && (c | 0x20) <= 122))
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Whether a value read from the page differs from its source: it has a character reference or a CR.
|
|
29
|
+
def needs_decoding?(s)
|
|
30
|
+
s.include?(AMP) || s.include?(CR)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# HTML input stream preprocessing: CR LF and a lone CR become LF.
|
|
34
|
+
def normalize_newlines(s)
|
|
35
|
+
return s unless s.include?(CR)
|
|
36
|
+
|
|
37
|
+
s.gsub("\r\n", "\n").tr("\r", "\n")
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Decodes the character references in s; CR LF and a lone CR become LF. With in_attribute, a legacy name without
|
|
41
|
+
# ';' followed by '=' or an alphanumeric is left as is (so "?a=1©=2" stays intact). Returns a new string or s.
|
|
42
|
+
def decode(s, in_attribute)
|
|
43
|
+
# Line breaks first, as a browser reads the page; a CR written as " " is decoded after that and stays.
|
|
44
|
+
s = normalize_newlines(s) if s.include?(CR)
|
|
45
|
+
amp = s.index(AMP)
|
|
46
|
+
return s if amp.nil?
|
|
47
|
+
|
|
48
|
+
n = s.bytesize
|
|
49
|
+
out = String.new(capacity: n, encoding: Encoding::BINARY)
|
|
50
|
+
pos = 0
|
|
51
|
+
while amp
|
|
52
|
+
out << s.byteslice(pos, amp - pos) if amp > pos
|
|
53
|
+
consumed = 0
|
|
54
|
+
text = nil
|
|
55
|
+
if amp + 1 < n
|
|
56
|
+
text, consumed = reference(s, amp + 1, in_attribute)
|
|
57
|
+
end
|
|
58
|
+
if text.nil?
|
|
59
|
+
out << AMP # not a reference: keep '&'
|
|
60
|
+
pos = amp + 1
|
|
61
|
+
else
|
|
62
|
+
out << text
|
|
63
|
+
pos = amp + 1 + consumed
|
|
64
|
+
end
|
|
65
|
+
amp = pos < n ? s.index(AMP, pos) : nil
|
|
66
|
+
end
|
|
67
|
+
out << s.byteslice(pos, n - pos) if pos < n
|
|
68
|
+
out
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# Decodes one character reference; s[at] is the char right after '&'. Returns [text, bytes consumed after '&'],
|
|
72
|
+
# or [nil, 0] when there is no reference.
|
|
73
|
+
def decode_reference(s, at)
|
|
74
|
+
return [nil, 0] if at >= s.bytesize
|
|
75
|
+
|
|
76
|
+
reference(s, at, false)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def reference(s, at, in_attribute)
|
|
80
|
+
if s.getbyte(at) == 35
|
|
81
|
+
cp, consumed = match_numeric(s, at)
|
|
82
|
+
return [nil, 0] if cp < 0
|
|
83
|
+
|
|
84
|
+
return [Chars.utf8(cp), consumed]
|
|
85
|
+
end
|
|
86
|
+
match_named(s, at, in_attribute)
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# s[at] is '#'. Returns [code point (-1: not a reference), bytes consumed].
|
|
90
|
+
def match_numeric(s, at)
|
|
91
|
+
n = s.bytesize
|
|
92
|
+
i = at + 1
|
|
93
|
+
c = i < n ? s.getbyte(i) : nil
|
|
94
|
+
hex = c == 120 || c == 88
|
|
95
|
+
i += 1 if hex
|
|
96
|
+
digits = i
|
|
97
|
+
value = 0
|
|
98
|
+
if hex
|
|
99
|
+
while i < n
|
|
100
|
+
c = s.getbyte(i)
|
|
101
|
+
if c >= 48 && c <= 57
|
|
102
|
+
d = c - 48
|
|
103
|
+
else
|
|
104
|
+
l = c | 0x20
|
|
105
|
+
break if l < 97 || l > 102
|
|
106
|
+
|
|
107
|
+
d = l - 87
|
|
108
|
+
end
|
|
109
|
+
value = value * 16 + d
|
|
110
|
+
value = 0x110000 if value > 0x10ffff # saturate on long digit runs
|
|
111
|
+
i += 1
|
|
112
|
+
end
|
|
113
|
+
else
|
|
114
|
+
while i < n
|
|
115
|
+
c = s.getbyte(i)
|
|
116
|
+
break if c < 48 || c > 57
|
|
117
|
+
|
|
118
|
+
value = value * 10 + c - 48
|
|
119
|
+
value = 0x110000 if value > 0x10ffff
|
|
120
|
+
i += 1
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
return [-1, 0] if i == digits # "&#" / "&#x" without digits is plain text
|
|
124
|
+
|
|
125
|
+
i += 1 if i < n && s.getbyte(i) == 59
|
|
126
|
+
if value == 0 || value > 0x10ffff || (value >= 0xd800 && value <= 0xdfff)
|
|
127
|
+
value = 0xfffd
|
|
128
|
+
elsif value >= 0x80 && value <= 0x9f
|
|
129
|
+
mapped = WINDOWS_1252[value - 0x80]
|
|
130
|
+
value = mapped if mapped != 0
|
|
131
|
+
end
|
|
132
|
+
[value, i - at]
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# s[at] is the char right after '&'. Returns [decoded text (nil: not a reference), bytes consumed].
|
|
136
|
+
def match_named(s, at, in_attribute)
|
|
137
|
+
len = s.bytesize
|
|
138
|
+
n = 0
|
|
139
|
+
limit = MAX_NAME + 1
|
|
140
|
+
n += 1 while n < limit && at + n < len && alnum?(s.getbyte(at + n))
|
|
141
|
+
return [nil, 0] if n < 2
|
|
142
|
+
|
|
143
|
+
# Full match: "&name;".
|
|
144
|
+
if at + n < len && s.getbyte(at + n) == 59
|
|
145
|
+
v = MAP[s.byteslice(at, n)]
|
|
146
|
+
return [v, n + 1] if v
|
|
147
|
+
end
|
|
148
|
+
# Longest legacy prefix without ';': "©2024" -> "(c)2024", "¬it;" -> "(not)it;".
|
|
149
|
+
k = n < MAX_LEGACY_NAME ? n : MAX_LEGACY_NAME
|
|
150
|
+
while k >= 2
|
|
151
|
+
name = s.byteslice(at, k)
|
|
152
|
+
if LEGACY[name]
|
|
153
|
+
if in_attribute && at + k < len
|
|
154
|
+
c = s.getbyte(at + k)
|
|
155
|
+
break if c == 61 || alnum?(c)
|
|
156
|
+
end
|
|
157
|
+
return [MAP[name], k]
|
|
158
|
+
end
|
|
159
|
+
k -= 1
|
|
160
|
+
end
|
|
161
|
+
[nil, 0]
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
end
|