fasthtmlparser 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.md +79 -0
- data/README.md +196 -0
- data/fasthtmlparser.gemspec +26 -0
- data/lib/fasthtmlparser/attr.rb +243 -0
- data/lib/fasthtmlparser/attrs.rb +278 -0
- data/lib/fasthtmlparser/chars.rb +258 -0
- data/lib/fasthtmlparser/css.rb +1061 -0
- data/lib/fasthtmlparser/decode.rb +164 -0
- data/lib/fasthtmlparser/entities.rb +2251 -0
- data/lib/fasthtmlparser/html.rb +52 -0
- data/lib/fasthtmlparser/html_document.rb +294 -0
- data/lib/fasthtmlparser/html_node.rb +706 -0
- data/lib/fasthtmlparser/links.rb +499 -0
- data/lib/fasthtmlparser/parser.rb +893 -0
- data/lib/fasthtmlparser/query.rb +221 -0
- data/lib/fasthtmlparser/tags.rb +761 -0
- data/lib/fasthtmlparser/text.rb +280 -0
- data/lib/fasthtmlparser/version.rb +5 -0
- data/lib/fasthtmlparser/xpath.rb +1151 -0
- data/lib/fasthtmlparser/xpath_compiler.rb +627 -0
- data/lib/fasthtmlparser.rb +35 -0
- metadata +66 -0
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FastHtmlParser
|
|
4
|
+
# @api private Visible text: tags, comments and invisible content removed, whitespace collapsed, entities decoded.
|
|
5
|
+
module Text
|
|
6
|
+
SPACE_RUNS = Chars::SPACE_RUNS
|
|
7
|
+
SPACE_BYTE = Chars::SPACE_BYTE
|
|
8
|
+
INVISIBLE = {
|
|
9
|
+
"style" => "</style", "script" => "</script", "noembed" => "</noembed", "noscript" => "</noscript",
|
|
10
|
+
"template" => "</template", "noframes" => "</noframes"
|
|
11
|
+
}.freeze
|
|
12
|
+
LT = "<"
|
|
13
|
+
GT = ">"
|
|
14
|
+
SP = " "
|
|
15
|
+
|
|
16
|
+
module_function
|
|
17
|
+
|
|
18
|
+
# s[i] is '<'. Returns the index of the last char of what is not text there: a comment, a tag, or a tag together
|
|
19
|
+
# with invisible content (script, style, ...) up to its end tag. -1: an unterminated tag.
|
|
20
|
+
def skip_markup(s, i)
|
|
21
|
+
n = s.bytesize
|
|
22
|
+
if i + 4 <= n && s.getbyte(i + 1) == 33 && s.getbyte(i + 2) == 45 && s.getbyte(i + 3) == 45
|
|
23
|
+
return Parser.find_comment_end(s, i + 4, n)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
tag_end = s.index(GT, i + 1)
|
|
27
|
+
return -1 if tag_end.nil?
|
|
28
|
+
|
|
29
|
+
end_tag = invisible_content_end(s, i + 1, tag_end)
|
|
30
|
+
return Parser.find_raw_text_end(s, tag_end + 1, end_tag, n) - 1 if end_tag
|
|
31
|
+
|
|
32
|
+
tag_end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# For the inside of an opening tag s[from, to), the end tag ("</script") when the element's content is never
|
|
36
|
+
# visible text (script, style, noscript, template, noframes, noembed), otherwise nil.
|
|
37
|
+
def invisible_content_end(s, from, to)
|
|
38
|
+
return nil if from >= to
|
|
39
|
+
|
|
40
|
+
first = s.getbyte(from) | 0x20
|
|
41
|
+
return nil if first != 0x73 && first != 0x74 && first != 0x6e # s, t, n
|
|
42
|
+
|
|
43
|
+
n = 1
|
|
44
|
+
while from + n < to && n <= 8
|
|
45
|
+
c = s.getbyte(from + n) | 0x20
|
|
46
|
+
break if c < 0x61 || c > 0x7a
|
|
47
|
+
|
|
48
|
+
n += 1
|
|
49
|
+
end
|
|
50
|
+
if from + n < to
|
|
51
|
+
c = s.getbyte(from + n)
|
|
52
|
+
return nil if c != 47 && !Chars.space_byte?(c)
|
|
53
|
+
end
|
|
54
|
+
end_ = INVISIBLE[s.byteslice(from, n).downcase(:ascii)]
|
|
55
|
+
return nil if end_.nil?
|
|
56
|
+
|
|
57
|
+
e = to
|
|
58
|
+
e -= 1 while e > from && Chars.space_byte?(s.getbyte(e - 1))
|
|
59
|
+
return nil if s.getbyte(e - 1) == 47 # <script src="x" />: no content
|
|
60
|
+
|
|
61
|
+
end_
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# The text runs of s with tags, comments and invisible content removed, whitespace not yet collapsed.
|
|
65
|
+
def text_runs(s)
|
|
66
|
+
lt = s.index(LT)
|
|
67
|
+
return s if lt.nil?
|
|
68
|
+
|
|
69
|
+
n = s.bytesize
|
|
70
|
+
out = String.new(capacity: n, encoding: Encoding::BINARY)
|
|
71
|
+
from = 0
|
|
72
|
+
while lt
|
|
73
|
+
unless Parser.markup_start?(s, lt, n)
|
|
74
|
+
lt = s.index(LT, lt + 1)
|
|
75
|
+
next
|
|
76
|
+
end
|
|
77
|
+
out << s.byteslice(from, lt - from) if lt > from
|
|
78
|
+
last = skip_markup(s, lt)
|
|
79
|
+
return out if last < 0 # an unterminated tag: the rest is markup
|
|
80
|
+
|
|
81
|
+
from = last + 1
|
|
82
|
+
lt = s.index(LT, from)
|
|
83
|
+
end
|
|
84
|
+
return s if from == 0
|
|
85
|
+
|
|
86
|
+
out << s.byteslice(from, n - from) if from < n
|
|
87
|
+
out
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# Whitespace runs as one ' ', trimmed.
|
|
91
|
+
def collapse(s)
|
|
92
|
+
return s unless SPACE_BYTE.match?(s)
|
|
93
|
+
|
|
94
|
+
t = s.gsub(SPACE_RUNS, SP)
|
|
95
|
+
return t if t.empty?
|
|
96
|
+
|
|
97
|
+
a = t.getbyte(0) == 32 ? 1 : 0
|
|
98
|
+
z = t.bytesize - (t.getbyte(-1) == 32 ? 1 : 0)
|
|
99
|
+
a == 0 && z == t.bytesize ? t : Chars.slice(t, a, z)
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# The visible text of an HTML fragment.
|
|
103
|
+
def clear_tags(s)
|
|
104
|
+
return +"".b if s.empty?
|
|
105
|
+
|
|
106
|
+
text = collapse(text_runs(s))
|
|
107
|
+
text.include?(Decode::AMP) ? Decode.decode(text, false) : text
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# Whether the visible text of the body (value) equals expected; n gives the full text.
|
|
111
|
+
def text?(value, expected, n)
|
|
112
|
+
return false if value.empty? || expected.empty?
|
|
113
|
+
|
|
114
|
+
# whitespace is collapsed even where text keeps it (a tag without children), as the .NET version compares
|
|
115
|
+
text = collapse(text_runs(value))
|
|
116
|
+
amp = text.index(Decode::AMP)
|
|
117
|
+
return text == expected if amp.nil?
|
|
118
|
+
|
|
119
|
+
# Text with entities must be decoded before comparing: the full text, when the part before it matches.
|
|
120
|
+
head = text.byteslice(0, amp)
|
|
121
|
+
head = head.byteslice(0, head.bytesize - 1) while head.end_with?(SP)
|
|
122
|
+
expected.start_with?(head) && n.text_bytes == expected
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# @api private Text with the visible line structure kept (HtmlNode#text_line_breaks).
|
|
127
|
+
class TextLines
|
|
128
|
+
BREAK_SPACE = 1 # table cell: word separator
|
|
129
|
+
BREAK_LINE = 2 # <br> and block elements (div, li, tr, ...): new line
|
|
130
|
+
BREAK_PARAGRAPH = 3 # p, h1-h6: new paragraph (blank line)
|
|
131
|
+
BREAK_SKIP_CONTENT = 4 # script, style, noscript: contents are not text
|
|
132
|
+
# inline tags (b, span, a, ...) are removed, words are not split
|
|
133
|
+
|
|
134
|
+
KEYS = TagNames::KEYS
|
|
135
|
+
BR = KEYS["br"]
|
|
136
|
+
|
|
137
|
+
KINDS = Array.new(TagNames::NAMES.size, 0)
|
|
138
|
+
%w[p h1 h2 h3 h4 h5 h6].each { |t| KINDS[KEYS[t]] = BREAK_PARAGRAPH }
|
|
139
|
+
%w[br div li ul ol dl dt dd table caption tr thead tbody tfoot section article aside nav header footer main address
|
|
140
|
+
blockquote pre hr figure figcaption form fieldset legend details summary dialog option html head body title
|
|
141
|
+
hgroup search menu center dir listing xmp plaintext multicol frameset marquee].each do |t|
|
|
142
|
+
KINDS[KEYS[t]] = BREAK_LINE
|
|
143
|
+
end
|
|
144
|
+
KINDS[KEYS["td"]] = BREAK_SPACE
|
|
145
|
+
KINDS[KEYS["th"]] = BREAK_SPACE
|
|
146
|
+
%w[script style noscript template noframes noembed].each { |t| KINDS[KEYS[t]] = BREAK_SKIP_CONTENT }
|
|
147
|
+
KINDS.freeze
|
|
148
|
+
|
|
149
|
+
SPACE_RUNS = Chars::SPACE_RUNS
|
|
150
|
+
SPACE_BYTE = Chars::SPACE_BYTE
|
|
151
|
+
PLAIN = /[^&<]*+/n
|
|
152
|
+
TAG_NAME_END = %r{[\t\r\n\f /]}n
|
|
153
|
+
|
|
154
|
+
attr_reader :out
|
|
155
|
+
|
|
156
|
+
def initialize
|
|
157
|
+
@out = String.new(encoding: Encoding::BINARY)
|
|
158
|
+
@pending_breaks = 0 # new lines requested since the last text char
|
|
159
|
+
@pending_space = false
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
# Writes text, collapsing whitespace and turning requested breaks into trimmed lines.
|
|
163
|
+
def chars(s)
|
|
164
|
+
unless SPACE_BYTE.match?(s)
|
|
165
|
+
text(s)
|
|
166
|
+
return
|
|
167
|
+
end
|
|
168
|
+
pos = 0
|
|
169
|
+
n = s.bytesize
|
|
170
|
+
while pos < n
|
|
171
|
+
i = s.index(SPACE_RUNS, pos)
|
|
172
|
+
if i.nil?
|
|
173
|
+
text(s.byteslice(pos, n - pos))
|
|
174
|
+
break
|
|
175
|
+
end
|
|
176
|
+
text(s.byteslice(pos, i - pos)) if i > pos
|
|
177
|
+
@pending_space = true
|
|
178
|
+
pos = i + $~[0].bytesize
|
|
179
|
+
end
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
def text(s)
|
|
183
|
+
unless @out.empty?
|
|
184
|
+
# nothing before the first text char: leading breaks and spaces are trimmed
|
|
185
|
+
if @pending_breaks > 0
|
|
186
|
+
@out << (@pending_breaks >= 2 ? "\n\n" : "\n") # at most one blank line
|
|
187
|
+
elsif @pending_space
|
|
188
|
+
@out << " "
|
|
189
|
+
end
|
|
190
|
+
end
|
|
191
|
+
@pending_breaks = 0
|
|
192
|
+
@pending_space = false
|
|
193
|
+
@out << s
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
def break!(kind, known)
|
|
197
|
+
if kind == BREAK_LINE
|
|
198
|
+
if known == BR
|
|
199
|
+
@pending_breaks += 1 # every <br> counts
|
|
200
|
+
elsif @pending_breaks < 1
|
|
201
|
+
@pending_breaks = 1
|
|
202
|
+
end
|
|
203
|
+
elsif @pending_breaks < 2
|
|
204
|
+
@pending_breaks = 2
|
|
205
|
+
end
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
# The text of an HTML fragment with the visible line structure kept. Lines are joined with "\n".
|
|
209
|
+
def self.clear_tags_line_breaks(value)
|
|
210
|
+
return +"".b if value.empty?
|
|
211
|
+
|
|
212
|
+
value = Decode.normalize_newlines(value) # values are read as a browser reads the page
|
|
213
|
+
w = new
|
|
214
|
+
n = value.bytesize
|
|
215
|
+
ss = StringScanner.new(value)
|
|
216
|
+
i = 0
|
|
217
|
+
while i < n
|
|
218
|
+
c = value.getbyte(i)
|
|
219
|
+
|
|
220
|
+
if c == 38
|
|
221
|
+
# '&': decoded chars go through the same whitespace/text logic ( collapses, < is plain text)
|
|
222
|
+
decoded, consumed = Decode.decode_reference(value, i + 1)
|
|
223
|
+
if decoded
|
|
224
|
+
w.chars(decoded)
|
|
225
|
+
i += 1 + consumed
|
|
226
|
+
else
|
|
227
|
+
w.chars(Decode::AMP)
|
|
228
|
+
i += 1
|
|
229
|
+
end
|
|
230
|
+
next
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
if c != 60 || !Parser.markup_start?(value, i, n)
|
|
234
|
+
# plain text up to the next '&' or '<'
|
|
235
|
+
ss.pos = i + 1
|
|
236
|
+
ss.skip(PLAIN)
|
|
237
|
+
j = ss.pos
|
|
238
|
+
w.chars(value.byteslice(i, j - i))
|
|
239
|
+
i = j
|
|
240
|
+
next
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
if i + 4 <= n && value.getbyte(i + 1) == 33 && value.getbyte(i + 2) == 45 && value.getbyte(i + 3) == 45
|
|
244
|
+
e = value.index("-->", i + 4)
|
|
245
|
+
break if e.nil?
|
|
246
|
+
|
|
247
|
+
i = e + 3
|
|
248
|
+
next
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
close_tag = value.index(Text::GT, i)
|
|
252
|
+
break if close_tag.nil? # unterminated tag: the rest is markup
|
|
253
|
+
|
|
254
|
+
tag = value.byteslice(i + 1, close_tag - i - 1)
|
|
255
|
+
i = close_tag + 1
|
|
256
|
+
|
|
257
|
+
closing = tag.getbyte(0) == 47
|
|
258
|
+
name = closing ? tag.byteslice(1, tag.bytesize - 1) : tag
|
|
259
|
+
stop = name.index(TAG_NAME_END)
|
|
260
|
+
name = name.byteslice(0, stop) if stop
|
|
261
|
+
known = Parser.lookup_tag(name)
|
|
262
|
+
next if known < 0 # unknown tag: just removed
|
|
263
|
+
|
|
264
|
+
case KINDS[known]
|
|
265
|
+
when BREAK_LINE, BREAK_PARAGRAPH
|
|
266
|
+
w.break!(KINDS[known], known)
|
|
267
|
+
when BREAK_SPACE
|
|
268
|
+
w.chars(Text::SP)
|
|
269
|
+
when BREAK_SKIP_CONTENT
|
|
270
|
+
if !closing && !Chars.trim_right_ascii(tag).end_with?("/")
|
|
271
|
+
# jump to the closing tag; it is removed on the next iteration
|
|
272
|
+
e = Chars.index_fold(value, "</#{TagNames::NAMES[known]}", i, n)
|
|
273
|
+
i = e < 0 ? n : e
|
|
274
|
+
end
|
|
275
|
+
end
|
|
276
|
+
end
|
|
277
|
+
w.out
|
|
278
|
+
end
|
|
279
|
+
end
|
|
280
|
+
end
|