fasthtmlparser 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.md +79 -0
- data/README.md +196 -0
- data/fasthtmlparser.gemspec +26 -0
- data/lib/fasthtmlparser/attr.rb +243 -0
- data/lib/fasthtmlparser/attrs.rb +278 -0
- data/lib/fasthtmlparser/chars.rb +258 -0
- data/lib/fasthtmlparser/css.rb +1061 -0
- data/lib/fasthtmlparser/decode.rb +164 -0
- data/lib/fasthtmlparser/entities.rb +2251 -0
- data/lib/fasthtmlparser/html.rb +52 -0
- data/lib/fasthtmlparser/html_document.rb +294 -0
- data/lib/fasthtmlparser/html_node.rb +706 -0
- data/lib/fasthtmlparser/links.rb +499 -0
- data/lib/fasthtmlparser/parser.rb +893 -0
- data/lib/fasthtmlparser/query.rb +221 -0
- data/lib/fasthtmlparser/tags.rb +761 -0
- data/lib/fasthtmlparser/text.rb +280 -0
- data/lib/fasthtmlparser/version.rb +5 -0
- data/lib/fasthtmlparser/xpath.rb +1151 -0
- data/lib/fasthtmlparser/xpath_compiler.rb +627 -0
- data/lib/fasthtmlparser.rb +35 -0
- metadata +66 -0
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FastHtmlParser
|
|
4
|
+
# An invalid CSS selector.
|
|
5
|
+
class CssSelectorError < ArgumentError; end
|
|
6
|
+
|
|
7
|
+
# An invalid XPath expression, or one that gives a number, string or boolean where tags are asked for.
|
|
8
|
+
class XPathError < ArgumentError; end
|
|
9
|
+
|
|
10
|
+
# Helpers for HTML fragments and lists of tags.
|
|
11
|
+
module Html
|
|
12
|
+
module_function
|
|
13
|
+
|
|
14
|
+
# The visible text of an HTML fragment, as HtmlNode#text gives it for a tag's content.
|
|
15
|
+
def to_text(html)
|
|
16
|
+
Chars.u(Text.clear_tags(html.to_s.b))
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# The text of an HTML fragment with the visible line structure kept, as HtmlNode#text_line_breaks.
|
|
20
|
+
def to_text_line_breaks(html)
|
|
21
|
+
Chars.u(TextLines.clear_tags_line_breaks(html.to_s.b))
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# Decodes the HTML character references in text (&amp;, &#169;, ...); CR LF and a lone CR become LF.
|
|
25
|
+
def decode(text)
|
|
26
|
+
Chars.u(Decode.decode(text.to_s.b, false))
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# The tags of the list at the level of the first one (the outermost tags of a find_all result).
|
|
30
|
+
def first_level(nodes)
|
|
31
|
+
level(nodes, nil)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# The tags of the list with the name, at the level of the first tag of the list.
|
|
35
|
+
def first_level_by_name(nodes, name)
|
|
36
|
+
level(nodes, Query.by(name))
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# The tags of the list with the name and the class, at the level of the first tag of the list.
|
|
40
|
+
def first_level_by_name_class(nodes, name, cls)
|
|
41
|
+
level(nodes, Query.by(name).has_class(cls))
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def level(nodes, query)
|
|
45
|
+
return [] if nodes.empty?
|
|
46
|
+
|
|
47
|
+
depth = nodes[0].depth
|
|
48
|
+
nodes.select { |n| n.depth == depth && (query.nil? || query.matches?(n)) }
|
|
49
|
+
end
|
|
50
|
+
private_class_method :level
|
|
51
|
+
end
|
|
52
|
+
end
|
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FastHtmlParser
|
|
4
|
+
# A parsed HTML or XML page: a flat list of tags in document order. Each tag holds only positions in the source;
|
|
5
|
+
# text, attributes and entities are read lazily, when they are asked for, and the values are substrings of the page.
|
|
6
|
+
# Documents are read-only and can be shared between threads.
|
|
7
|
+
#
|
|
8
|
+
# doc = FastHtmlParser::HtmlDocument.parse(html, "https://example.com/catalog/")
|
|
9
|
+
# doc.css("article.product_pod").each do |pod|
|
|
10
|
+
# title = pod.css_first("h3 a").attr("title")
|
|
11
|
+
# price = pod.find_by_name_class(:p, "price_color").text
|
|
12
|
+
# end
|
|
13
|
+
# doc.xpath_value("//dt[.='Price']/following-sibling::dd[1]")
|
|
14
|
+
class HtmlDocument
|
|
15
|
+
include Enumerable
|
|
16
|
+
|
|
17
|
+
BOM = "\xEF\xBB\xBF".b.freeze
|
|
18
|
+
|
|
19
|
+
# @api private The tags as parallel arrays, by index in document order: Tag key (-1 for a name that is not in
|
|
20
|
+
# Tag), custom lower-case name ("" for the others), parent index (-1 at the top level), depth, the opening '<',
|
|
21
|
+
# the last '>' (of the end tag when there is one), right after the opening tag and the last byte before the end
|
|
22
|
+
# tag (both 0 when the tag is not closed).
|
|
23
|
+
attr_reader :keys, :customs, :parents, :depths, :starts, :ends, :inner_starts, :inner_ends
|
|
24
|
+
|
|
25
|
+
# The number of tags (comments included).
|
|
26
|
+
attr_reader :size
|
|
27
|
+
|
|
28
|
+
# Parses an HTML or XML page given as a String (UTF-8 bytes in any String encoding; a leading byte order mark is
|
|
29
|
+
# skipped). With the page URL, relative links (HtmlNode#href, HtmlNode#src) are resolved against it.
|
|
30
|
+
def self.parse(html, url = "")
|
|
31
|
+
src = html.b
|
|
32
|
+
src = src.byteslice(3, src.bytesize - 3) if src.start_with?(BOM)
|
|
33
|
+
new(src.freeze, url)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# @api private Use HtmlDocument.parse.
|
|
37
|
+
def initialize(source, url)
|
|
38
|
+
@source = source
|
|
39
|
+
@url = Chars.b(url.to_s).dup.freeze
|
|
40
|
+
@keys, @customs, @parents, @depths, @starts, @ends, @inner_starts, @inner_ends = Parser.parse_tags(source)
|
|
41
|
+
@size = @keys.size
|
|
42
|
+
@made = Array.new(@size) # the tag objects made so far
|
|
43
|
+
@css_tree = nil
|
|
44
|
+
@base = nil
|
|
45
|
+
@base_parsed = false
|
|
46
|
+
@source_utf8 = nil
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# @api private The page source as binary bytes.
|
|
50
|
+
def source_bytes
|
|
51
|
+
@source
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# @api private The page URL as binary bytes.
|
|
55
|
+
def url_bytes
|
|
56
|
+
@url
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# @api private Sibling links and positions, built by the first selector that needs them.
|
|
60
|
+
def css_tree
|
|
61
|
+
@css_tree ||= CssTree.new(self)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# @api private The page URL parsed once, for resolving relative links.
|
|
65
|
+
def base
|
|
66
|
+
unless @base_parsed
|
|
67
|
+
@base = Links.absolute_url(@url)
|
|
68
|
+
@base_parsed = true
|
|
69
|
+
end
|
|
70
|
+
@base
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# The page source (UTF-8, frozen). Positions (HtmlNode#start, HtmlNode#end) are byte offsets into it:
|
|
74
|
+
# source.byteslice(tag.start, tag.end - tag.start + 1).
|
|
75
|
+
def source
|
|
76
|
+
@source_utf8 ||= @source.dup.force_encoding(Encoding::UTF_8).freeze
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# The page URL given to parse, or "".
|
|
80
|
+
def url
|
|
81
|
+
Chars.u(@url)
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# Every tag (comments included) in document order.
|
|
85
|
+
def nodes
|
|
86
|
+
Array.new(@size) { |i| @made[i] ||= HtmlNode.new(self, i) }
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
alias length size
|
|
90
|
+
|
|
91
|
+
# The number of tags; with an argument or a block, as Enumerable#count.
|
|
92
|
+
def count(*args, &block)
|
|
93
|
+
args.empty? && block.nil? ? @size : super
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def empty?
|
|
97
|
+
@size.zero?
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# The tag at index i in document order (negative: from the end), or nil when there is none.
|
|
101
|
+
def node(i)
|
|
102
|
+
i += @size if i.negative?
|
|
103
|
+
return nil if i.negative? || i >= @size
|
|
104
|
+
|
|
105
|
+
@made[i] ||= HtmlNode.new(self, i)
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
alias [] node
|
|
109
|
+
|
|
110
|
+
# @api private The tag at an index that exists.
|
|
111
|
+
def at(i)
|
|
112
|
+
@made[i] ||= HtmlNode.new(self, i)
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def each
|
|
116
|
+
return enum_for(:each) { @size } unless block_given?
|
|
117
|
+
|
|
118
|
+
i = 0
|
|
119
|
+
while i < @size
|
|
120
|
+
yield(@made[i] ||= HtmlNode.new(self, i))
|
|
121
|
+
i += 1
|
|
122
|
+
end
|
|
123
|
+
self
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def inspect
|
|
127
|
+
"#<#{self.class.name} #{@size} tags, #{@source.bytesize} bytes>"
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# ----- search -----
|
|
131
|
+
|
|
132
|
+
# The first tag in the document that matches the query, or nil.
|
|
133
|
+
def find(query)
|
|
134
|
+
query.find_in(self, 0, Float::INFINITY)
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# All tags in the document that match the query.
|
|
138
|
+
def find_all(query)
|
|
139
|
+
query.find_all_in(self, 0, Float::INFINITY)
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# The tags that match a CSS selector (as querySelectorAll). Raises CssSelectorError on an invalid selector.
|
|
143
|
+
def css(selector)
|
|
144
|
+
CssQuery.cached(selector).all(self)
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# The first tag that matches a CSS selector, or nil.
|
|
148
|
+
def css_first(selector)
|
|
149
|
+
CssQuery.cached(selector).first(self)
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
# The tags an XPath expression selects. Raises XPathError on an invalid expression or one that gives a number,
|
|
153
|
+
# string or boolean (use xpath_value).
|
|
154
|
+
def xpath(expression)
|
|
155
|
+
XPathQuery.cached(expression).nodes(self)
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
# The first tag an XPath expression selects, or nil.
|
|
159
|
+
def xpath_first(expression)
|
|
160
|
+
xpath(expression).first
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# The values an XPath expression selects: decoded attributes, text(), visible text of tags, or the result of
|
|
164
|
+
# count(), string(), ... as one value.
|
|
165
|
+
def xpath_values(expression)
|
|
166
|
+
XPathQuery.cached(expression).values(self)
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# The first value an XPath expression selects, or "".
|
|
170
|
+
def xpath_value(expression)
|
|
171
|
+
XPathQuery.cached(expression).value(self) || +""
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
# <shortcuts document>
|
|
175
|
+
# Generated by tools/gen.rb; do not edit by hand.
|
|
176
|
+
|
|
177
|
+
# Returns the first tag with the id in the document, or nil.
|
|
178
|
+
def find_by_id(id)
|
|
179
|
+
return nil if id.empty?
|
|
180
|
+
|
|
181
|
+
find(Query.any.id(id))
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# Returns the first tag with the name in the document, or nil.
|
|
185
|
+
def find_by_name(name)
|
|
186
|
+
find(Query.by(name))
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
# Returns the first tag with the name and the inner HTML in the document, or nil.
|
|
190
|
+
def find_by_name_body(name, body)
|
|
191
|
+
find(Query.by(name).body(body))
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
# Returns the first tag with the name and the decoded attribute value in the document, or nil.
|
|
195
|
+
def find_by_name_attribute(name, attribute, value)
|
|
196
|
+
find(Query.by(name).attr(attribute, value))
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
# Returns the first tag with the name and the class in the document, or nil.
|
|
200
|
+
def find_by_name_class(name, cls)
|
|
201
|
+
find(Query.by(name).has_class(cls))
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
# Returns the first tag with the name, the class and the inner HTML in the document, or nil.
|
|
205
|
+
def find_by_name_class_body(name, cls, body)
|
|
206
|
+
find(Query.by(name).has_class(cls).body(body))
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
# Returns the first tag with the name, and the class and visible text when they are not empty in the document, or nil.
|
|
210
|
+
def find_by_name_class_text(name, cls, text)
|
|
211
|
+
find(Query.opt_text(Query.opt_class(Query.by(name), cls), text))
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
# Returns the first tag with the name, and the id and visible text when they are not empty in the document, or nil.
|
|
215
|
+
def find_by_name_id_text(name, id, text)
|
|
216
|
+
find(Query.opt_text(Query.opt_id(Query.by(name), id), text))
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
# Returns the first tag with the name and the visible text in the document, or nil.
|
|
220
|
+
def find_by_name_text(name, text)
|
|
221
|
+
find(Query.by(name).text(text))
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
# Returns the first tag with the name and the id in the document, or nil.
|
|
225
|
+
def find_by_name_id(name, id)
|
|
226
|
+
find(Query.by(name).id(id))
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
# Returns the first tag with the name whose id starts with the prefix in the document, or nil.
|
|
230
|
+
def find_by_name_id_starts_with(name, id_prefix)
|
|
231
|
+
find(Query.by(name).id_starts_with(id_prefix))
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
# Returns the first tag with the name whose inner HTML starts with the prefix in the document, or nil.
|
|
235
|
+
def find_by_name_body_starts_with(name, body_prefix)
|
|
236
|
+
find(Query.by(name).body_starts_with(body_prefix))
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
# Returns the first tag with the name whose visible text starts with the prefix in the document, or nil.
|
|
240
|
+
def find_by_name_text_starts_with(name, text_prefix)
|
|
241
|
+
return nil if text_prefix.empty?
|
|
242
|
+
|
|
243
|
+
find(Query.by(name).text_starts_with(text_prefix))
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
# Returns the first tag with the name and the class whose visible text starts with the prefix in the document, or nil.
|
|
247
|
+
def find_by_name_class_text_starts_with(name, cls, text_prefix)
|
|
248
|
+
return nil if text_prefix.empty?
|
|
249
|
+
|
|
250
|
+
find(Query.by(name).has_class(cls).text_starts_with(text_prefix))
|
|
251
|
+
end
|
|
252
|
+
|
|
253
|
+
# Returns the first tag with the name and the itemprop in the document, or nil.
|
|
254
|
+
def find_by_name_itemprop(name, itemprop)
|
|
255
|
+
find(Query.by(name).attr(Attr::ITEMPROP, itemprop))
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
# Returns all tags with the name in the document.
|
|
259
|
+
def find_all_by_name(name)
|
|
260
|
+
find_all(Query.by(name))
|
|
261
|
+
end
|
|
262
|
+
|
|
263
|
+
# Returns all tags with the name and the decoded attribute value in the document.
|
|
264
|
+
def find_all_by_name_attribute(name, attribute, value)
|
|
265
|
+
find_all(Query.by(name).attr(attribute, value))
|
|
266
|
+
end
|
|
267
|
+
|
|
268
|
+
# Returns all tags with the name and the class in the document.
|
|
269
|
+
def find_all_by_name_class(name, cls)
|
|
270
|
+
find_all(Query.by(name).has_class(cls))
|
|
271
|
+
end
|
|
272
|
+
|
|
273
|
+
# Returns all tags with the name and the id in the document.
|
|
274
|
+
def find_all_by_name_id(name, id)
|
|
275
|
+
find_all(Query.by(name).id(id))
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
# Returns all tags with the name whose id starts with the prefix in the document.
|
|
279
|
+
def find_all_by_name_id_starts_with(name, id_prefix)
|
|
280
|
+
find_all(Query.by(name).id_starts_with(id_prefix))
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
# Returns all tags with the name and the itemprop in the document.
|
|
284
|
+
def find_all_by_name_itemprop(name, itemprop)
|
|
285
|
+
find_all(Query.by(name).attr(Attr::ITEMPROP, itemprop))
|
|
286
|
+
end
|
|
287
|
+
|
|
288
|
+
# Returns all tags with the name whose inner HTML starts with the prefix in the document.
|
|
289
|
+
def find_all_by_name_body_starts_with(name, body_prefix)
|
|
290
|
+
find_all(Query.by(name).body_starts_with(body_prefix))
|
|
291
|
+
end
|
|
292
|
+
# </shortcuts document>
|
|
293
|
+
end
|
|
294
|
+
end
|