fasthtmlparser 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,52 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FastHtmlParser
4
+ # An invalid CSS selector.
5
+ class CssSelectorError < ArgumentError; end
6
+
7
+ # An invalid XPath expression, or one that gives a number, string or boolean where tags are asked for.
8
+ class XPathError < ArgumentError; end
9
+
10
+ # Helpers for HTML fragments and lists of tags.
11
+ module Html
12
+ module_function
13
+
14
+ # The visible text of an HTML fragment, as HtmlNode#text gives it for a tag's content.
15
+ def to_text(html)
16
+ Chars.u(Text.clear_tags(html.to_s.b))
17
+ end
18
+
19
+ # The text of an HTML fragment with the visible line structure kept, as HtmlNode#text_line_breaks.
20
+ def to_text_line_breaks(html)
21
+ Chars.u(TextLines.clear_tags_line_breaks(html.to_s.b))
22
+ end
23
+
24
+ # Decodes the HTML character references in text (&amp;amp;, &amp;#169;, ...); CR LF and a lone CR become LF.
25
+ def decode(text)
26
+ Chars.u(Decode.decode(text.to_s.b, false))
27
+ end
28
+
29
+ # The tags of the list at the level of the first one (the outermost tags of a find_all result).
30
+ def first_level(nodes)
31
+ level(nodes, nil)
32
+ end
33
+
34
+ # The tags of the list with the name, at the level of the first tag of the list.
35
+ def first_level_by_name(nodes, name)
36
+ level(nodes, Query.by(name))
37
+ end
38
+
39
+ # The tags of the list with the name and the class, at the level of the first tag of the list.
40
+ def first_level_by_name_class(nodes, name, cls)
41
+ level(nodes, Query.by(name).has_class(cls))
42
+ end
43
+
44
+ def level(nodes, query)
45
+ return [] if nodes.empty?
46
+
47
+ depth = nodes[0].depth
48
+ nodes.select { |n| n.depth == depth && (query.nil? || query.matches?(n)) }
49
+ end
50
+ private_class_method :level
51
+ end
52
+ end
@@ -0,0 +1,294 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FastHtmlParser
4
+ # A parsed HTML or XML page: a flat list of tags in document order. Each tag holds only positions in the source;
5
+ # text, attributes and entities are read lazily, when they are asked for, and the values are substrings of the page.
6
+ # Documents are read-only and can be shared between threads.
7
+ #
8
+ # doc = FastHtmlParser::HtmlDocument.parse(html, "https://example.com/catalog/")
9
+ # doc.css("article.product_pod").each do |pod|
10
+ # title = pod.css_first("h3 a").attr("title")
11
+ # price = pod.find_by_name_class(:p, "price_color").text
12
+ # end
13
+ # doc.xpath_value("//dt[.='Price']/following-sibling::dd[1]")
14
+ class HtmlDocument
15
+ include Enumerable
16
+
17
+ BOM = "\xEF\xBB\xBF".b.freeze
18
+
19
+ # @api private The tags as parallel arrays, by index in document order: Tag key (-1 for a name that is not in
20
+ # Tag), custom lower-case name ("" for the others), parent index (-1 at the top level), depth, the opening '<',
21
+ # the last '>' (of the end tag when there is one), right after the opening tag and the last byte before the end
22
+ # tag (both 0 when the tag is not closed).
23
+ attr_reader :keys, :customs, :parents, :depths, :starts, :ends, :inner_starts, :inner_ends
24
+
25
+ # The number of tags (comments included).
26
+ attr_reader :size
27
+
28
+ # Parses an HTML or XML page given as a String (UTF-8 bytes in any String encoding; a leading byte order mark is
29
+ # skipped). With the page URL, relative links (HtmlNode#href, HtmlNode#src) are resolved against it.
30
+ def self.parse(html, url = "")
31
+ src = html.b
32
+ src = src.byteslice(3, src.bytesize - 3) if src.start_with?(BOM)
33
+ new(src.freeze, url)
34
+ end
35
+
36
+ # @api private Use HtmlDocument.parse.
37
+ def initialize(source, url)
38
+ @source = source
39
+ @url = Chars.b(url.to_s).dup.freeze
40
+ @keys, @customs, @parents, @depths, @starts, @ends, @inner_starts, @inner_ends = Parser.parse_tags(source)
41
+ @size = @keys.size
42
+ @made = Array.new(@size) # the tag objects made so far
43
+ @css_tree = nil
44
+ @base = nil
45
+ @base_parsed = false
46
+ @source_utf8 = nil
47
+ end
48
+
49
+ # @api private The page source as binary bytes.
50
+ def source_bytes
51
+ @source
52
+ end
53
+
54
+ # @api private The page URL as binary bytes.
55
+ def url_bytes
56
+ @url
57
+ end
58
+
59
+ # @api private Sibling links and positions, built by the first selector that needs them.
60
+ def css_tree
61
+ @css_tree ||= CssTree.new(self)
62
+ end
63
+
64
+ # @api private The page URL parsed once, for resolving relative links.
65
+ def base
66
+ unless @base_parsed
67
+ @base = Links.absolute_url(@url)
68
+ @base_parsed = true
69
+ end
70
+ @base
71
+ end
72
+
73
+ # The page source (UTF-8, frozen). Positions (HtmlNode#start, HtmlNode#end) are byte offsets into it:
74
+ # source.byteslice(tag.start, tag.end - tag.start + 1).
75
+ def source
76
+ @source_utf8 ||= @source.dup.force_encoding(Encoding::UTF_8).freeze
77
+ end
78
+
79
+ # The page URL given to parse, or "".
80
+ def url
81
+ Chars.u(@url)
82
+ end
83
+
84
+ # Every tag (comments included) in document order.
85
+ def nodes
86
+ Array.new(@size) { |i| @made[i] ||= HtmlNode.new(self, i) }
87
+ end
88
+
89
+ alias length size
90
+
91
+ # The number of tags; with an argument or a block, as Enumerable#count.
92
+ def count(*args, &block)
93
+ args.empty? && block.nil? ? @size : super
94
+ end
95
+
96
+ def empty?
97
+ @size.zero?
98
+ end
99
+
100
+ # The tag at index i in document order (negative: from the end), or nil when there is none.
101
+ def node(i)
102
+ i += @size if i.negative?
103
+ return nil if i.negative? || i >= @size
104
+
105
+ @made[i] ||= HtmlNode.new(self, i)
106
+ end
107
+
108
+ alias [] node
109
+
110
+ # @api private The tag at an index that exists.
111
+ def at(i)
112
+ @made[i] ||= HtmlNode.new(self, i)
113
+ end
114
+
115
+ def each
116
+ return enum_for(:each) { @size } unless block_given?
117
+
118
+ i = 0
119
+ while i < @size
120
+ yield(@made[i] ||= HtmlNode.new(self, i))
121
+ i += 1
122
+ end
123
+ self
124
+ end
125
+
126
+ def inspect
127
+ "#<#{self.class.name} #{@size} tags, #{@source.bytesize} bytes>"
128
+ end
129
+
130
+ # ----- search -----
131
+
132
+ # The first tag in the document that matches the query, or nil.
133
+ def find(query)
134
+ query.find_in(self, 0, Float::INFINITY)
135
+ end
136
+
137
+ # All tags in the document that match the query.
138
+ def find_all(query)
139
+ query.find_all_in(self, 0, Float::INFINITY)
140
+ end
141
+
142
+ # The tags that match a CSS selector (as querySelectorAll). Raises CssSelectorError on an invalid selector.
143
+ def css(selector)
144
+ CssQuery.cached(selector).all(self)
145
+ end
146
+
147
+ # The first tag that matches a CSS selector, or nil.
148
+ def css_first(selector)
149
+ CssQuery.cached(selector).first(self)
150
+ end
151
+
152
+ # The tags an XPath expression selects. Raises XPathError on an invalid expression or one that gives a number,
153
+ # string or boolean (use xpath_value).
154
+ def xpath(expression)
155
+ XPathQuery.cached(expression).nodes(self)
156
+ end
157
+
158
+ # The first tag an XPath expression selects, or nil.
159
+ def xpath_first(expression)
160
+ xpath(expression).first
161
+ end
162
+
163
+ # The values an XPath expression selects: decoded attributes, text(), visible text of tags, or the result of
164
+ # count(), string(), ... as one value.
165
+ def xpath_values(expression)
166
+ XPathQuery.cached(expression).values(self)
167
+ end
168
+
169
+ # The first value an XPath expression selects, or "".
170
+ def xpath_value(expression)
171
+ XPathQuery.cached(expression).value(self) || +""
172
+ end
173
+
174
+ # <shortcuts document>
175
+ # Generated by tools/gen.rb; do not edit by hand.
176
+
177
+ # Returns the first tag with the id in the document, or nil.
178
+ def find_by_id(id)
179
+ return nil if id.empty?
180
+
181
+ find(Query.any.id(id))
182
+ end
183
+
184
+ # Returns the first tag with the name in the document, or nil.
185
+ def find_by_name(name)
186
+ find(Query.by(name))
187
+ end
188
+
189
+ # Returns the first tag with the name and the inner HTML in the document, or nil.
190
+ def find_by_name_body(name, body)
191
+ find(Query.by(name).body(body))
192
+ end
193
+
194
+ # Returns the first tag with the name and the decoded attribute value in the document, or nil.
195
+ def find_by_name_attribute(name, attribute, value)
196
+ find(Query.by(name).attr(attribute, value))
197
+ end
198
+
199
+ # Returns the first tag with the name and the class in the document, or nil.
200
+ def find_by_name_class(name, cls)
201
+ find(Query.by(name).has_class(cls))
202
+ end
203
+
204
+ # Returns the first tag with the name, the class and the inner HTML in the document, or nil.
205
+ def find_by_name_class_body(name, cls, body)
206
+ find(Query.by(name).has_class(cls).body(body))
207
+ end
208
+
209
+ # Returns the first tag with the name, and the class and visible text when they are not empty in the document, or nil.
210
+ def find_by_name_class_text(name, cls, text)
211
+ find(Query.opt_text(Query.opt_class(Query.by(name), cls), text))
212
+ end
213
+
214
+ # Returns the first tag with the name, and the id and visible text when they are not empty in the document, or nil.
215
+ def find_by_name_id_text(name, id, text)
216
+ find(Query.opt_text(Query.opt_id(Query.by(name), id), text))
217
+ end
218
+
219
+ # Returns the first tag with the name and the visible text in the document, or nil.
220
+ def find_by_name_text(name, text)
221
+ find(Query.by(name).text(text))
222
+ end
223
+
224
+ # Returns the first tag with the name and the id in the document, or nil.
225
+ def find_by_name_id(name, id)
226
+ find(Query.by(name).id(id))
227
+ end
228
+
229
+ # Returns the first tag with the name whose id starts with the prefix in the document, or nil.
230
+ def find_by_name_id_starts_with(name, id_prefix)
231
+ find(Query.by(name).id_starts_with(id_prefix))
232
+ end
233
+
234
+ # Returns the first tag with the name whose inner HTML starts with the prefix in the document, or nil.
235
+ def find_by_name_body_starts_with(name, body_prefix)
236
+ find(Query.by(name).body_starts_with(body_prefix))
237
+ end
238
+
239
+ # Returns the first tag with the name whose visible text starts with the prefix in the document, or nil.
240
+ def find_by_name_text_starts_with(name, text_prefix)
241
+ return nil if text_prefix.empty?
242
+
243
+ find(Query.by(name).text_starts_with(text_prefix))
244
+ end
245
+
246
+ # Returns the first tag with the name and the class whose visible text starts with the prefix in the document, or nil.
247
+ def find_by_name_class_text_starts_with(name, cls, text_prefix)
248
+ return nil if text_prefix.empty?
249
+
250
+ find(Query.by(name).has_class(cls).text_starts_with(text_prefix))
251
+ end
252
+
253
+ # Returns the first tag with the name and the itemprop in the document, or nil.
254
+ def find_by_name_itemprop(name, itemprop)
255
+ find(Query.by(name).attr(Attr::ITEMPROP, itemprop))
256
+ end
257
+
258
+ # Returns all tags with the name in the document.
259
+ def find_all_by_name(name)
260
+ find_all(Query.by(name))
261
+ end
262
+
263
+ # Returns all tags with the name and the decoded attribute value in the document.
264
+ def find_all_by_name_attribute(name, attribute, value)
265
+ find_all(Query.by(name).attr(attribute, value))
266
+ end
267
+
268
+ # Returns all tags with the name and the class in the document.
269
+ def find_all_by_name_class(name, cls)
270
+ find_all(Query.by(name).has_class(cls))
271
+ end
272
+
273
+ # Returns all tags with the name and the id in the document.
274
+ def find_all_by_name_id(name, id)
275
+ find_all(Query.by(name).id(id))
276
+ end
277
+
278
+ # Returns all tags with the name whose id starts with the prefix in the document.
279
+ def find_all_by_name_id_starts_with(name, id_prefix)
280
+ find_all(Query.by(name).id_starts_with(id_prefix))
281
+ end
282
+
283
+ # Returns all tags with the name and the itemprop in the document.
284
+ def find_all_by_name_itemprop(name, itemprop)
285
+ find_all(Query.by(name).attr(Attr::ITEMPROP, itemprop))
286
+ end
287
+
288
+ # Returns all tags with the name whose inner HTML starts with the prefix in the document.
289
+ def find_all_by_name_body_starts_with(name, body_prefix)
290
+ find_all(Query.by(name).body_starts_with(body_prefix))
291
+ end
292
+ # </shortcuts document>
293
+ end
294
+ end