fasthtmlparser 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.md +79 -0
- data/README.md +196 -0
- data/fasthtmlparser.gemspec +26 -0
- data/lib/fasthtmlparser/attr.rb +243 -0
- data/lib/fasthtmlparser/attrs.rb +278 -0
- data/lib/fasthtmlparser/chars.rb +258 -0
- data/lib/fasthtmlparser/css.rb +1061 -0
- data/lib/fasthtmlparser/decode.rb +164 -0
- data/lib/fasthtmlparser/entities.rb +2251 -0
- data/lib/fasthtmlparser/html.rb +52 -0
- data/lib/fasthtmlparser/html_document.rb +294 -0
- data/lib/fasthtmlparser/html_node.rb +706 -0
- data/lib/fasthtmlparser/links.rb +499 -0
- data/lib/fasthtmlparser/parser.rb +893 -0
- data/lib/fasthtmlparser/query.rb +221 -0
- data/lib/fasthtmlparser/tags.rb +761 -0
- data/lib/fasthtmlparser/text.rb +280 -0
- data/lib/fasthtmlparser/version.rb +5 -0
- data/lib/fasthtmlparser/xpath.rb +1151 -0
- data/lib/fasthtmlparser/xpath_compiler.rb +627 -0
- data/lib/fasthtmlparser.rb +35 -0
- metadata +66 -0
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FastHtmlParser
|
|
4
|
+
# Selects tags by name and, optionally, class, id, attribute value, visible text, inner HTML and the source of the
|
|
5
|
+
# opening tag. Queries are immutable: every method returns a new one. Start with Query.by(:div),
|
|
6
|
+
# Query.by("my-widget") or Query.any.
|
|
7
|
+
#
|
|
8
|
+
# doc.find_all(Query.by(:article).has_class("product_pod"))
|
|
9
|
+
# pod.find(Query.by(:p).has_class("price_color")).text
|
|
10
|
+
class Query
|
|
11
|
+
SYMBOL_KEYS = TagNames::NAMES.each_with_index.to_h { |n, i| [n.to_sym, i] }.freeze
|
|
12
|
+
|
|
13
|
+
# @api private Tag key, or -1: by name (tag_name) or any (tag_name "")
|
|
14
|
+
attr_reader :key
|
|
15
|
+
# @api private the name as given (binary); "" for a known Symbol and for any
|
|
16
|
+
attr_reader :tag_name
|
|
17
|
+
|
|
18
|
+
# @api private Use Query.by or Query.any.
|
|
19
|
+
def initialize(key, tag_name, plain, cls, id, attr_name, attr_value, text, id_starts_with, text_starts_with, body,
|
|
20
|
+
body_starts_with, source)
|
|
21
|
+
@key = key
|
|
22
|
+
@tag_name = tag_name
|
|
23
|
+
@class = cls
|
|
24
|
+
@id = id
|
|
25
|
+
@attr_name = attr_name
|
|
26
|
+
@attr_value = attr_value
|
|
27
|
+
@text = text
|
|
28
|
+
@id_starts_with = id_starts_with
|
|
29
|
+
@text_starts_with = text_starts_with
|
|
30
|
+
@body = body
|
|
31
|
+
@body_starts_with = body_starts_with
|
|
32
|
+
@source = source
|
|
33
|
+
@plain = plain # no criteria but the name
|
|
34
|
+
freeze
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Selects tags by name, case-insensitively: a Tag symbol (:div), or any name as a String or Symbol.
|
|
38
|
+
def self.by(name)
|
|
39
|
+
if name.is_a?(Symbol)
|
|
40
|
+
q = BY_SYMBOL[name]
|
|
41
|
+
return q if q
|
|
42
|
+
|
|
43
|
+
name = name.to_s
|
|
44
|
+
end
|
|
45
|
+
n = Query.value(name)
|
|
46
|
+
new(Parser.lookup_tag(n), n, true, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
ANY = new(-1, Chars::EMPTY, true, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
|
|
50
|
+
BY_SYMBOL = SYMBOL_KEYS.transform_values do |k|
|
|
51
|
+
new(k, Chars::EMPTY, true, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
|
|
52
|
+
end.freeze
|
|
53
|
+
|
|
54
|
+
# Selects every tag.
|
|
55
|
+
def self.any
|
|
56
|
+
ANY
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# @api private A criterion as the library compares it: frozen, binary when it is not ASCII.
|
|
60
|
+
def self.value(s)
|
|
61
|
+
return s if s.frozen? && s.ascii_only?
|
|
62
|
+
|
|
63
|
+
s = s.to_s unless s.is_a?(String)
|
|
64
|
+
return s.dup.freeze if s.ascii_only?
|
|
65
|
+
|
|
66
|
+
s.b.freeze
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Adds a class criterion: one of the space-separated classes, case-insensitive.
|
|
70
|
+
def has_class(cls)
|
|
71
|
+
Query.new(@key, @tag_name, false, Query.value(cls), @id, @attr_name, @attr_value, @text, @id_starts_with,
|
|
72
|
+
@text_starts_with, @body, @body_starts_with, @source)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# Adds an id criterion.
|
|
76
|
+
def id(id)
|
|
77
|
+
Query.new(@key, @tag_name, false, @class, Query.value(id), @attr_name, @attr_value, @text, @id_starts_with,
|
|
78
|
+
@text_starts_with, @body, @body_starts_with, @source)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# Adds an attribute criterion: the decoded value equals value.
|
|
82
|
+
def attr(name, value)
|
|
83
|
+
Query.new(@key, @tag_name, false, @class, @id, Query.value(name).downcase(:ascii).freeze, Query.value(value), @text,
|
|
84
|
+
@id_starts_with, @text_starts_with, @body, @body_starts_with, @source)
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Adds a visible-text criterion (see HtmlNode#text).
|
|
88
|
+
def text(text)
|
|
89
|
+
Query.new(@key, @tag_name, false, @class, @id, @attr_name, @attr_value, Query.value(text), @id_starts_with,
|
|
90
|
+
@text_starts_with, @body, @body_starts_with, @source)
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Adds a criterion: the decoded id starts with prefix.
|
|
94
|
+
def id_starts_with(prefix)
|
|
95
|
+
Query.new(@key, @tag_name, false, @class, @id, @attr_name, @attr_value, @text, Query.value(prefix),
|
|
96
|
+
@text_starts_with, @body, @body_starts_with, @source)
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# Adds a criterion: the visible text starts with prefix.
|
|
100
|
+
def text_starts_with(prefix)
|
|
101
|
+
Query.new(@key, @tag_name, false, @class, @id, @attr_name, @attr_value, @text, @id_starts_with, Query.value(prefix),
|
|
102
|
+
@body, @body_starts_with, @source)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Adds a criterion: the inner HTML (trimmed) equals body.
|
|
106
|
+
def body(body)
|
|
107
|
+
Query.new(@key, @tag_name, false, @class, @id, @attr_name, @attr_value, @text, @id_starts_with, @text_starts_with,
|
|
108
|
+
Query.value(body), @body_starts_with, @source)
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# Adds a criterion: the inner HTML (trimmed) starts with prefix.
|
|
112
|
+
def body_starts_with(prefix)
|
|
113
|
+
Query.new(@key, @tag_name, false, @class, @id, @attr_name, @attr_value, @text, @id_starts_with, @text_starts_with,
|
|
114
|
+
@body, Query.value(prefix), @source)
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# Adds a criterion: the opening tag without '<' and '>' (see HtmlNode#source) equals source.
|
|
118
|
+
def source(source)
|
|
119
|
+
Query.new(@key, @tag_name, false, @class, @id, @attr_name, @attr_value, @text, @id_starts_with, @text_starts_with,
|
|
120
|
+
@body, @body_starts_with, Query.value(source))
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# Whether the tag matches every criterion; false for nil.
|
|
124
|
+
def matches?(n)
|
|
125
|
+
!n.nil? && name_matches_at?(n.doc, n.index) && (@plain || matches_more?(n))
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# @api private The first tag from index from on that matches, up to the tags that end after end_; or nil.
|
|
129
|
+
def find_in(d, from, end_)
|
|
130
|
+
scan(d, from, end_, true).first
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# @api private All tags from index from on that match, up to the tags that end after end_.
|
|
134
|
+
def find_all_in(d, from, end_)
|
|
135
|
+
scan(d, from, end_, false)
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# @api private Whether the tag at index i has the name of the query.
|
|
139
|
+
def name_matches_at?(d, i)
|
|
140
|
+
if @key >= 0
|
|
141
|
+
k = d.keys[i]
|
|
142
|
+
return k == @key || (k < 0 && !@tag_name.empty? && Chars.equal_fold(d.customs[i], @tag_name))
|
|
143
|
+
end
|
|
144
|
+
return true if @tag_name.empty?
|
|
145
|
+
|
|
146
|
+
k = d.keys[i]
|
|
147
|
+
Chars.equal_fold(k >= 0 ? TagNames::NAMES[k] : d.customs[i], @tag_name)
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
# @api private The older "optional" shortcuts skip an empty criterion.
|
|
151
|
+
def self.opt_class(q, cls)
|
|
152
|
+
cls.empty? ? q : q.has_class(cls)
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# @api private
|
|
156
|
+
def self.opt_id(q, id)
|
|
157
|
+
id.empty? ? q : q.id(id)
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
# @api private
|
|
161
|
+
def self.opt_text(q, text)
|
|
162
|
+
text.empty? ? q : q.text(text)
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
# @api private
|
|
166
|
+
def self.opt_body(q, body)
|
|
167
|
+
body.empty? ? q : q.body(body)
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
private
|
|
171
|
+
|
|
172
|
+
def scan(d, from, end_, first_only)
|
|
173
|
+
out = []
|
|
174
|
+
ends = d.ends
|
|
175
|
+
keys = d.keys
|
|
176
|
+
key = @key
|
|
177
|
+
plain = @plain
|
|
178
|
+
named = !@tag_name.empty?
|
|
179
|
+
count = d.size
|
|
180
|
+
i = from
|
|
181
|
+
while i < count
|
|
182
|
+
break if ends[i] > end_
|
|
183
|
+
|
|
184
|
+
# the name on the arrays first (a known name: one int compare), the tag object only for a candidate
|
|
185
|
+
if key >= 0
|
|
186
|
+
k = keys[i]
|
|
187
|
+
if k != key && (k >= 0 || !named || !Chars.equal_fold(d.customs[i], @tag_name))
|
|
188
|
+
i += 1
|
|
189
|
+
next
|
|
190
|
+
end
|
|
191
|
+
elsif named && !name_matches_at?(d, i)
|
|
192
|
+
i += 1
|
|
193
|
+
next
|
|
194
|
+
end
|
|
195
|
+
n = d.at(i)
|
|
196
|
+
if plain || matches_more?(n)
|
|
197
|
+
out << n
|
|
198
|
+
break if first_only
|
|
199
|
+
end
|
|
200
|
+
i += 1
|
|
201
|
+
end
|
|
202
|
+
out
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
# The criteria other than the name.
|
|
206
|
+
def matches_more?(n)
|
|
207
|
+
return false if @id && (@id.empty? || !Attrs.attr_equals?(Attrs.attr_raw(n, Attrs::ID), @id))
|
|
208
|
+
return false if @attr_name && !Attrs.attr_equals?(Attrs.attr_raw(n, @attr_name), @attr_value)
|
|
209
|
+
return false if @class && !Attrs.has_class?(n, @class)
|
|
210
|
+
return false if @id_starts_with && !Attrs.attr_starts_with?(Attrs.attr_raw(n, Attrs::ID), @id_starts_with)
|
|
211
|
+
return false if @source && n.source_bytes != @source
|
|
212
|
+
return false if @body && n.body_bytes != @body
|
|
213
|
+
return false if @body_starts_with && !n.body_bytes.start_with?(@body_starts_with)
|
|
214
|
+
# the text last: it is the most expensive check
|
|
215
|
+
return false if @text && !Text.text?(n.body_bytes, @text, n)
|
|
216
|
+
return false if @text_starts_with && !n.text_bytes.start_with?(@text_starts_with)
|
|
217
|
+
|
|
218
|
+
true
|
|
219
|
+
end
|
|
220
|
+
end
|
|
221
|
+
end
|