fasthtmlparser 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.md +79 -0
- data/README.md +196 -0
- data/fasthtmlparser.gemspec +26 -0
- data/lib/fasthtmlparser/attr.rb +243 -0
- data/lib/fasthtmlparser/attrs.rb +278 -0
- data/lib/fasthtmlparser/chars.rb +258 -0
- data/lib/fasthtmlparser/css.rb +1061 -0
- data/lib/fasthtmlparser/decode.rb +164 -0
- data/lib/fasthtmlparser/entities.rb +2251 -0
- data/lib/fasthtmlparser/html.rb +52 -0
- data/lib/fasthtmlparser/html_document.rb +294 -0
- data/lib/fasthtmlparser/html_node.rb +706 -0
- data/lib/fasthtmlparser/links.rb +499 -0
- data/lib/fasthtmlparser/parser.rb +893 -0
- data/lib/fasthtmlparser/query.rb +221 -0
- data/lib/fasthtmlparser/tags.rb +761 -0
- data/lib/fasthtmlparser/text.rb +280 -0
- data/lib/fasthtmlparser/version.rb +5 -0
- data/lib/fasthtmlparser/xpath.rb +1151 -0
- data/lib/fasthtmlparser/xpath_compiler.rb +627 -0
- data/lib/fasthtmlparser.rb +35 -0
- metadata +66 -0
|
@@ -0,0 +1,627 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FastHtmlParser
|
|
4
|
+
# @api private The tokenizer and parser of XPath 1.0 expressions (section 3 of the spec). Positions in error
|
|
5
|
+
# messages are in chars.
|
|
6
|
+
class XPathCompiler
|
|
7
|
+
T_END = 0
|
|
8
|
+
T_SLASH = 1
|
|
9
|
+
T_DOUBLE_SLASH = 2
|
|
10
|
+
T_LPAREN = 3
|
|
11
|
+
T_RPAREN = 4
|
|
12
|
+
T_LBRACKET = 5
|
|
13
|
+
T_RBRACKET = 6
|
|
14
|
+
T_DOT = 7
|
|
15
|
+
T_DOT_DOT = 8
|
|
16
|
+
T_AT = 9
|
|
17
|
+
T_COMMA = 10
|
|
18
|
+
T_COLON_COLON = 11
|
|
19
|
+
T_PIPE = 12
|
|
20
|
+
T_PLUS = 13
|
|
21
|
+
T_MINUS = 14
|
|
22
|
+
T_EQ = 15
|
|
23
|
+
T_NEQ = 16
|
|
24
|
+
T_LT = 17
|
|
25
|
+
T_LE = 18
|
|
26
|
+
T_GT = 19
|
|
27
|
+
T_GE = 20
|
|
28
|
+
T_STAR = 21 # name test *
|
|
29
|
+
T_MULTIPLY = 22 # operator *
|
|
30
|
+
T_AND = 23
|
|
31
|
+
T_OR = 24
|
|
32
|
+
T_DIV = 25
|
|
33
|
+
T_MOD = 26
|
|
34
|
+
T_NAME = 27
|
|
35
|
+
T_LITERAL = 28
|
|
36
|
+
T_NUMBER = 29
|
|
37
|
+
T_DOLLAR = 30
|
|
38
|
+
|
|
39
|
+
X = XPathQuery
|
|
40
|
+
|
|
41
|
+
Token = Struct.new(:k, :t, :n, :p)
|
|
42
|
+
|
|
43
|
+
SINGLE = { "(" => T_LPAREN, ")" => T_RPAREN, "[" => T_LBRACKET, "]" => T_RBRACKET, "@" => T_AT, "," => T_COMMA,
|
|
44
|
+
"|" => T_PIPE, "+" => T_PLUS, "-" => T_MINUS, "=" => T_EQ, "$" => T_DOLLAR }.freeze
|
|
45
|
+
|
|
46
|
+
NODE_TYPES = { "node" => true, "text" => true, "comment" => true, "processing-instruction" => true }.freeze
|
|
47
|
+
OPERATORS = { "and" => T_AND, "or" => T_OR, "div" => T_DIV, "mod" => T_MOD }.freeze
|
|
48
|
+
OPERAND_ENDS = [T_RPAREN, T_RBRACKET, T_DOT, T_DOT_DOT, T_STAR, T_NAME, T_LITERAL, T_NUMBER].freeze
|
|
49
|
+
AXES = {
|
|
50
|
+
"child" => X::AXIS_CHILD,
|
|
51
|
+
"descendant" => X::AXIS_DESCENDANT,
|
|
52
|
+
"descendant-or-self" => X::AXIS_DESCENDANT_OR_SELF,
|
|
53
|
+
"self" => X::AXIS_SELF,
|
|
54
|
+
"parent" => X::AXIS_PARENT,
|
|
55
|
+
"ancestor" => X::AXIS_ANCESTOR,
|
|
56
|
+
"ancestor-or-self" => X::AXIS_ANCESTOR_OR_SELF,
|
|
57
|
+
"following-sibling" => X::AXIS_FOLLOWING_SIBLING,
|
|
58
|
+
"preceding-sibling" => X::AXIS_PRECEDING_SIBLING,
|
|
59
|
+
"following" => X::AXIS_FOLLOWING,
|
|
60
|
+
"preceding" => X::AXIS_PRECEDING,
|
|
61
|
+
"attribute" => X::AXIS_ATTRIBUTE
|
|
62
|
+
}.freeze
|
|
63
|
+
|
|
64
|
+
MANY = 1 << 30
|
|
65
|
+
|
|
66
|
+
# name => [min args, max args, type]
|
|
67
|
+
FUNCTIONS = {
|
|
68
|
+
"last" => [0, 0, X::TYPE_NUMBER],
|
|
69
|
+
"position" => [0, 0, X::TYPE_NUMBER],
|
|
70
|
+
"count" => [1, 1, X::TYPE_NUMBER],
|
|
71
|
+
"name" => [0, 1, X::TYPE_STRING],
|
|
72
|
+
"local-name" => [0, 1, X::TYPE_STRING],
|
|
73
|
+
"string" => [0, 1, X::TYPE_STRING],
|
|
74
|
+
"concat" => [2, MANY, X::TYPE_STRING],
|
|
75
|
+
"starts-with" => [2, 2, X::TYPE_BOOLEAN],
|
|
76
|
+
"ends-with" => [2, 2, X::TYPE_BOOLEAN],
|
|
77
|
+
"contains" => [2, 2, X::TYPE_BOOLEAN],
|
|
78
|
+
"substring-before" => [2, 2, X::TYPE_STRING],
|
|
79
|
+
"substring-after" => [2, 2, X::TYPE_STRING],
|
|
80
|
+
"substring" => [2, 3, X::TYPE_STRING],
|
|
81
|
+
"string-length" => [0, 1, X::TYPE_NUMBER],
|
|
82
|
+
"normalize-space" => [0, 1, X::TYPE_STRING],
|
|
83
|
+
"translate" => [3, 3, X::TYPE_STRING],
|
|
84
|
+
"lower-case" => [1, 1, X::TYPE_STRING],
|
|
85
|
+
"upper-case" => [1, 1, X::TYPE_STRING],
|
|
86
|
+
"not" => [1, 1, X::TYPE_BOOLEAN],
|
|
87
|
+
"true" => [0, 0, X::TYPE_BOOLEAN],
|
|
88
|
+
"false" => [0, 0, X::TYPE_BOOLEAN],
|
|
89
|
+
"boolean" => [1, 1, X::TYPE_BOOLEAN],
|
|
90
|
+
"number" => [0, 1, X::TYPE_NUMBER],
|
|
91
|
+
"sum" => [1, 1, X::TYPE_NUMBER],
|
|
92
|
+
"floor" => [1, 1, X::TYPE_NUMBER],
|
|
93
|
+
"ceiling" => [1, 1, X::TYPE_NUMBER],
|
|
94
|
+
"round" => [1, 1, X::TYPE_NUMBER]
|
|
95
|
+
}.freeze
|
|
96
|
+
|
|
97
|
+
LETTER = /\A\p{L}\z/
|
|
98
|
+
LETTER_OR_DIGIT = /\A[\p{L}\p{Nd}]\z/
|
|
99
|
+
|
|
100
|
+
def initialize(expression)
|
|
101
|
+
@expression = expression
|
|
102
|
+
unless expression.dup.force_encoding(Encoding::UTF_8).valid_encoding?
|
|
103
|
+
raise XPathError, "invalid XPath #{Chars.quote(expression)}: not UTF-8"
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
@source = Chars.chars(expression)
|
|
107
|
+
@tokens = []
|
|
108
|
+
@index = 0
|
|
109
|
+
tokenize
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# The compiled expression.
|
|
113
|
+
def parse
|
|
114
|
+
expr = parse_expr
|
|
115
|
+
fail!("unexpected #{describe(peek(0))}") if peek(0).k != T_END
|
|
116
|
+
expr
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
private
|
|
120
|
+
|
|
121
|
+
def fail_at!(msg, position)
|
|
122
|
+
raise XPathError, "invalid XPath #{Chars.quote(@expression)} at position #{position}: #{msg}"
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def fail!(msg)
|
|
126
|
+
fail_at!(msg, peek(0).p)
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def text(from, to)
|
|
130
|
+
@source[from...to].join
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
def utf8(ch)
|
|
134
|
+
ch.dup.force_encoding(Encoding::UTF_8)
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def letter?(ch)
|
|
138
|
+
return Chars.alpha_byte?(ch.getbyte(0)) if ch.bytesize == 1
|
|
139
|
+
|
|
140
|
+
LETTER.match?(utf8(ch))
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def xname_start?(ch)
|
|
144
|
+
ch == "_" || letter?(ch)
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def xname_char?(ch)
|
|
148
|
+
if ch.bytesize == 1
|
|
149
|
+
c = ch.getbyte(0)
|
|
150
|
+
return Chars.alpha_byte?(c) || Chars.digit_byte?(c) || c == 95 || c == 45 || c == 46
|
|
151
|
+
end
|
|
152
|
+
LETTER_OR_DIGIT.match?(utf8(ch))
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
def digit?(ch)
|
|
156
|
+
!ch.nil? && ch.bytesize == 1 && Chars.digit_byte?(ch.getbyte(0))
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def space?(ch)
|
|
160
|
+
Chars.space_at(ch, 0, ch.bytesize) == ch.bytesize
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
def tokenize
|
|
164
|
+
s = @source
|
|
165
|
+
n = s.size
|
|
166
|
+
i = 0
|
|
167
|
+
loop do
|
|
168
|
+
i += 1 while i < n && space?(s[i])
|
|
169
|
+
if i >= n
|
|
170
|
+
@tokens << Token.new(T_END, Chars::EMPTY, 0.0, i)
|
|
171
|
+
return
|
|
172
|
+
end
|
|
173
|
+
start = i
|
|
174
|
+
ch = s[i]
|
|
175
|
+
nxt = s[i + 1]
|
|
176
|
+
|
|
177
|
+
# XPath 1.0 lexical rule: after a token that can end an operand, '*' and and/or/div/mod are operators
|
|
178
|
+
operator_expected = !@tokens.empty? && OPERAND_ENDS.include?(@tokens[-1].k)
|
|
179
|
+
|
|
180
|
+
kind = T_END
|
|
181
|
+
txt = Chars::EMPTY
|
|
182
|
+
number = 0.0
|
|
183
|
+
if ch == "/"
|
|
184
|
+
if nxt == "/"
|
|
185
|
+
kind = T_DOUBLE_SLASH
|
|
186
|
+
i += 2
|
|
187
|
+
else
|
|
188
|
+
kind = T_SLASH
|
|
189
|
+
i += 1
|
|
190
|
+
end
|
|
191
|
+
elsif SINGLE.key?(ch)
|
|
192
|
+
kind = SINGLE[ch]
|
|
193
|
+
i += 1
|
|
194
|
+
elsif ch == "*"
|
|
195
|
+
kind = operator_expected ? T_MULTIPLY : T_STAR
|
|
196
|
+
i += 1
|
|
197
|
+
elsif ch == "!"
|
|
198
|
+
fail_at!("'!' must be followed by '='", i) if nxt != "="
|
|
199
|
+
kind = T_NEQ
|
|
200
|
+
i += 2
|
|
201
|
+
elsif ch == "<" || ch == ">"
|
|
202
|
+
if nxt == "="
|
|
203
|
+
kind = ch == "<" ? T_LE : T_GE
|
|
204
|
+
i += 2
|
|
205
|
+
else
|
|
206
|
+
kind = ch == "<" ? T_LT : T_GT
|
|
207
|
+
i += 1
|
|
208
|
+
end
|
|
209
|
+
elsif ch == ":"
|
|
210
|
+
fail_at!("unexpected ':'", i) if nxt != ":"
|
|
211
|
+
kind = T_COLON_COLON
|
|
212
|
+
i += 2
|
|
213
|
+
elsif ch == '"' || ch == "'"
|
|
214
|
+
stop = ((i + 1)...n).find { |k| s[k] == ch }
|
|
215
|
+
fail_at!("unterminated string literal", i) if stop.nil?
|
|
216
|
+
kind = T_LITERAL
|
|
217
|
+
txt = text(i + 1, stop).freeze
|
|
218
|
+
i = stop + 1
|
|
219
|
+
elsif ch == "." && nxt == "."
|
|
220
|
+
kind = T_DOT_DOT
|
|
221
|
+
i += 2
|
|
222
|
+
elsif ch == "." && !digit?(nxt)
|
|
223
|
+
kind = T_DOT
|
|
224
|
+
i += 1
|
|
225
|
+
elsif digit?(ch) || ch == "."
|
|
226
|
+
i += 1 while i < n && digit?(s[i])
|
|
227
|
+
if i < n && s[i] == "."
|
|
228
|
+
i += 1
|
|
229
|
+
i += 1 while i < n && digit?(s[i])
|
|
230
|
+
end
|
|
231
|
+
kind = T_NUMBER
|
|
232
|
+
number = text(start, i).to_f
|
|
233
|
+
elsif xname_start?(ch)
|
|
234
|
+
i += 1
|
|
235
|
+
while i < n
|
|
236
|
+
if xname_char?(s[i])
|
|
237
|
+
i += 1
|
|
238
|
+
elsif s[i] == ":" && i + 1 < n && s[i + 1] != ":" && (xname_start?(s[i + 1]) || s[i + 1] == "*")
|
|
239
|
+
# a prefixed name (dc:creator) or prefix:*
|
|
240
|
+
i += 2
|
|
241
|
+
break if s[i - 1] == "*"
|
|
242
|
+
else
|
|
243
|
+
break
|
|
244
|
+
end
|
|
245
|
+
end
|
|
246
|
+
kind = T_NAME
|
|
247
|
+
txt = text(start, i).freeze
|
|
248
|
+
if operator_expected
|
|
249
|
+
op = OPERATORS[txt]
|
|
250
|
+
fail_at!("expected an operator, found '#{Chars.u(txt.dup)}'", start) if op.nil?
|
|
251
|
+
kind = op
|
|
252
|
+
end
|
|
253
|
+
else
|
|
254
|
+
fail_at!("unexpected character '#{Chars.u(ch.dup)}'", i)
|
|
255
|
+
end
|
|
256
|
+
@tokens << Token.new(kind, txt, number, start)
|
|
257
|
+
end
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
# ----- parser helpers -----
|
|
261
|
+
|
|
262
|
+
def peek(offset)
|
|
263
|
+
i = @index + offset
|
|
264
|
+
@tokens[i < @tokens.size ? i : @tokens.size - 1]
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
def advance
|
|
268
|
+
t = @tokens[@index]
|
|
269
|
+
@index += 1
|
|
270
|
+
t
|
|
271
|
+
end
|
|
272
|
+
|
|
273
|
+
def accept(kind)
|
|
274
|
+
return false if peek(0).k != kind
|
|
275
|
+
|
|
276
|
+
@index += 1
|
|
277
|
+
true
|
|
278
|
+
end
|
|
279
|
+
|
|
280
|
+
def expect(kind, what)
|
|
281
|
+
fail!("expected #{what}, found #{describe(peek(0))}") unless accept(kind)
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
def describe(t)
|
|
285
|
+
case t.k
|
|
286
|
+
when T_END then "the end of the expression"
|
|
287
|
+
when T_NAME, T_LITERAL then "'#{Chars.u(t.t.dup)}'"
|
|
288
|
+
when T_NUMBER then X.number_to_string(t.n)
|
|
289
|
+
else
|
|
290
|
+
to = [@source.size, t.p + 2].min
|
|
291
|
+
"'#{Chars.u(text(t.p, to).rstrip)}'"
|
|
292
|
+
end
|
|
293
|
+
end
|
|
294
|
+
|
|
295
|
+
# ----- expressions -----
|
|
296
|
+
|
|
297
|
+
def expr(op, type, pos, **fields)
|
|
298
|
+
X::Expr.new(op: op, type: type, pos: pos, **fields)
|
|
299
|
+
end
|
|
300
|
+
|
|
301
|
+
def positional?(e)
|
|
302
|
+
e.type == X::TYPE_NUMBER || e.pos
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
def step(axis, test, preds = [])
|
|
306
|
+
X::Step.new(axis, test, preds.freeze, preds.any? { |p| positional?(p) }).freeze
|
|
307
|
+
end
|
|
308
|
+
|
|
309
|
+
def node_test(kind, name = Chars::EMPTY)
|
|
310
|
+
X::NodeTest.new(kind, name, kind == X::TK_NAME ? Query.by(name) : nil).freeze
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
# Whether the expression is a bare @name (one attribute step, no predicates); its name (lower case), or nil.
|
|
314
|
+
def attribute_name(e)
|
|
315
|
+
return nil if e.op != X::E_PATH || !e.start.nil? || e.absolute || e.steps.size != 1
|
|
316
|
+
|
|
317
|
+
s = e.steps[0]
|
|
318
|
+
return nil if s.axis != X::AXIS_ATTRIBUTE || !s.preds.empty? || s.test.k != X::TK_NAME
|
|
319
|
+
|
|
320
|
+
s.test.name.downcase(:ascii).freeze
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
# @name = 'literal' is compared in place; everything else by the general rules.
|
|
324
|
+
def make_equality(op, left, right)
|
|
325
|
+
l = attribute_name(left)
|
|
326
|
+
if l && right.op == X::E_LITERAL
|
|
327
|
+
return expr(X::E_ATTR_EQUALS, X::TYPE_BOOLEAN, false, name: l, value: right.value, negate: op == X::OP_NEQ)
|
|
328
|
+
end
|
|
329
|
+
|
|
330
|
+
r = attribute_name(right)
|
|
331
|
+
if r && left.op == X::E_LITERAL
|
|
332
|
+
return expr(X::E_ATTR_EQUALS, X::TYPE_BOOLEAN, false, name: r, value: left.value, negate: op == X::OP_NEQ)
|
|
333
|
+
end
|
|
334
|
+
|
|
335
|
+
compare_expr(op, left, right)
|
|
336
|
+
end
|
|
337
|
+
|
|
338
|
+
def compare_expr(op, left, right)
|
|
339
|
+
expr(X::E_COMPARE, X::TYPE_BOOLEAN, left.pos || right.pos, cmp: op, l: left, r: right)
|
|
340
|
+
end
|
|
341
|
+
|
|
342
|
+
def arith(op, left, right)
|
|
343
|
+
expr(X::E_ARITH, X::TYPE_NUMBER, left.pos || right.pos, arith: op, l: left, r: right)
|
|
344
|
+
end
|
|
345
|
+
|
|
346
|
+
# //x is descendant-or-self::node()/child::x. Without positional predicates on x it is the same as
|
|
347
|
+
# descendant::x: one linear scan of the tag list instead of the children of every node.
|
|
348
|
+
def optimize(steps)
|
|
349
|
+
steps = steps.dup
|
|
350
|
+
i = 0
|
|
351
|
+
while i < steps.size - 1
|
|
352
|
+
s = steps[i]
|
|
353
|
+
if s.axis == X::AXIS_DESCENDANT_OR_SELF && s.test.equal?(X::ANY_NODE_TEST) && s.preds.empty?
|
|
354
|
+
nxt = steps[i + 1]
|
|
355
|
+
if nxt.axis == X::AXIS_CHILD && !nxt.positional
|
|
356
|
+
steps[i] = step(X::AXIS_DESCENDANT, nxt.test, nxt.preds)
|
|
357
|
+
steps.delete_at(i + 1)
|
|
358
|
+
elsif nxt.axis == X::AXIS_CHILD || nxt.axis == X::AXIS_ATTRIBUTE
|
|
359
|
+
# only the root and elements have children or attributes: skip the text nodes
|
|
360
|
+
steps[i] = step(X::AXIS_DESCENDANT_OR_SELF, X::CONTAINERS_TEST)
|
|
361
|
+
end
|
|
362
|
+
end
|
|
363
|
+
i += 1
|
|
364
|
+
end
|
|
365
|
+
steps.freeze
|
|
366
|
+
end
|
|
367
|
+
|
|
368
|
+
def can_start_step?(t)
|
|
369
|
+
k = t.k
|
|
370
|
+
k == T_DOT || k == T_DOT_DOT || k == T_AT || k == T_STAR || k == T_NAME
|
|
371
|
+
end
|
|
372
|
+
|
|
373
|
+
# ----- grammar (XPath 1.0, section 3) -----
|
|
374
|
+
|
|
375
|
+
def parse_expr
|
|
376
|
+
parse_or
|
|
377
|
+
end
|
|
378
|
+
|
|
379
|
+
def parse_or
|
|
380
|
+
left = parse_and
|
|
381
|
+
while accept(T_OR)
|
|
382
|
+
right = parse_and
|
|
383
|
+
left = expr(X::E_OR, X::TYPE_BOOLEAN, left.pos || right.pos, l: left, r: right)
|
|
384
|
+
end
|
|
385
|
+
left
|
|
386
|
+
end
|
|
387
|
+
|
|
388
|
+
def parse_and
|
|
389
|
+
left = parse_equality
|
|
390
|
+
while accept(T_AND)
|
|
391
|
+
right = parse_equality
|
|
392
|
+
left = expr(X::E_AND, X::TYPE_BOOLEAN, left.pos || right.pos, l: left, r: right)
|
|
393
|
+
end
|
|
394
|
+
left
|
|
395
|
+
end
|
|
396
|
+
|
|
397
|
+
def parse_equality
|
|
398
|
+
left = parse_relational
|
|
399
|
+
loop do
|
|
400
|
+
if accept(T_EQ)
|
|
401
|
+
op = X::OP_EQ
|
|
402
|
+
elsif accept(T_NEQ)
|
|
403
|
+
op = X::OP_NEQ
|
|
404
|
+
else
|
|
405
|
+
return left
|
|
406
|
+
end
|
|
407
|
+
left = make_equality(op, left, parse_relational)
|
|
408
|
+
end
|
|
409
|
+
end
|
|
410
|
+
|
|
411
|
+
def parse_relational
|
|
412
|
+
left = parse_additive
|
|
413
|
+
loop do
|
|
414
|
+
if accept(T_LT)
|
|
415
|
+
op = X::OP_LT
|
|
416
|
+
elsif accept(T_LE)
|
|
417
|
+
op = X::OP_LE
|
|
418
|
+
elsif accept(T_GT)
|
|
419
|
+
op = X::OP_GT
|
|
420
|
+
elsif accept(T_GE)
|
|
421
|
+
op = X::OP_GE
|
|
422
|
+
else
|
|
423
|
+
return left
|
|
424
|
+
end
|
|
425
|
+
left = compare_expr(op, left, parse_additive)
|
|
426
|
+
end
|
|
427
|
+
end
|
|
428
|
+
|
|
429
|
+
def parse_additive
|
|
430
|
+
left = parse_multiplicative
|
|
431
|
+
loop do
|
|
432
|
+
if accept(T_PLUS)
|
|
433
|
+
left = arith(X::ARITH_ADD, left, parse_multiplicative)
|
|
434
|
+
elsif accept(T_MINUS)
|
|
435
|
+
left = arith(X::ARITH_SUB, left, parse_multiplicative)
|
|
436
|
+
else
|
|
437
|
+
return left
|
|
438
|
+
end
|
|
439
|
+
end
|
|
440
|
+
end
|
|
441
|
+
|
|
442
|
+
def parse_multiplicative
|
|
443
|
+
left = parse_unary
|
|
444
|
+
loop do
|
|
445
|
+
if accept(T_MULTIPLY)
|
|
446
|
+
left = arith(X::ARITH_MUL, left, parse_unary)
|
|
447
|
+
elsif accept(T_DIV)
|
|
448
|
+
left = arith(X::ARITH_DIV, left, parse_unary)
|
|
449
|
+
elsif accept(T_MOD)
|
|
450
|
+
left = arith(X::ARITH_MOD, left, parse_unary)
|
|
451
|
+
else
|
|
452
|
+
return left
|
|
453
|
+
end
|
|
454
|
+
end
|
|
455
|
+
end
|
|
456
|
+
|
|
457
|
+
def parse_unary
|
|
458
|
+
if accept(T_MINUS)
|
|
459
|
+
operand = parse_unary
|
|
460
|
+
return expr(X::E_NEGATE, X::TYPE_NUMBER, operand.pos, e: operand)
|
|
461
|
+
end
|
|
462
|
+
parse_union
|
|
463
|
+
end
|
|
464
|
+
|
|
465
|
+
def parse_union
|
|
466
|
+
left = parse_path
|
|
467
|
+
while peek(0).k == T_PIPE
|
|
468
|
+
position = advance.p
|
|
469
|
+
right = parse_path
|
|
470
|
+
if left.type != X::TYPE_NODE_SET || right.type != X::TYPE_NODE_SET
|
|
471
|
+
fail_at!("'|' needs node-sets on both sides", position)
|
|
472
|
+
end
|
|
473
|
+
left = expr(X::E_UNION, X::TYPE_NODE_SET, left.pos || right.pos, l: left, r: right)
|
|
474
|
+
end
|
|
475
|
+
left
|
|
476
|
+
end
|
|
477
|
+
|
|
478
|
+
def parse_path
|
|
479
|
+
t = peek(0)
|
|
480
|
+
is_filter = t.k == T_LITERAL || t.k == T_NUMBER || t.k == T_LPAREN || t.k == T_DOLLAR ||
|
|
481
|
+
(t.k == T_NAME && peek(1).k == T_LPAREN && !NODE_TYPES.key?(t.t))
|
|
482
|
+
return parse_location_path unless is_filter
|
|
483
|
+
|
|
484
|
+
position = t.p
|
|
485
|
+
primary = parse_primary
|
|
486
|
+
predicates = parse_predicates
|
|
487
|
+
unless predicates.empty?
|
|
488
|
+
fail_at!("a predicate needs a node-set", position) if primary.type != X::TYPE_NODE_SET
|
|
489
|
+
primary = expr(X::E_FILTER, X::TYPE_NODE_SET, primary.pos, primary: primary, preds: predicates.freeze)
|
|
490
|
+
end
|
|
491
|
+
k = peek(0).k
|
|
492
|
+
return primary if k != T_SLASH && k != T_DOUBLE_SLASH
|
|
493
|
+
|
|
494
|
+
fail_at!("'/' needs a node-set on the left", position) if primary.type != X::TYPE_NODE_SET
|
|
495
|
+
steps = optimize(parse_relative_steps([], true))
|
|
496
|
+
expr(X::E_PATH, X::TYPE_NODE_SET, primary.pos, start: primary, absolute: false, steps: steps)
|
|
497
|
+
end
|
|
498
|
+
|
|
499
|
+
def parse_location_path
|
|
500
|
+
steps = []
|
|
501
|
+
absolute = false
|
|
502
|
+
if accept(T_SLASH)
|
|
503
|
+
absolute = true
|
|
504
|
+
steps = parse_relative_steps(steps, false) if can_start_step?(peek(0))
|
|
505
|
+
elsif peek(0).k == T_DOUBLE_SLASH
|
|
506
|
+
absolute = true
|
|
507
|
+
steps = parse_relative_steps(steps, true)
|
|
508
|
+
else
|
|
509
|
+
fail!("unexpected #{describe(peek(0))}") unless can_start_step?(peek(0))
|
|
510
|
+
steps = parse_relative_steps(steps, false)
|
|
511
|
+
end
|
|
512
|
+
path = expr(X::E_PATH, X::TYPE_NODE_SET, false, start: nil, absolute: absolute, steps: optimize(steps))
|
|
513
|
+
name = attribute_name(path)
|
|
514
|
+
path.attr = name if name # read in place where a string is needed
|
|
515
|
+
path
|
|
516
|
+
end
|
|
517
|
+
|
|
518
|
+
# Steps separated by '/' or '//'; after_slash: the next token is a '/' or '//' to read first.
|
|
519
|
+
def parse_relative_steps(steps, after_slash)
|
|
520
|
+
steps << parse_step unless after_slash
|
|
521
|
+
loop do
|
|
522
|
+
if accept(T_SLASH)
|
|
523
|
+
steps << parse_step
|
|
524
|
+
elsif accept(T_DOUBLE_SLASH)
|
|
525
|
+
steps << step(X::AXIS_DESCENDANT_OR_SELF, X::ANY_NODE_TEST)
|
|
526
|
+
steps << parse_step
|
|
527
|
+
else
|
|
528
|
+
return steps
|
|
529
|
+
end
|
|
530
|
+
end
|
|
531
|
+
end
|
|
532
|
+
|
|
533
|
+
def parse_step
|
|
534
|
+
return step(X::AXIS_SELF, X::ANY_NODE_TEST) if accept(T_DOT)
|
|
535
|
+
return step(X::AXIS_PARENT, X::ANY_NODE_TEST) if accept(T_DOT_DOT)
|
|
536
|
+
|
|
537
|
+
axis = X::AXIS_CHILD
|
|
538
|
+
if accept(T_AT)
|
|
539
|
+
axis = X::AXIS_ATTRIBUTE
|
|
540
|
+
elsif peek(0).k == T_NAME && peek(1).k == T_COLON_COLON
|
|
541
|
+
name = advance
|
|
542
|
+
a = AXES[name.t]
|
|
543
|
+
if a.nil?
|
|
544
|
+
fail_at!("the namespace axis is not supported", name.p) if name.t == "namespace"
|
|
545
|
+
fail_at!("unknown axis '#{Chars.u(name.t.dup)}'", name.p)
|
|
546
|
+
end
|
|
547
|
+
axis = a
|
|
548
|
+
advance
|
|
549
|
+
end
|
|
550
|
+
test = parse_node_test
|
|
551
|
+
step(axis, test, parse_predicates)
|
|
552
|
+
end
|
|
553
|
+
|
|
554
|
+
def parse_node_test
|
|
555
|
+
return node_test(X::TK_WILDCARD) if accept(T_STAR)
|
|
556
|
+
|
|
557
|
+
t = peek(0)
|
|
558
|
+
fail!("expected a node test, found #{describe(t)}") if t.k != T_NAME
|
|
559
|
+
advance
|
|
560
|
+
name = t.t
|
|
561
|
+
if peek(0).k == T_LPAREN && NODE_TYPES.key?(name)
|
|
562
|
+
advance
|
|
563
|
+
accept(T_LITERAL) if name == "processing-instruction"
|
|
564
|
+
expect(T_RPAREN, "')'")
|
|
565
|
+
return case name
|
|
566
|
+
when "node" then X::ANY_NODE_TEST
|
|
567
|
+
when "text" then node_test(X::TK_TEXT)
|
|
568
|
+
when "comment" then node_test(X::TK_COMMENT)
|
|
569
|
+
else node_test(X::TK_PI)
|
|
570
|
+
end
|
|
571
|
+
end
|
|
572
|
+
return node_test(X::TK_PREFIX, name.byteslice(0, name.bytesize - 1).freeze) if name.end_with?(":*")
|
|
573
|
+
|
|
574
|
+
node_test(X::TK_NAME, name)
|
|
575
|
+
end
|
|
576
|
+
|
|
577
|
+
def parse_predicates
|
|
578
|
+
predicates = []
|
|
579
|
+
while accept(T_LBRACKET)
|
|
580
|
+
p = parse_expr
|
|
581
|
+
expect(T_RBRACKET, "']'")
|
|
582
|
+
# [@name]: an existence check instead of a node-set
|
|
583
|
+
name = attribute_name(p)
|
|
584
|
+
p = expr(X::E_ATTR_EXISTS, X::TYPE_BOOLEAN, false, name: name) if name
|
|
585
|
+
predicates << p
|
|
586
|
+
end
|
|
587
|
+
predicates
|
|
588
|
+
end
|
|
589
|
+
|
|
590
|
+
def parse_primary
|
|
591
|
+
t = advance
|
|
592
|
+
case t.k
|
|
593
|
+
when T_LITERAL
|
|
594
|
+
return expr(X::E_LITERAL, X::TYPE_STRING, false, value: t.t)
|
|
595
|
+
when T_NUMBER
|
|
596
|
+
return expr(X::E_NUMBER, X::TYPE_NUMBER, false, value: t.n)
|
|
597
|
+
when T_LPAREN
|
|
598
|
+
e = parse_expr
|
|
599
|
+
expect(T_RPAREN, "')'")
|
|
600
|
+
return e
|
|
601
|
+
when T_DOLLAR
|
|
602
|
+
fail_at!("variables are not supported", t.p)
|
|
603
|
+
end
|
|
604
|
+
|
|
605
|
+
# function call
|
|
606
|
+
info = FUNCTIONS[t.t]
|
|
607
|
+
fail_at!("unknown function '#{Chars.u(t.t.dup)}()'", t.p) if info.nil?
|
|
608
|
+
expect(T_LPAREN, "'('")
|
|
609
|
+
args = []
|
|
610
|
+
unless accept(T_RPAREN)
|
|
611
|
+
loop do
|
|
612
|
+
args << parse_expr
|
|
613
|
+
break unless accept(T_COMMA)
|
|
614
|
+
end
|
|
615
|
+
expect(T_RPAREN, "')'")
|
|
616
|
+
end
|
|
617
|
+
fn = t.t
|
|
618
|
+
fail_at!("wrong number of arguments for #{fn}()", t.p) if args.size < info[0] || args.size > info[1]
|
|
619
|
+
needs_node_set = fn == "count" || fn == "sum" || fn == "name" || fn == "local-name"
|
|
620
|
+
if needs_node_set && !args.empty? && args[0].type != X::TYPE_NODE_SET
|
|
621
|
+
fail_at!("#{fn}() needs a node-set argument", t.p)
|
|
622
|
+
end
|
|
623
|
+
pos = fn == "position" || fn == "last" || args.any?(&:pos)
|
|
624
|
+
expr(X::E_FUNCTION, info[2], pos, fn: fn, args: args.freeze)
|
|
625
|
+
end
|
|
626
|
+
end
|
|
627
|
+
end
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Ruby port of z3l.FastHtmlParser: a fast HTML / XML parser for scraping, with XPath 1.0 and CSS selectors. A page
|
|
4
|
+
# becomes a flat list of tags that hold only positions in the source; text, attributes and entities are read when they
|
|
5
|
+
# are asked for.
|
|
6
|
+
#
|
|
7
|
+
# require "fasthtmlparser"
|
|
8
|
+
#
|
|
9
|
+
# doc = FastHtmlParser.parse(html, "https://example.com/catalog/")
|
|
10
|
+
# doc.css("article.product_pod").map { |pod| pod.css_first("h3 a").attr("title") }
|
|
11
|
+
module FastHtmlParser
|
|
12
|
+
# Parses an HTML or XML page; see HtmlDocument.parse.
|
|
13
|
+
def self.parse(html, url = "")
|
|
14
|
+
HtmlDocument.parse(html, url)
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
require "strscan"
|
|
19
|
+
require_relative "fasthtmlparser/version"
|
|
20
|
+
require_relative "fasthtmlparser/tags"
|
|
21
|
+
require_relative "fasthtmlparser/attr"
|
|
22
|
+
require_relative "fasthtmlparser/entities"
|
|
23
|
+
require_relative "fasthtmlparser/chars"
|
|
24
|
+
require_relative "fasthtmlparser/html"
|
|
25
|
+
require_relative "fasthtmlparser/parser"
|
|
26
|
+
require_relative "fasthtmlparser/decode"
|
|
27
|
+
require_relative "fasthtmlparser/attrs"
|
|
28
|
+
require_relative "fasthtmlparser/text"
|
|
29
|
+
require_relative "fasthtmlparser/links"
|
|
30
|
+
require_relative "fasthtmlparser/query"
|
|
31
|
+
require_relative "fasthtmlparser/css"
|
|
32
|
+
require_relative "fasthtmlparser/xpath"
|
|
33
|
+
require_relative "fasthtmlparser/xpath_compiler"
|
|
34
|
+
require_relative "fasthtmlparser/html_node"
|
|
35
|
+
require_relative "fasthtmlparser/html_document"
|