fasthtmlparser 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1151 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FastHtmlParser
4
+ # @api private A node of the XPath data model over the flat tag list: the root, an element, a comment, a text node
5
+ # (a run of source text between tags) or an attribute (positions in the opening tag).
6
+ class XNode
7
+ ROOT = 0
8
+ ELEMENT = 1
9
+ COMMENT = 2
10
+ TEXT = 3
11
+ ATTRIBUTE = 4
12
+
13
+ # tag: element / comment: the tag; text / attribute: the parent element (nil: the root)
14
+ # start: text: first byte; attribute: name start. length: text: length; attribute: name length
15
+ # value_start: attribute: value start (-1 for no value)
16
+ # major, minor: document order and identity: tags and text nodes by their first byte; attributes right after
17
+ # their element
18
+ attr_reader :kind, :doc, :tag, :start, :length, :value_start, :value_length, :major, :minor
19
+
20
+ def initialize(kind, doc, tag, start = 0, length = 0, value_start = 0, value_length = 0)
21
+ @kind = kind
22
+ @doc = doc
23
+ @tag = tag
24
+ @start = start
25
+ @length = length
26
+ @value_start = value_start
27
+ @value_length = value_length
28
+ @major = case kind
29
+ when ROOT then -1
30
+ when TEXT then start
31
+ else tag.start
32
+ end
33
+ @minor = kind == ATTRIBUTE ? start : 0
34
+ end
35
+
36
+ def self.root(d)
37
+ new(ROOT, d, nil)
38
+ end
39
+
40
+ def self.of_tag(n)
41
+ new(n.key == Parser::COMMENT ? COMMENT : ELEMENT, n.doc, n)
42
+ end
43
+
44
+ # The first source position the node covers (for following / preceding).
45
+ def range_start
46
+ @kind == ROOT ? -1 : @major
47
+ end
48
+
49
+ # The last source position the node covers.
50
+ def range_end
51
+ case @kind
52
+ when ROOT then Float::INFINITY
53
+ when ELEMENT, COMMENT then @tag.end
54
+ when TEXT then @start + @length - 1
55
+ else @tag.start # attribute: the children of its element follow it
56
+ end
57
+ end
58
+
59
+ def attr_name
60
+ @doc.source_bytes.byteslice(@start, @length)
61
+ end
62
+ end
63
+
64
+ # A compiled XPath 1.0 expression over the flat tag list: the root, elements, comments, text nodes (runs of source
65
+ # text between tags) and attributes (positions in the opening tag). Compile once and reuse it on any number of
66
+ # documents; the string methods (node.xpath(...), doc.xpath_value(...), ...) compile and cache expressions
67
+ # automatically.
68
+ #
69
+ # All axes (except namespace), predicates, |, the operators and the XPath 1.0 functions are supported, plus
70
+ # ends-with(), lower-case() and upper-case(). HTML rules apply: names are case-insensitive, the string-value of an
71
+ # element is its visible text (as HtmlNode#text), attribute values and text are decoded. Variables are not
72
+ # supported. On a node, relative paths start at the node; '/' and '//' search the whole document.
73
+ class XPathQuery
74
+ # value types
75
+ TYPE_NODE_SET = 0
76
+ TYPE_NUMBER = 1
77
+ TYPE_STRING = 2
78
+ TYPE_BOOLEAN = 3
79
+ TYPE_NAMES = %w[node-set number string boolean].freeze
80
+
81
+ # expressions
82
+ E_LITERAL = 0 # value
83
+ E_NUMBER = 1 # value
84
+ E_AND = 2 # l, r
85
+ E_OR = 3 # l, r
86
+ E_COMPARE = 4 # cmp, l, r
87
+ E_ARITH = 5 # arith, l, r
88
+ E_NEGATE = 6 # e
89
+ E_UNION = 7 # l, r
90
+ E_ATTR_EQUALS = 8 # name, value, negate: @name = 'literal' compared in place
91
+ E_ATTR_EXISTS = 9 # name: [@name]
92
+ E_PATH = 10 # start (expr or nil), absolute, steps; attr: a bare @name
93
+ E_FILTER = 11 # primary, preds
94
+ E_FUNCTION = 12 # fn, args
95
+
96
+ OP_EQ = 0
97
+ OP_NEQ = 1
98
+ OP_LT = 2
99
+ OP_LE = 3
100
+ OP_GT = 4
101
+ OP_GE = 5
102
+
103
+ ARITH_ADD = 0
104
+ ARITH_SUB = 1
105
+ ARITH_MUL = 2
106
+ ARITH_DIV = 3
107
+ ARITH_MOD = 4
108
+
109
+ # node tests
110
+ TK_NAME = 0
111
+ TK_PREFIX = 1 # dc:*
112
+ TK_WILDCARD = 2 # *
113
+ TK_NODE = 3 # node()
114
+ TK_TEXT = 4 # text()
115
+ TK_COMMENT = 5 # comment()
116
+ TK_PI = 6 # processing-instruction(): never matches
117
+ TK_CONTAINER = 7 # internal: the root and elements (// before a child or attribute step)
118
+
119
+ AXIS_CHILD = 0
120
+ AXIS_DESCENDANT = 1
121
+ AXIS_DESCENDANT_OR_SELF = 2
122
+ AXIS_SELF = 3
123
+ AXIS_PARENT = 4
124
+ AXIS_ANCESTOR = 5
125
+ AXIS_ANCESTOR_OR_SELF = 6
126
+ AXIS_FOLLOWING_SIBLING = 7
127
+ AXIS_PRECEDING_SIBLING = 8
128
+ AXIS_FOLLOWING = 9
129
+ AXIS_PRECEDING = 10
130
+ AXIS_ATTRIBUTE = 11
131
+
132
+ # @api private
133
+ Expr = Struct.new(:op, :type, :pos, :value, :l, :r, :cmp, :arith, :e, :name, :negate, :start, :absolute, :steps,
134
+ :primary, :preds, :fn, :args, :attr, keyword_init: true)
135
+ # @api private
136
+ Step = Struct.new(:axis, :test, :preds, :positional)
137
+ # @api private
138
+ NodeTest = Struct.new(:k, :name, :q)
139
+
140
+ ANY_NODE_TEST = NodeTest.new(TK_NODE, Chars::EMPTY, nil).freeze
141
+ CONTAINERS_TEST = NodeTest.new(TK_CONTAINER, Chars::EMPTY, nil).freeze
142
+
143
+ @cache = {}
144
+ @cache_lock = Mutex.new
145
+
146
+ def initialize(expression, expr)
147
+ @expression = expression
148
+ @expr = expr
149
+ freeze
150
+ end
151
+
152
+ # Compiles an XPath 1.0 expression; raises XPathError on an invalid one.
153
+ def self.compile(expression)
154
+ s = Chars.b(expression.to_s)
155
+ new(expression.to_s.dup.freeze, XPathCompiler.new(s).parse)
156
+ end
157
+
158
+ # @api private compiles and caches
159
+ def self.cached(expression)
160
+ q = @cache[expression]
161
+ return q if q
162
+
163
+ q = compile(expression)
164
+ @cache_lock.synchronize do
165
+ @cache.clear if @cache.size >= 1024 # dynamically built expressions must not grow the cache forever
166
+ @cache[expression.frozen? ? expression : expression.dup.freeze] = q
167
+ end
168
+ q
169
+ end
170
+
171
+ # The source of the expression.
172
+ attr_reader :expression
173
+
174
+ def to_s
175
+ @expression
176
+ end
177
+
178
+ def inspect
179
+ "#<#{self.class.name} #{@expression}>"
180
+ end
181
+
182
+ # The selected tags (elements and comments) in document order, evaluated on the document. Raises XPathError when
183
+ # the expression does not give a node-set (use value).
184
+ def nodes(doc)
185
+ tags_of(XNode.root(doc))
186
+ end
187
+
188
+ # nodes evaluated on a tag: relative paths start at it.
189
+ def nodes_from(node)
190
+ tags_of(XNode.of_tag(node))
191
+ end
192
+
193
+ # The string-values of the selected nodes, or the result of an expression that gives a number, string or boolean
194
+ # (count(//a), concat(...)) as one value.
195
+ def values(doc)
196
+ values_of(XNode.root(doc))
197
+ end
198
+
199
+ # values evaluated on a tag.
200
+ def values_from(node)
201
+ values_of(XNode.of_tag(node))
202
+ end
203
+
204
+ # The string-value of the first selected node, or the result of the expression; nil when it selects no node.
205
+ def value(doc)
206
+ value_of(XNode.root(doc))
207
+ end
208
+
209
+ # value evaluated on a tag.
210
+ def value_from(node)
211
+ value_of(XNode.of_tag(node))
212
+ end
213
+
214
+ # The shortest round-trip digits, written without an exponent (XPath has none), as JavaScript writes a number.
215
+ def self.number_to_string(d)
216
+ return "NaN" if d.nan?
217
+ return d > 0 ? "Infinity" : "-Infinity" if d.infinite?
218
+ return "0" if d == 0
219
+ return d.to_i.to_s if d == d.floor && d.abs < 1e15
220
+
221
+ # the fewest significant digits that read back as d (Float#to_s), placed without an exponent
222
+ mantissa, exponent = d.abs.to_s.split("e")
223
+ int_part, frac = mantissa.split(".")
224
+ digits = int_part + frac.to_s
225
+ point_at = int_part.size + exponent.to_i
226
+ while digits.size > 1 && digits.start_with?("0")
227
+ digits = digits[1..]
228
+ point_at -= 1
229
+ end
230
+ digits = digits.sub(/0+\z/, "")
231
+ len = digits.size
232
+ out = d < 0 ? +"-" : +""
233
+ if point_at <= 0
234
+ out << "0." << ("0" * -point_at) << digits
235
+ elsif point_at >= len
236
+ out << digits << ("0" * (point_at - len))
237
+ else
238
+ out << digits[0, point_at] << "." << digits[point_at..]
239
+ end
240
+ out
241
+ end
242
+
243
+ private
244
+
245
+ def tags_of(context)
246
+ if @expr.type != TYPE_NODE_SET
247
+ raise XPathError, "the XPath #{Chars.quote(@expression)} gives a #{TYPE_NAMES[@expr.type]}, not tags; " \
248
+ "use xpath_value"
249
+ end
250
+ out = []
251
+ XPathEvaluator.new(context.doc).evaluate(@expr, context).each do |n|
252
+ out << n.tag if n.kind == XNode::ELEMENT || n.kind == XNode::COMMENT
253
+ end
254
+ out
255
+ end
256
+
257
+ def values_of(context)
258
+ ev = XPathEvaluator.new(context.doc)
259
+ result = ev.evaluate(@expr, context)
260
+ return [Chars.u(ev.to_str(result))] unless result.is_a?(Array)
261
+
262
+ result.map { |n| Chars.u(ev.string_value(n)) }
263
+ end
264
+
265
+ def value_of(context)
266
+ ev = XPathEvaluator.new(context.doc)
267
+ result = ev.evaluate(@expr, context)
268
+ if result.is_a?(Array)
269
+ return result.empty? ? nil : Chars.u(ev.string_value(result[0]))
270
+ end
271
+
272
+ Chars.u(ev.to_str(result))
273
+ end
274
+ end
275
+
276
+ # @api private The evaluation of one expression on one document: values are a node-set (an Array of XNode), a
277
+ # Float, a String (binary) or true / false.
278
+ class XPathEvaluator
279
+ X = XPathQuery
280
+ NAN = Float::NAN
281
+ INF = Float::INFINITY
282
+ EMPTY = Chars::EMPTY
283
+ NUMBER_CHARS = /\A-?[0-9.]*\z/n
284
+
285
+ def initialize(doc)
286
+ @doc = doc
287
+ @all = nil
288
+ end
289
+
290
+ def evaluate(expr, context)
291
+ eval_expr(expr, context, 1, 1)
292
+ end
293
+
294
+ # ----- node tests -----
295
+
296
+ def matches_tag?(test, n)
297
+ matches_tag_at?(test, n.doc, n.index)
298
+ end
299
+
300
+ # The node test on the tag at index i, read from the arrays of the document.
301
+ def matches_tag_at?(test, d, i)
302
+ case test.k
303
+ when 0 # TK_NAME
304
+ test.q.name_matches_at?(d, i)
305
+ when 1 # TK_PREFIX
306
+ k = d.keys[i]
307
+ name = k >= 0 ? TagNames::NAMES[k] : d.customs[i]
308
+ len = test.name.bytesize
309
+ CssQuery.element_at?(d, i) && name.bytesize >= len && Chars.equal_fold(name.byteslice(0, len), test.name)
310
+ when 2, 7 # TK_WILDCARD, TK_CONTAINER
311
+ CssQuery.element_at?(d, i)
312
+ when 3 # TK_NODE
313
+ CssQuery.element_at?(d, i) || d.keys[i] == Parser::COMMENT
314
+ when 5 # TK_COMMENT
315
+ d.keys[i] == Parser::COMMENT
316
+ else
317
+ false
318
+ end
319
+ end
320
+
321
+ def matches_text?(test)
322
+ test.k == X::TK_NODE || test.k == X::TK_TEXT
323
+ end
324
+
325
+ def matches_tags?(test)
326
+ test.k != X::TK_TEXT && test.k != X::TK_PI
327
+ end
328
+
329
+ def matches_attribute?(test, attribute)
330
+ case test.k
331
+ when 0 # TK_NAME
332
+ Chars.equal_fold(attribute, test.name)
333
+ when 1 # TK_PREFIX
334
+ len = test.name.bytesize
335
+ attribute.bytesize >= len && Chars.equal_fold(attribute.byteslice(0, len), test.name)
336
+ when 2, 3 # TK_WILDCARD, TK_NODE
337
+ true
338
+ else
339
+ false
340
+ end
341
+ end
342
+
343
+ def matches?(test, x, attribute_axis)
344
+ case x.kind
345
+ when 0 # ROOT
346
+ test.k == X::TK_NODE || test.k == X::TK_CONTAINER
347
+ when 1, 2 # ELEMENT, COMMENT
348
+ matches_tag?(test, x.tag)
349
+ when 3 # TEXT
350
+ matches_text?(test)
351
+ else
352
+ return matches_attribute?(test, x.attr_name) if attribute_axis
353
+
354
+ test.k == X::TK_NODE
355
+ end
356
+ end
357
+
358
+ # ----- navigation -----
359
+
360
+ # Appends the children of the root or an element that match the test, text nodes included, in document order.
361
+ def add_children(x, test, out)
362
+ want_text = matches_text?(test)
363
+ want_tags = matches_tags?(test)
364
+ return if !want_text && !want_tags
365
+
366
+ doc = x.doc
367
+ if x.kind == XNode::ROOT
368
+ first = 0
369
+ end_ = INF
370
+ child_depth = 0
371
+ p = 0
372
+ text_end = doc.source_bytes.bytesize
373
+ elsif x.kind == XNode::ELEMENT && x.tag.inner_start > 0 && x.tag.inner_end > 0
374
+ t = x.tag
375
+ first = t.index + 1
376
+ end_ = t.end
377
+ child_depth = t.depth + 1
378
+ p = t.inner_start
379
+ text_end = t.inner_end + 1
380
+ else
381
+ return
382
+ end
383
+ ends = doc.ends
384
+ depths = doc.depths
385
+ starts = doc.starts
386
+ parent = x.kind == XNode::ELEMENT ? x.tag : nil
387
+ count = doc.size
388
+ i = first
389
+ while i < count
390
+ break if ends[i] > end_
391
+
392
+ if depths[i] == child_depth
393
+ start = starts[i]
394
+ if want_text && start > p
395
+ e = start < text_end ? start : text_end
396
+ out << XNode.new(XNode::TEXT, doc, parent, p, e - p) if e > p
397
+ end
398
+ out << XNode.of_tag(doc.at(i)) if want_tags && matches_tag_at?(test, doc, i)
399
+ p = ends[i] + 1
400
+ end
401
+ i += 1
402
+ end
403
+ out << XNode.new(XNode::TEXT, doc, parent, p, text_end - p) if want_text && text_end > p
404
+ end
405
+
406
+ # Appends the descendants of the root or an element that match the test, in document order.
407
+ def add_descendants(x, test, out)
408
+ if matches_text?(test)
409
+ add_descendants_with_text(x, test, out)
410
+ return
411
+ end
412
+ return unless matches_tags?(test)
413
+
414
+ if x.kind == XNode::ROOT
415
+ first = 0
416
+ end_ = INF
417
+ elsif x.kind == XNode::ELEMENT && x.tag.inner_start > 0 && x.tag.inner_end > 0
418
+ first = x.tag.index + 1
419
+ end_ = x.tag.end
420
+ else
421
+ return
422
+ end
423
+ # tags only: one linear scan of the flat list
424
+ d = x.doc
425
+ ends = d.ends
426
+ count = d.size
427
+ i = first
428
+ if test.k == X::TK_NAME
429
+ q = test.q
430
+ qk = q.key
431
+ keys = d.keys
432
+ named = !q.tag_name.empty?
433
+ while i < count
434
+ break if ends[i] > end_
435
+
436
+ # a known name: one int compare on the arrays
437
+ k = keys[i]
438
+ if qk >= 0 ? k == qk || (k < 0 && named && Chars.equal_fold(d.customs[i], q.tag_name)) : q.name_matches_at?(d, i)
439
+ out << XNode.new(XNode::ELEMENT, d, d.at(i))
440
+ end
441
+ i += 1
442
+ end
443
+ return
444
+ end
445
+ while i < count
446
+ break if ends[i] > end_
447
+
448
+ out << XNode.of_tag(d.at(i)) if matches_tag_at?(test, d, i)
449
+ i += 1
450
+ end
451
+ end
452
+
453
+ def add_descendants_with_text(x, test, out)
454
+ children = []
455
+ add_children(x, X::ANY_NODE_TEST, children)
456
+ children.each do |child|
457
+ out << child if matches?(test, child, false)
458
+ add_descendants_with_text(child, test, out) if child.kind == XNode::ELEMENT
459
+ end
460
+ end
461
+
462
+ def parent_node(x)
463
+ case x.kind
464
+ when 0 # ROOT
465
+ nil
466
+ when 3, 4 # TEXT, ATTRIBUTE
467
+ x.tag.nil? ? XNode.root(x.doc) : XNode.of_tag(x.tag)
468
+ else
469
+ t = x.tag
470
+ t.depth > 0 && t.parent_index >= 0 ? XNode.of_tag(x.doc.at(t.parent_index)) : XNode.root(x.doc)
471
+ end
472
+ end
473
+
474
+ def add_attributes(x, test, out)
475
+ return if x.kind != XNode::ELEMENT
476
+
477
+ s = AttrScanner.new(x.tag)
478
+ while s.next?
479
+ if matches_attribute?(test, s.name)
480
+ out << XNode.new(XNode::ATTRIBUTE, x.doc, x.tag, s.name_start, s.name_length, s.value_start, s.value_length)
481
+ end
482
+ end
483
+ end
484
+
485
+ # The raw value of the first attribute with the name (lower case); nil when the element has none.
486
+ def attribute_of(x, name)
487
+ return nil if x.kind != XNode::ELEMENT
488
+
489
+ raw = Attrs.attr_raw(x.tag, name)
490
+ return raw unless raw.empty?
491
+
492
+ Attrs.has_attribute?(x.tag, name) ? raw : nil
493
+ end
494
+
495
+ # The root and elements give their visible text; text nodes and attributes their decoded value; comments their
496
+ # content.
497
+ def string_value(x)
498
+ case x.kind
499
+ when 0 # ROOT
500
+ Text.clear_tags(x.doc.source_bytes)
501
+ when 1 # ELEMENT
502
+ x.tag.text_bytes
503
+ when 4 # ATTRIBUTE
504
+ x.value_start < 0 ? +"".b : Decode.decode(x.doc.source_bytes.byteslice(x.value_start, x.value_length), true)
505
+ when 3 # TEXT
506
+ text = x.doc.source_bytes.byteslice(x.start, x.length)
507
+ return Decode.normalize_newlines(text) if x.tag && Parser.raw_text?(x.tag.key)
508
+
509
+ Decode.decode(text, false)
510
+ else
511
+ # comment
512
+ t = x.tag
513
+ source = Chars.slice(t.doc.source_bytes, t.start, t.end + 1)
514
+ source = source.byteslice(4, source.bytesize - 4) if source.start_with?("<!--")
515
+ source = source.byteslice(0, source.bytesize - 3) if source.end_with?("-->")
516
+ source
517
+ end
518
+ end
519
+
520
+ # name() / local-name(): the lower-case tag or attribute name; "" for other nodes.
521
+ def node_name(x)
522
+ return x.tag.name_bytes if x.kind == XNode::ELEMENT
523
+ return Chars.lower(x.attr_name) if x.kind == XNode::ATTRIBUTE
524
+
525
+ EMPTY
526
+ end
527
+
528
+ # ----- values -----
529
+
530
+ def to_boolean(v)
531
+ case v
532
+ when true, false then v
533
+ when Float then v != 0 && !v.nan?
534
+ when String then !v.empty?
535
+ else !v.empty?
536
+ end
537
+ end
538
+
539
+ def to_number(v)
540
+ case v
541
+ when Float then v
542
+ when true then 1.0
543
+ when false then 0.0
544
+ when String then string_to_number(v)
545
+ else v.empty? ? NAN : string_to_number(string_value(v[0]))
546
+ end
547
+ end
548
+
549
+ # Optional '-', digits with an optional decimal point; anything else is NaN.
550
+ def string_to_number(value)
551
+ s = Chars.trim_space(value)
552
+ return NAN unless NUMBER_CHARS.match?(s)
553
+
554
+ body = s.start_with?("-") ? s.byteslice(1, s.bytesize - 1) : s
555
+ dots = body.count(".")
556
+ return NAN if dots > 1 || (body.bytesize - dots).zero?
557
+
558
+ s.to_f
559
+ end
560
+
561
+ def to_str(v)
562
+ case v
563
+ when String then v
564
+ when true then "true"
565
+ when false then "false"
566
+ when Float then X.number_to_string(v)
567
+ else v.empty? ? EMPTY : string_value(v[0])
568
+ end
569
+ end
570
+
571
+ def compare_keys(a, b)
572
+ c = a.major <=> b.major
573
+ c == 0 ? a.minor <=> b.minor : c
574
+ end
575
+
576
+ # Puts the nodes in document order without duplicates.
577
+ def sort_unique(nodes)
578
+ count = nodes.size
579
+ return nodes if count < 2
580
+
581
+ sorted = true
582
+ i = 1
583
+ while i < count
584
+ a = nodes[i - 1]
585
+ b = nodes[i]
586
+ if a.major > b.major || (a.major == b.major && a.minor >= b.minor)
587
+ sorted = false
588
+ break
589
+ end
590
+ i += 1
591
+ end
592
+ return nodes if sorted
593
+
594
+ # stable: ties keep their order
595
+ nodes = nodes.each_with_index.sort_by { |n, k| [n.major, n.minor, k] }.map(&:first)
596
+ out = [nodes[0]]
597
+ last = nodes[0]
598
+ i = 1
599
+ while i < count
600
+ n = nodes[i]
601
+ if n.major != last.major || n.minor != last.minor
602
+ out << n
603
+ last = n
604
+ end
605
+ i += 1
606
+ end
607
+ out
608
+ end
609
+
610
+ def all_nodes
611
+ return @all if @all
612
+
613
+ all = []
614
+ add_descendants(XNode.root(@doc), X::ANY_NODE_TEST, all)
615
+ @all = all
616
+ end
617
+
618
+ # ----- evaluation -----
619
+
620
+ def eval_nodes(e, node, position, size)
621
+ v = eval_expr(e, node, position, size)
622
+ return v if v.is_a?(Array)
623
+
624
+ raise XPathError, "the XPath expression does not give a node-set"
625
+ end
626
+
627
+ def eval_string(e, node, position, size)
628
+ if e.attr
629
+ # a bare @name: the string-value of the first such attribute, read in place
630
+ raw = node.kind == XNode::ELEMENT ? Attrs.attr_raw(node.tag, e.attr) : EMPTY
631
+ return Decode.needs_decoding?(raw) ? Decode.decode(raw, true) : raw
632
+ end
633
+ to_str(eval_expr(e, node, position, size))
634
+ end
635
+
636
+ def eval_number(e, node, position, size)
637
+ to_number(eval_expr(e, node, position, size))
638
+ end
639
+
640
+ def eval_bool(e, node, position, size)
641
+ case e.op
642
+ when 0 # E_LITERAL
643
+ !e.value.empty?
644
+ when 2 # E_AND
645
+ eval_bool(e.l, node, position, size) && eval_bool(e.r, node, position, size)
646
+ when 3 # E_OR
647
+ eval_bool(e.l, node, position, size) || eval_bool(e.r, node, position, size)
648
+ when 4 # E_COMPARE
649
+ compare(e.cmp, eval_expr(e.l, node, position, size), eval_expr(e.r, node, position, size))
650
+ when 8 # E_ATTR_EQUALS
651
+ # '!=' is true when the attribute exists and differs (a missing attribute is an empty node-set)
652
+ raw = attribute_of(node, e.name)
653
+ !raw.nil? && Attrs.attr_equals?(raw, e.value) != e.negate
654
+ when 9 # E_ATTR_EXISTS
655
+ !attribute_of(node, e.name).nil?
656
+ else
657
+ to_boolean(eval_expr(e, node, position, size))
658
+ end
659
+ end
660
+
661
+ # A number compares with the position, anything else is converted to boolean.
662
+ def eval_predicate(e, node, position, size)
663
+ return eval_bool(e, node, position, size) if e.type == X::TYPE_BOOLEAN
664
+
665
+ v = eval_expr(e, node, position, size)
666
+ return v == position if v.is_a?(Float)
667
+
668
+ to_boolean(v)
669
+ end
670
+
671
+ def eval_expr(e, node, position, size)
672
+ case e.op
673
+ when 0, 1 # E_LITERAL, E_NUMBER
674
+ e.value
675
+ when 2, 3, 4, 8, 9 # E_AND, E_OR, E_COMPARE, E_ATTR_EQUALS, E_ATTR_EXISTS
676
+ eval_bool(e, node, position, size)
677
+ when 5 # E_ARITH
678
+ x = eval_number(e.l, node, position, size)
679
+ y = eval_number(e.r, node, position, size)
680
+ case e.arith
681
+ when 0 then x + y # ARITH_ADD
682
+ when 1 then x - y # ARITH_SUB
683
+ when 2 then x * y # ARITH_MUL
684
+ when 3 then x / y # ARITH_DIV
685
+ else fmod(x, y)
686
+ end
687
+ when 6 # E_NEGATE
688
+ -eval_number(e.e, node, position, size)
689
+ when 7 # E_UNION
690
+ a = eval_nodes(e.l, node, position, size)
691
+ b = eval_nodes(e.r, node, position, size)
692
+ sort_unique(a + b)
693
+ when 10 # E_PATH
694
+ current = if e.start
695
+ eval_nodes(e.start, node, position, size)
696
+ elsif e.absolute
697
+ [XNode.root(@doc)]
698
+ else
699
+ [node]
700
+ end
701
+ e.steps.each do |s|
702
+ break if current.empty?
703
+
704
+ current = apply_step(s, current)
705
+ end
706
+ current
707
+ when 11 # E_FILTER
708
+ filter(eval_nodes(e.primary, node, position, size), e.preds)
709
+ else
710
+ eval_function(e, node, position, size)
711
+ end
712
+ end
713
+
714
+ # C fmod: the remainder has the sign of x.
715
+ def fmod(x, y)
716
+ return NAN if x.nan? || y.nan? || x.infinite? || y == 0
717
+ return x if y.infinite?
718
+
719
+ r = x % y
720
+ r -= y if r != 0 && ((r < 0) != (x < 0))
721
+ r
722
+ end
723
+
724
+ def compare(op, a, b)
725
+ if a.is_a?(Array)
726
+ if b.is_a?(Array)
727
+ return false if a.empty? || b.empty?
728
+
729
+ values_b = b.map { |n| string_value(n) }
730
+ a.each do |x|
731
+ value = string_value(x)
732
+ values_b.each { |other| return true if compare_atoms(op, value, other) }
733
+ end
734
+ return false
735
+ end
736
+ return compare_with_nodes(op, a, b, true)
737
+ end
738
+ return compare_with_nodes(op, b, a, false) if b.is_a?(Array)
739
+
740
+ compare_atoms(op, a, b)
741
+ end
742
+
743
+ def compare_with_nodes(op, nodes, other, nodes_on_left)
744
+ if other == true || other == false
745
+ as_bool = !nodes.empty?
746
+ return nodes_on_left ? compare_atoms(op, as_bool, other) : compare_atoms(op, other, as_bool)
747
+ end
748
+ is_number = other.is_a?(Float)
749
+ nodes.each do |n|
750
+ s = string_value(n)
751
+ value = is_number ? string_to_number(s) : s
752
+ return true if nodes_on_left ? compare_atoms(op, value, other) : compare_atoms(op, other, value)
753
+ end
754
+ false
755
+ end
756
+
757
+ def compare_atoms(op, a, b)
758
+ if op == X::OP_EQ || op == X::OP_NEQ
759
+ eq = if a == true || a == false || b == true || b == false
760
+ to_boolean(a) == to_boolean(b)
761
+ elsif a.is_a?(Float) || b.is_a?(Float)
762
+ to_number(a) == to_number(b)
763
+ else
764
+ to_str(a) == to_str(b)
765
+ end
766
+ return eq == (op == X::OP_EQ)
767
+ end
768
+ x = to_number(a)
769
+ y = to_number(b)
770
+ case op
771
+ when 2 then x < y # OP_LT
772
+ when 3 then x <= y # OP_LE
773
+ when 4 then x > y # OP_GT
774
+ else x >= y
775
+ end
776
+ end
777
+
778
+ # ----- steps -----
779
+
780
+ def reverse?(axis)
781
+ axis == X::AXIS_ANCESTOR || axis == X::AXIS_ANCESTOR_OR_SELF || axis == X::AXIS_PRECEDING ||
782
+ axis == X::AXIS_PRECEDING_SIBLING
783
+ end
784
+
785
+ def apply_step(step, input)
786
+ if input.size == 1
787
+ out = []
788
+ select_from(step, input[0], out)
789
+ return out
790
+ end
791
+
792
+ # Nested input nodes (//div//a on divs inside divs, //a/ancestor::* on links in one list) give the same nodes
793
+ # again and again: n nested nodes would collect about n^2 nodes before the duplicates are removed. When the
794
+ # predicates do not depend on the position, a node passes them or not whichever input node it comes from, so
795
+ # the input nodes whose results are already collected are skipped: the work stays linear.
796
+ unless step.positional
797
+ case step.axis
798
+ when 1, 2 # AXIS_DESCENDANT, AXIS_DESCENDANT_OR_SELF
799
+ if !matches_text?(step.test) && in_document_order?(input)
800
+ return sort_unique(select_descendants_of_nested(step, input))
801
+ end
802
+ when 5, 6 # AXIS_ANCESTOR, AXIS_ANCESTOR_OR_SELF
803
+ return sort_unique(select_shared_ancestors(step, input))
804
+ end
805
+ end
806
+
807
+ out = []
808
+ input.each { |x| select_from(step, x, out) }
809
+ sort_unique(out)
810
+ end
811
+
812
+ # The descendant axes (tags only) on input nodes in document order. A tag that lies in the part of the tag list
813
+ # already scanned for an earlier input node is skipped: that scan has tested it and every tag its own scan would
814
+ # test.
815
+ def select_descendants_of_nested(step, input)
816
+ ends = @doc.ends
817
+ count = @doc.size
818
+ out = []
819
+ scanned_to = -1 # the tags before this index were scanned for an earlier input node
820
+ input.each do |x|
821
+ if x.kind == XNode::ELEMENT || x.kind == XNode::COMMENT
822
+ t = x.tag
823
+ next if t.index < scanned_to
824
+
825
+ select_from(step, x, out)
826
+ # where add_descendants stops for this tag
827
+ if x.kind == XNode::ELEMENT && t.inner_start > 0 && t.inner_end > 0
828
+ k = t.index + 1
829
+ k += 1 while k < count && ends[k] <= t.end
830
+ scanned_to = k if k > scanned_to
831
+ end
832
+ else
833
+ select_from(step, x, out)
834
+ scanned_to = INF if x.kind == XNode::ROOT
835
+ end
836
+ end
837
+ out
838
+ end
839
+
840
+ # The walk up from an input node stops at an ancestor already reached from an earlier one, because that walk has
841
+ # gone on to the root. Only the root and elements are marked: they are the only parents.
842
+ def select_shared_ancestors(step, input)
843
+ visited = {}
844
+ out = []
845
+ test = step.test
846
+ input.each do |x|
847
+ start = out.size
848
+ current = x
849
+ if step.axis == X::AXIS_ANCESTOR_OR_SELF
850
+ container = current.kind == XNode::ROOT || current.kind == XNode::ELEMENT
851
+ next if container && visited[current.major]
852
+
853
+ visited[current.major] = true if container
854
+ out << current if matches?(test, current, false)
855
+ end
856
+ loop do
857
+ p = parent_node(current)
858
+ break if p.nil? || visited[p.major]
859
+
860
+ visited[p.major] = true
861
+ current = p
862
+ out << current if matches?(test, current, false)
863
+ end
864
+ if !step.preds.empty? && out.size > start
865
+ kept = filter(out[start..], step.preds)
866
+ out[start..] = kept
867
+ end
868
+ end
869
+ out
870
+ end
871
+
872
+ # Appends the nodes of the axis that pass the test and the predicates, in document order.
873
+ def select_from(step, x, out)
874
+ reverse = reverse?(step.axis)
875
+ if step.preds.empty? && !reverse
876
+ collect(step, x, out)
877
+ return
878
+ end
879
+ selected = []
880
+ collect(step, x, selected)
881
+ selected = filter(selected, step.preds)
882
+ selected.reverse! if reverse
883
+ out.concat(selected)
884
+ end
885
+
886
+ # Appends the nodes of the axis that pass the node test, in axis order (reverse axes: nearest first).
887
+ def collect(step, x, out)
888
+ test = step.test
889
+ case step.axis
890
+ when 0 # AXIS_CHILD
891
+ add_children(x, test, out)
892
+ when 1 # AXIS_DESCENDANT
893
+ add_descendants(x, test, out)
894
+ when 2 # AXIS_DESCENDANT_OR_SELF
895
+ out << x if matches?(test, x, false)
896
+ add_descendants(x, test, out)
897
+ when 3 # AXIS_SELF
898
+ out << x if matches?(test, x, false)
899
+ when 4 # AXIS_PARENT
900
+ p = parent_node(x)
901
+ out << p if !p.nil? && matches?(test, p, false)
902
+ when 5, 6 # AXIS_ANCESTOR, AXIS_ANCESTOR_OR_SELF
903
+ current = x
904
+ out << current if step.axis == X::AXIS_ANCESTOR_OR_SELF && matches?(test, current, false)
905
+ until (current = parent_node(current)).nil?
906
+ out << current if matches?(test, current, false)
907
+ end
908
+ when 7, 8 # AXIS_FOLLOWING_SIBLING, AXIS_PRECEDING_SIBLING
909
+ collect_siblings(x, test, out, step.axis == X::AXIS_FOLLOWING_SIBLING)
910
+ when 9 # AXIS_FOLLOWING
911
+ collect_following(x, test, out)
912
+ when 10 # AXIS_PRECEDING
913
+ collect_preceding(x, test, out)
914
+ when 11 # AXIS_ATTRIBUTE
915
+ add_attributes(x, test, out)
916
+ end
917
+ end
918
+
919
+ def in_document_order?(nodes)
920
+ i = 1
921
+ while i < nodes.size
922
+ return false if compare_keys(nodes[i - 1], nodes[i]) >= 0
923
+
924
+ i += 1
925
+ end
926
+ true
927
+ end
928
+
929
+ def collect_siblings(x, test, out, following)
930
+ return if x.kind == XNode::ROOT || x.kind == XNode::ATTRIBUTE
931
+
932
+ if !matches_text?(test) && x.kind != XNode::TEXT
933
+ # tags only: walk the flat list; deeper tags are inside the siblings, a shallower one ends the parent
934
+ t = x.tag
935
+ d = t.doc
936
+ depths = d.depths
937
+ count = d.size
938
+ step_by = following ? 1 : -1
939
+ i = t.index + step_by
940
+ while i >= 0 && i < count
941
+ break if depths[i] < t.depth
942
+
943
+ out << XNode.of_tag(d.at(i)) if depths[i] == t.depth && matches_tag_at?(test, d, i)
944
+ i += step_by
945
+ end
946
+ return
947
+ end
948
+ parent = parent_node(x)
949
+ return if parent.nil?
950
+
951
+ siblings = []
952
+ add_children(parent, X::ANY_NODE_TEST, siblings)
953
+ index = siblings.index { |s| compare_keys(s, x).zero? }
954
+ return if index.nil?
955
+
956
+ if following
957
+ ((index + 1)...siblings.size).each { |i| out << siblings[i] if matches?(test, siblings[i], false) }
958
+ else
959
+ (index - 1).downto(0) { |i| out << siblings[i] if matches?(test, siblings[i], false) }
960
+ end
961
+ end
962
+
963
+ def collect_following(x, test, out)
964
+ return if x.kind == XNode::ROOT
965
+
966
+ after = x.range_end
967
+ if matches_text?(test)
968
+ all_nodes.each { |n| out << n if n.range_start > after && matches?(test, n, false) }
969
+ return
970
+ end
971
+ d = @doc
972
+ starts = d.starts
973
+ i = x.kind == XNode::ELEMENT || x.kind == XNode::COMMENT ? x.tag.index + 1 : 0
974
+ count = d.size
975
+ while i < count
976
+ out << XNode.of_tag(d.at(i)) if starts[i] > after && matches_tag_at?(test, d, i)
977
+ i += 1
978
+ end
979
+ end
980
+
981
+ def collect_preceding(x, test, out)
982
+ return if x.kind == XNode::ROOT
983
+
984
+ before = x.range_start
985
+ if matches_text?(test)
986
+ all = all_nodes
987
+ (all.size - 1).downto(0) { |i| out << all[i] if all[i].range_end < before && matches?(test, all[i], false) }
988
+ return
989
+ end
990
+ d = @doc
991
+ ends = d.ends
992
+ i = x.kind == XNode::ELEMENT || x.kind == XNode::COMMENT ? x.tag.index - 1 : d.size - 1
993
+ while i >= 0
994
+ out << XNode.of_tag(d.at(i)) if ends[i] < before && matches_tag_at?(test, d, i)
995
+ i -= 1
996
+ end
997
+ end
998
+
999
+ # Applies the predicates one after another; positions are 1-based in the order of the list.
1000
+ def filter(nodes, predicates)
1001
+ predicates.each do |p|
1002
+ size = nodes.size
1003
+ kept = []
1004
+ nodes.each_with_index { |n, i| kept << n if eval_predicate(p, n, i + 1, size) }
1005
+ nodes = kept
1006
+ end
1007
+ nodes
1008
+ end
1009
+
1010
+ # ----- functions -----
1011
+
1012
+ def round(v)
1013
+ return v if v.nan? || v.infinite?
1014
+
1015
+ float_floor(v + 0.5)
1016
+ end
1017
+
1018
+ def float_floor(v)
1019
+ return v if v.nan? || v.infinite?
1020
+
1021
+ r = v.floor.to_f
1022
+ r == 0 && v < 0 ? -0.0 : r
1023
+ end
1024
+
1025
+ def float_ceil(v)
1026
+ return v if v.nan? || v.infinite?
1027
+
1028
+ r = v.ceil.to_f
1029
+ r == 0 && v < 0 ? -0.0 : r
1030
+ end
1031
+
1032
+ # Splits on Unicode whitespace and joins with one space.
1033
+ def normalize_space(s)
1034
+ s.split(Chars::SPACE_RUNS).reject(&:empty?).join(" ").b
1035
+ end
1036
+
1037
+ def eval_function(e, node, position, size)
1038
+ args = e.args
1039
+ case e.fn
1040
+ when "last"
1041
+ size.to_f
1042
+ when "position"
1043
+ position.to_f
1044
+ when "count"
1045
+ eval_nodes(args[0], node, position, size).size.to_f
1046
+ when "name", "local-name"
1047
+ name = EMPTY
1048
+ if args.empty?
1049
+ name = node_name(node)
1050
+ else
1051
+ set = eval_nodes(args[0], node, position, size)
1052
+ name = node_name(set[0]) unless set.empty?
1053
+ end
1054
+ if e.fn == "local-name"
1055
+ colon = name.rindex(":")
1056
+ return colon.nil? ? name : name.byteslice(colon + 1, name.bytesize - colon - 1)
1057
+ end
1058
+ name
1059
+ when "string"
1060
+ args.empty? ? string_value(node) : eval_string(args[0], node, position, size)
1061
+ when "concat"
1062
+ s = String.new(encoding: Encoding::BINARY)
1063
+ args.each { |a| s << eval_string(a, node, position, size) }
1064
+ s
1065
+ when "starts-with"
1066
+ eval_string(args[0], node, position, size).start_with?(eval_string(args[1], node, position, size))
1067
+ when "ends-with"
1068
+ eval_string(args[0], node, position, size).end_with?(eval_string(args[1], node, position, size))
1069
+ when "contains"
1070
+ eval_string(args[0], node, position, size).include?(eval_string(args[1], node, position, size))
1071
+ when "substring-before"
1072
+ s = eval_string(args[0], node, position, size)
1073
+ i = s.index(eval_string(args[1], node, position, size))
1074
+ i.nil? ? EMPTY : s.byteslice(0, i)
1075
+ when "substring-after"
1076
+ s = eval_string(args[0], node, position, size)
1077
+ t = eval_string(args[1], node, position, size)
1078
+ i = s.index(t)
1079
+ i.nil? ? EMPTY : s.byteslice(i + t.bytesize, s.bytesize - i - t.bytesize)
1080
+ when "substring"
1081
+ substring(e, node, position, size)
1082
+ when "string-length"
1083
+ # UTF-16 units, as .NET counts
1084
+ Chars.utf16_length(args.empty? ? string_value(node) : eval_string(args[0], node, position, size)).to_f
1085
+ when "normalize-space"
1086
+ normalize_space(args.empty? ? string_value(node) : eval_string(args[0], node, position, size))
1087
+ when "translate"
1088
+ s = eval_string(args[0], node, position, size)
1089
+ from = Chars.chars(eval_string(args[1], node, position, size))
1090
+ to = Chars.chars(eval_string(args[2], node, position, size))
1091
+ map = {}
1092
+ from.each_with_index { |ch, k| map[ch] = to[k] || EMPTY unless map.key?(ch) }
1093
+ return s if map.empty?
1094
+
1095
+ out = String.new(encoding: Encoding::BINARY)
1096
+ Chars.chars(s).each { |ch| out << (map.key?(ch) ? map[ch] : ch) }
1097
+ out
1098
+ when "lower-case"
1099
+ Chars.lower(eval_string(args[0], node, position, size))
1100
+ when "upper-case"
1101
+ Chars.upper(eval_string(args[0], node, position, size))
1102
+ when "not"
1103
+ !eval_bool(args[0], node, position, size)
1104
+ when "true"
1105
+ true
1106
+ when "false"
1107
+ false
1108
+ when "boolean"
1109
+ eval_bool(args[0], node, position, size)
1110
+ when "number"
1111
+ args.empty? ? string_to_number(string_value(node)) : eval_number(args[0], node, position, size)
1112
+ when "sum"
1113
+ sum = 0.0
1114
+ eval_nodes(args[0], node, position, size).each { |n| sum += string_to_number(string_value(n)) }
1115
+ sum
1116
+ when "floor"
1117
+ float_floor(eval_number(args[0], node, position, size))
1118
+ when "ceiling"
1119
+ float_ceil(eval_number(args[0], node, position, size))
1120
+ when "round"
1121
+ round(eval_number(args[0], node, position, size))
1122
+ else
1123
+ raise ArgumentError, "unknown XPath function #{e.fn}"
1124
+ end
1125
+ end
1126
+
1127
+ # XPath 1.0: the chars at 1-based positions p with round(start) <= p < round(start) + round(length).
1128
+ def substring(e, node, position, size)
1129
+ args = e.args
1130
+ s = eval_string(args[0], node, position, size)
1131
+ from = round(eval_number(args[1], node, position, size))
1132
+ to = args.size > 2 ? from + round(eval_number(args[2], node, position, size)) : INF
1133
+ return EMPTY if from.nan? || to.nan?
1134
+
1135
+ ascii = s.ascii_only?
1136
+ chars = ascii ? nil : Chars.chars(s)
1137
+ n = ascii ? s.bytesize : chars.size
1138
+ first = if from <= 1 then 0
1139
+ elsif from > n then n
1140
+ else from.to_i - 1
1141
+ end
1142
+ end_ = if to >= n + 1 then n
1143
+ elsif to <= 1 then 0
1144
+ else to.to_i - 1
1145
+ end
1146
+ return EMPTY if end_ <= first
1147
+
1148
+ ascii ? s.byteslice(first, end_ - first) : chars[first, end_ - first].join.b
1149
+ end
1150
+ end
1151
+ end