fasthtmlparser 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/LICENSE.md +1 -1
- data/README.md +2 -0
- data/lib/fasthtmlparser/version.rb +1 -1
- data/lib/fasthtmlparser/xpath.rb +14 -1
- data/lib/fasthtmlparser/xpath_compiler.rb +27 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 1eb9d918d0eb1683e04662e5aeca7950b148dbd80116d43dd5f04f434b9a9034
|
|
4
|
+
data.tar.gz: 4631038216a5a0cb6b652ee2ba745d85fede3902cb5e53908a36e74963ae5644
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 4fe78070839a8674229a9eb60274979caf2bba98277ed9ea9c3a4c2937670a6ba0a10a3f8e958230f04c3667dacb8cac02012e84caf2303e4dd1e8539f7aa7c9
|
|
7
|
+
data.tar.gz: 1fdf6b18e212321e788e5c0706738fc3a23d5dbd918f39a25e25e3b0c3b52042daf37c1fdfc42366c7d1d59db5dafd81220fb251a1b39e62e67fd2273256fb61
|
data/LICENSE.md
CHANGED
|
@@ -53,7 +53,7 @@ more**, the Licensee must purchase a Commercial License.
|
|
|
53
53
|
- A Licensee that stops being a Small Licensee (its Annual Revenue or Headcount reaches the
|
|
54
54
|
threshold) while already using the Software must purchase the Commercial License within
|
|
55
55
|
30 days.
|
|
56
|
-
- To purchase: https://
|
|
56
|
+
- To purchase: https://fasthtmlparser.com/#pricing, or contact **romanti3172@gmail.com** for an invoice.
|
|
57
57
|
|
|
58
58
|
The Commercial License grants the same rights as section 2, without the revenue and headcount
|
|
59
59
|
limits.
|
data/README.md
CHANGED
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
[](https://github.com/romanti3172/fasthtmlparser-rb/actions/workflows/ci.yml)
|
|
4
4
|
[](https://rubygems.org/gems/fasthtmlparser)
|
|
5
5
|
|
|
6
|
+
**Website:** [fasthtmlparser.com](https://fasthtmlparser.com): try the parser on any page, documentation and every language.
|
|
7
|
+
|
|
6
8
|
Ruby port of [z3l.FastHtmlParser](https://github.com/romanti3172/z3l.FastHtmlParser): a fast HTML / XML parser for
|
|
7
9
|
scraping, with **XPath 1.0** and **CSS selectors**. It is built for extracting data, not for building a DOM: a page
|
|
8
10
|
becomes a flat list of tags that hold only positions in the source. Text, attributes and entities are read when you
|
data/lib/fasthtmlparser/xpath.rb
CHANGED
|
@@ -92,6 +92,9 @@ module FastHtmlParser
|
|
|
92
92
|
E_PATH = 10 # start (expr or nil), absolute, steps; attr: a bare @name
|
|
93
93
|
E_FILTER = 11 # primary, preds
|
|
94
94
|
E_FUNCTION = 12 # fn, args
|
|
95
|
+
# fn, l (prefix), name, r (suffix), value (needle): contains/starts-with/ends-with(concat(l, @name, r), value),
|
|
96
|
+
# tested on the attribute in place (prefix and suffix may be empty)
|
|
97
|
+
E_ATTR_TEST = 13
|
|
95
98
|
|
|
96
99
|
OP_EQ = 0
|
|
97
100
|
OP_NEQ = 1
|
|
@@ -653,6 +656,16 @@ module FastHtmlParser
|
|
|
653
656
|
!raw.nil? && Attrs.attr_equals?(raw, e.value) != e.negate
|
|
654
657
|
when 9 # E_ATTR_EXISTS
|
|
655
658
|
!attribute_of(node, e.name).nil?
|
|
659
|
+
when 13 # E_ATTR_TEST
|
|
660
|
+
# a missing attribute is an empty node-set, whose string-value is ""
|
|
661
|
+
raw = attribute_of(node, e.name) || EMPTY
|
|
662
|
+
raw = Decode.decode(raw, true) if Decode.needs_decoding?(raw)
|
|
663
|
+
value = e.l.empty? && e.r.empty? ? raw : String.new(e.l, encoding: Encoding::BINARY) << raw << e.r
|
|
664
|
+
case e.fn
|
|
665
|
+
when "contains" then value.include?(e.value)
|
|
666
|
+
when "starts-with" then value.start_with?(e.value)
|
|
667
|
+
else value.end_with?(e.value)
|
|
668
|
+
end
|
|
656
669
|
else
|
|
657
670
|
to_boolean(eval_expr(e, node, position, size))
|
|
658
671
|
end
|
|
@@ -672,7 +685,7 @@ module FastHtmlParser
|
|
|
672
685
|
case e.op
|
|
673
686
|
when 0, 1 # E_LITERAL, E_NUMBER
|
|
674
687
|
e.value
|
|
675
|
-
when 2, 3, 4, 8, 9 # E_AND, E_OR, E_COMPARE, E_ATTR_EQUALS, E_ATTR_EXISTS
|
|
688
|
+
when 2, 3, 4, 8, 9, 13 # E_AND, E_OR, E_COMPARE, E_ATTR_EQUALS, E_ATTR_EXISTS, E_ATTR_TEST
|
|
676
689
|
eval_bool(e, node, position, size)
|
|
677
690
|
when 5 # E_ARITH
|
|
678
691
|
x = eval_number(e.l, node, position, size)
|
|
@@ -621,7 +621,33 @@ module FastHtmlParser
|
|
|
621
621
|
fail_at!("#{fn}() needs a node-set argument", t.p)
|
|
622
622
|
end
|
|
623
623
|
pos = fn == "position" || fn == "last" || args.any?(&:pos)
|
|
624
|
-
expr(X::E_FUNCTION, info[2], pos, fn: fn, args: args.freeze)
|
|
624
|
+
attribute_test(fn, args) || expr(X::E_FUNCTION, info[2], pos, fn: fn, args: args.freeze)
|
|
625
625
|
end
|
|
626
|
+
|
|
627
|
+
# contains(@a, 'x'), starts-with(...), ends-with(...), also with the attribute wrapped in literals
|
|
628
|
+
# (contains(concat(' ', @class, ' '), ' x '), the usual class test), are evaluated on the attribute in place,
|
|
629
|
+
# without a node-set for every element. nil: a plain function call.
|
|
630
|
+
def attribute_test(fn, args)
|
|
631
|
+
return nil unless %w[contains starts-with ends-with].include?(fn) && args[1].op == X::E_LITERAL
|
|
632
|
+
|
|
633
|
+
arg = args[0]
|
|
634
|
+
name = attribute_name(arg)
|
|
635
|
+
prefix = suffix = EMPTY_BINARY
|
|
636
|
+
if name.nil?
|
|
637
|
+
return nil unless arg.op == X::E_FUNCTION && arg.fn == "concat" && arg.args.size == 3 &&
|
|
638
|
+
arg.args[0].op == X::E_LITERAL && arg.args[2].op == X::E_LITERAL
|
|
639
|
+
|
|
640
|
+
name = attribute_name(arg.args[1])
|
|
641
|
+
return nil if name.nil?
|
|
642
|
+
|
|
643
|
+
prefix = arg.args[0].value.b.freeze
|
|
644
|
+
suffix = arg.args[2].value.b.freeze
|
|
645
|
+
end
|
|
646
|
+
expr(X::E_ATTR_TEST, X::TYPE_BOOLEAN, false, fn: fn, l: prefix, name: name, r: suffix,
|
|
647
|
+
value: args[1].value.b.freeze)
|
|
648
|
+
end
|
|
649
|
+
|
|
650
|
+
EMPTY_BINARY = "".b.freeze
|
|
651
|
+
private_constant :EMPTY_BINARY
|
|
626
652
|
end
|
|
627
653
|
end
|