fasthtmlparser 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.md +79 -0
- data/README.md +196 -0
- data/fasthtmlparser.gemspec +26 -0
- data/lib/fasthtmlparser/attr.rb +243 -0
- data/lib/fasthtmlparser/attrs.rb +278 -0
- data/lib/fasthtmlparser/chars.rb +258 -0
- data/lib/fasthtmlparser/css.rb +1061 -0
- data/lib/fasthtmlparser/decode.rb +164 -0
- data/lib/fasthtmlparser/entities.rb +2251 -0
- data/lib/fasthtmlparser/html.rb +52 -0
- data/lib/fasthtmlparser/html_document.rb +294 -0
- data/lib/fasthtmlparser/html_node.rb +706 -0
- data/lib/fasthtmlparser/links.rb +499 -0
- data/lib/fasthtmlparser/parser.rb +893 -0
- data/lib/fasthtmlparser/query.rb +221 -0
- data/lib/fasthtmlparser/tags.rb +761 -0
- data/lib/fasthtmlparser/text.rb +280 -0
- data/lib/fasthtmlparser/version.rb +5 -0
- data/lib/fasthtmlparser/xpath.rb +1151 -0
- data/lib/fasthtmlparser/xpath_compiler.rb +627 -0
- data/lib/fasthtmlparser.rb +35 -0
- metadata +66 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 24f9a12891873d08d5ae813a2c5c1289871c68a1be17dae6be79539d3de10169
|
|
4
|
+
data.tar.gz: 0caf2683698d389ffa302357974666f02f920047b8c9fcfb8700b5a98a32cbcc
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: a8078a025140b4b879aad856879d9c6d212f62c8c1e551761c806df27d03344e4897547cd02721b3ce69a0c917c94863e6474e260a0172dfdfbb8052c4f3e5e5
|
|
7
|
+
data.tar.gz: 98605f58999f0ac0977c57ee736fdf727ac4444f2283666b0045639c3b17fef4be6cff2f51a87b2f191a745c41a10b9a5f80a658f2ca628447c959ef34ae7010
|
data/LICENSE.md
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# z3l.FastHtmlParser License
|
|
2
|
+
|
|
3
|
+
Copyright © 2026 z3l. All rights reserved.
|
|
4
|
+
|
|
5
|
+
This license applies to the z3l.FastHtmlParser library, its source code, binaries and
|
|
6
|
+
NuGet packages (the "Software").
|
|
7
|
+
|
|
8
|
+
## 1. Definitions
|
|
9
|
+
|
|
10
|
+
- **"Licensee"** — the individual or legal entity that uses the Software, together with its
|
|
11
|
+
Affiliates.
|
|
12
|
+
- **"Affiliate"** — any entity that controls, is controlled by, or is under common control with
|
|
13
|
+
the Licensee.
|
|
14
|
+
- **"Annual Revenue"** — the total gross revenue of the Licensee (including all Affiliates) for
|
|
15
|
+
the last completed fiscal year, in US dollars or equivalent.
|
|
16
|
+
- **"Headcount"** — the total number of employees and individual contractors working for the
|
|
17
|
+
Licensee (including all Affiliates) on a full-time or full-time-equivalent basis, at any time
|
|
18
|
+
during the last 12 months.
|
|
19
|
+
- **"Small Licensee"** — a Licensee whose Annual Revenue is **below USD 1,000,000** and whose
|
|
20
|
+
Headcount is **below 100**. Both conditions must be met.
|
|
21
|
+
|
|
22
|
+
## 2. Free License (Small Licensees)
|
|
23
|
+
|
|
24
|
+
If the Licensee is a Small Licensee (Annual Revenue **below USD 1,000,000** and Headcount
|
|
25
|
+
**below 100**), the Licensee is granted, free of charge, a worldwide, non-exclusive,
|
|
26
|
+
non-transferable license to use, copy and modify the Software and to distribute it as part of the
|
|
27
|
+
Licensee's own applications and services, including commercial ones.
|
|
28
|
+
|
|
29
|
+
## 3. Commercial License (all other Licensees)
|
|
30
|
+
|
|
31
|
+
If the Licensee's Annual Revenue is **USD 1,000,000 or more**, or its Headcount is **100 or
|
|
32
|
+
more**, the Licensee must purchase a Commercial License.
|
|
33
|
+
|
|
34
|
+
- Price: a **one-time payment of USD 1,000** per Licensee (the Licensee together with all its
|
|
35
|
+
Affiliates).
|
|
36
|
+
- The Commercial License is perpetual and covers all current and future versions of the
|
|
37
|
+
Software, any number of developers, applications, servers and deployments of the Licensee.
|
|
38
|
+
- A Licensee that stops being a Small Licensee (its Annual Revenue or Headcount reaches the
|
|
39
|
+
threshold) while already using the Software must purchase the Commercial License within
|
|
40
|
+
30 days.
|
|
41
|
+
- To purchase: https://romanti3172.github.io/z3l.FastHtmlParser/#pricing, or contact **romanti3172@gmail.com** for an invoice.
|
|
42
|
+
|
|
43
|
+
The Commercial License grants the same rights as section 2, without the revenue and headcount
|
|
44
|
+
limits.
|
|
45
|
+
|
|
46
|
+
**One license for all z3l libraries.** The Commercial License is not limited to the Software: it
|
|
47
|
+
also covers z3l.Scraper and every other library and package published by z3l, now or in the
|
|
48
|
+
future, on the same terms. The Licensee pays USD 1,000 once and needs no other z3l license.
|
|
49
|
+
|
|
50
|
+
## 4. Conditions (both licenses)
|
|
51
|
+
|
|
52
|
+
1. This license text and the copyright notice must be kept in all copies or substantial portions
|
|
53
|
+
of the Software's source code.
|
|
54
|
+
2. The Licensee may not sell, sublicense or redistribute the Software on its own, or as part of a
|
|
55
|
+
library, SDK or component whose main purpose is HTML/XML parsing, so that it competes with the
|
|
56
|
+
Software.
|
|
57
|
+
3. Modified versions of the Software may not be published or distributed under a different
|
|
58
|
+
license.
|
|
59
|
+
4. End users of the Licensee's applications and services do not need their own license to use
|
|
60
|
+
those applications and services.
|
|
61
|
+
|
|
62
|
+
## 5. Termination
|
|
63
|
+
|
|
64
|
+
The license terminates automatically if the Licensee breaches it and does not cure the breach
|
|
65
|
+
within 30 days after becoming aware of it. On termination the Licensee must stop using and
|
|
66
|
+
distributing the Software.
|
|
67
|
+
|
|
68
|
+
## 6. No Warranty
|
|
69
|
+
|
|
70
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
|
71
|
+
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
72
|
+
NONINFRINGEMENT.
|
|
73
|
+
|
|
74
|
+
## 7. Limitation of Liability
|
|
75
|
+
|
|
76
|
+
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
77
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
|
78
|
+
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. THE TOTAL LIABILITY OF
|
|
79
|
+
THE COPYRIGHT HOLDERS SHALL NOT EXCEED THE AMOUNT PAID BY THE LICENSEE FOR THE SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
# fasthtmlparser for Ruby
|
|
2
|
+
|
|
3
|
+
[](https://github.com/romanti3172/fasthtmlparser-rb/actions/workflows/ci.yml)
|
|
4
|
+
[](https://rubygems.org/gems/fasthtmlparser)
|
|
5
|
+
|
|
6
|
+
Ruby port of [z3l.FastHtmlParser](https://github.com/romanti3172/z3l.FastHtmlParser): a fast HTML / XML parser for
|
|
7
|
+
scraping, with **XPath 1.0** and **CSS selectors**. It is built for extracting data, not for building a DOM: a page
|
|
8
|
+
becomes a flat list of tags that hold only positions in the source. Text, attributes and entities are read when you
|
|
9
|
+
ask for them, and the values are substrings of the page.
|
|
10
|
+
|
|
11
|
+
The whole API is ported: the parser, attributes, HTML5 entities, visible text and text with line breaks,
|
|
12
|
+
`find` / `find_all` and `Query`, navigation, links, XPath 1.0, CSS selectors and the one-call shortcuts. There are also
|
|
13
|
+
ports for [Go](https://github.com/romanti3172/fasthtmlparser),
|
|
14
|
+
[JavaScript](https://github.com/romanti3172/fasthtmlparser-js), [Rust](https://github.com/romanti3172/fasthtmlparser-rs),
|
|
15
|
+
[Python](https://github.com/romanti3172/fasthtmlparser-py), [Java](https://github.com/romanti3172/fasthtmlparser-java)
|
|
16
|
+
and [PHP](https://github.com/romanti3172/fasthtmlparser-php).
|
|
17
|
+
|
|
18
|
+
```
|
|
19
|
+
gem install fasthtmlparser
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Pure Ruby, no dependencies. Ruby 3.2 or later.
|
|
23
|
+
|
|
24
|
+
```ruby
|
|
25
|
+
require "fasthtmlparser"
|
|
26
|
+
include FastHtmlParser # Query, Tag, Attr, Html, CssQuery, XPathQuery
|
|
27
|
+
|
|
28
|
+
doc = FastHtmlParser.parse(page, "https://example.com/catalog/") # UTF-8; a byte order mark is skipped
|
|
29
|
+
|
|
30
|
+
# CSS selectors, as querySelectorAll / querySelector / matches
|
|
31
|
+
doc.css("article.product_pod").each do |pod|
|
|
32
|
+
title = pod.css_first("h3 a").attr("title")
|
|
33
|
+
price = pod.css_first("p.price_color").text
|
|
34
|
+
end
|
|
35
|
+
doc.css_first("ul#menu > li:last-child a").attr("href")
|
|
36
|
+
doc.css("tr:has(> td.price)")
|
|
37
|
+
card.css("> span") # the children of card
|
|
38
|
+
|
|
39
|
+
# XPath 1.0
|
|
40
|
+
doc.xpath("//div[@class='card']//span[@class='price']")
|
|
41
|
+
doc.xpath_values("//a[contains(@href, '/product/')]/@href")
|
|
42
|
+
doc.xpath_value("//dt[.='Price']/following-sibling::dd[1]")
|
|
43
|
+
doc.xpath_value("count(//table//tr)") # "12"
|
|
44
|
+
|
|
45
|
+
# compile once, reuse on any number of pages (and threads)
|
|
46
|
+
TITLES = XPathQuery.compile("//article//h3/a/@title")
|
|
47
|
+
TITLES.values(doc)
|
|
48
|
+
|
|
49
|
+
# search by name, class, id, attribute, text
|
|
50
|
+
doc.find_all(Query.by(:article).has_class("product_pod")).each do |pod|
|
|
51
|
+
pod.find(Query.by(:p).has_class("price_color")).text
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# one-call shortcuts (find_by_name_*, find_all_by_name_*, is_name_*?, parent_by_name, next_sibling_by_name,
|
|
55
|
+
# attribute_*, ...)
|
|
56
|
+
pod = doc.find_by_name_class(:article, "product_pod")
|
|
57
|
+
pod.find_by_name_itemprop(:span, "price").text
|
|
58
|
+
doc.find_by_name_text(:dt, "Price").next_sibling_by_name(:dd).text
|
|
59
|
+
doc.find_by_name_class("my-widget", "x") # any name works as a String or Symbol
|
|
60
|
+
|
|
61
|
+
# reading a tag
|
|
62
|
+
tag.text # visible text, entities decoded
|
|
63
|
+
tag.text_line_breaks # <br> and block elements -> new line, <p> and headings -> paragraph
|
|
64
|
+
tag.attr(Attr::DATA_SRC) # decoded value; attr_raw: as written; has_attr?, has_class?, attrs
|
|
65
|
+
tag.href # absolute URL, resolved against the page URL; src for src
|
|
66
|
+
tag.href_mail_to # the address of a mailto: link; href_telephone for tel:
|
|
67
|
+
tag.source # the opening tag without '<' and '>'
|
|
68
|
+
tag.name; tag.tag; tag.depth; tag.index; tag.start; tag.end
|
|
69
|
+
|
|
70
|
+
# navigation
|
|
71
|
+
tag.parent; tag.children; tag.next_sibling; tag.previous_sibling; tag.next; tag.previous
|
|
72
|
+
tag.parent_matching(Query.by(:table))
|
|
73
|
+
tag.text_after_until(Query.by(:br)) # the source between the tag and the next <br>
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Queries: `Query.by(:div)`, `Query.by("custom-tag")` or `Query.any`, then `.has_class`, `.id`, `.attr`, `.text`,
|
|
77
|
+
`.id_starts_with`, `.text_starts_with`, `.body`, `.body_starts_with`, `.source`. Queries are immutable (frozen).
|
|
78
|
+
Tag names are symbols (`Tag::DIV` is `:div`, `Tag::COMMENT` is `:"!--"`) and `Attr` holds string constants
|
|
79
|
+
(`Attr::CLASS`, `Attr::DATA_SRC`); the two modules cover about 250 tag and 230 attribute names (HTML, SVG, MathML,
|
|
80
|
+
RSS, Atom, sitemap), and any other name works as a String or Symbol. Every method that takes a tag name takes a Symbol
|
|
81
|
+
or a String, case-insensitively.
|
|
82
|
+
|
|
83
|
+
The names are those of the other ports in Ruby style: `findByNameClass` is `find_by_name_class`, the predicates end
|
|
84
|
+
with `?` (`has_attr?`, `has_class?`, `has_children?`, `is?(query)`, `is_css?(selector)`, `is_name_class?(...)`).
|
|
85
|
+
|
|
86
|
+
A missing tag is `nil`; a missing value is `""`. The string methods (`css`, `xpath`, ...) compile and cache the query
|
|
87
|
+
and raise `CssSelectorError` / `XPathError` (both `ArgumentError`s) on an invalid one; `xpath` on an expression that
|
|
88
|
+
gives a number or string raises an `XPathError` (use `xpath_value`). `Html.to_text`, `Html.to_text_line_breaks` and
|
|
89
|
+
`Html.decode` work on HTML fragments. A document is an `Enumerable` of its tags (`doc[i]`, `doc.size`, `doc.each`).
|
|
90
|
+
|
|
91
|
+
Documents are read-only, and documents and compiled queries can be shared between threads. The tags are kept as
|
|
92
|
+
arrays of positions; a tag object (`HtmlNode`) is made the first time a search, a selector or `node(i)` returns it.
|
|
93
|
+
|
|
94
|
+
CSS: type, `*`, `#id`, `.class`, attribute selectors with all operators and the `i` / `s` flags, the four combinators,
|
|
95
|
+
selector lists, `:first-child`, `:last-child`, `:only-child`, `:*-of-type`, `:nth-child(An+B [of S])` and its
|
|
96
|
+
relatives, `:not()`, `:is()`, `:where()`, `:has()`, `:empty`, `:root`, `:scope`, `:link`, `:checked`, `:disabled`,
|
|
97
|
+
`:enabled`, `:required`, `:optional`, `:contains("text")`. XPath: all axes except namespace, predicates, `|`, the
|
|
98
|
+
operators and all XPath 1.0 functions, plus `ends-with()`, `lower-case()`, `upper-case()`. HTML rules apply: names are
|
|
99
|
+
case-insensitive, the string-value of an element is its visible text, attribute values and text are decoded.
|
|
100
|
+
|
|
101
|
+
Parsing follows what a browser makes of the page where it matters for scraping: HTML5 optional end tags (`<li>`, `<p>`,
|
|
102
|
+
`<td>`, `<tr>`, `<option>` without end tags give the same elements as a browser), raw text in `<script>`, `<style>`,
|
|
103
|
+
`<textarea>`, `<title>`, comments as HTML5 reads them, void elements, nesting flattened past 512 levels, all 2125
|
|
104
|
+
named entities, CR LF read as LF. XML documents (RSS, Atom, sitemaps) are parsed by XML rules. Not done: full HTML5
|
|
105
|
+
tree construction (no implied `<tbody>` or `<html>`), and documents are read-only.
|
|
106
|
+
|
|
107
|
+
## Strings and positions
|
|
108
|
+
|
|
109
|
+
The page is read as UTF-8 bytes, whatever the encoding of the String you pass (the string itself is not changed), and
|
|
110
|
+
every string the library gives back is UTF-8. Positions (`start`, `end`) are **byte offsets** into the source, as in
|
|
111
|
+
the Go, Rust, Python and PHP ports: `doc.source.byteslice(tag.start, tag.end - tag.start + 1)` is the whole element.
|
|
112
|
+
(.NET, Java and JavaScript count UTF-16 units; the conformance data holds byte offsets, so every port is checked
|
|
113
|
+
against the same numbers.) XPath counts as .NET does: `string-length()` gives UTF-16 units (an emoji is 2),
|
|
114
|
+
`substring()` and `translate()` work on characters. Broken UTF-8 does not raise: the bytes are kept as they are.
|
|
115
|
+
|
|
116
|
+
Intended differences from .NET: `text_line_breaks` joins lines with `"\n"` on every OS (.NET uses
|
|
117
|
+
`Environment.NewLine`), and a missing value is `""` where .NET returns null. Links resolved against the page are
|
|
118
|
+
written as .NET's `System.Uri` writes them (host in lower case, no default port, readable escapes); absolute links are
|
|
119
|
+
returned as written.
|
|
120
|
+
|
|
121
|
+
## Same results as the .NET version
|
|
122
|
+
|
|
123
|
+
The conformance tests compare the port with z3l.FastHtmlParser 1.4.1 (.NET) on 117 pages: the test fixtures, the
|
|
124
|
+
broken-markup snippets of the .NET cross-check tests, extra edge cases (entities, raw text, comments, XML, custom
|
|
125
|
+
elements, 600-deep nesting, Unicode whitespace, links of every kind, line breaks), and every fixture with
|
|
126
|
+
`</li> </p> </td> </tr>` left out, plus three copies of each with random end tags dropped, names upper-cased and the
|
|
127
|
+
page cut off, and copies saved with CR LF and with a lone CR.
|
|
128
|
+
|
|
129
|
+
- `test_tags_texts_attributes_navigation_and_links_*`, every tag (over 15,000): names, parents, depths, positions,
|
|
130
|
+
visible text, text with line breaks, 10 attribute values, the opening tag source, next / previous tag and sibling,
|
|
131
|
+
children, the resolved href and src, mailto: and tel: values, and the text after the tag must be identical.
|
|
132
|
+
- `test_xpath_and_css_results_*`, 569 queries: every string of the .NET XPath and CSS tests plus extra ones for the
|
|
133
|
+
axes, functions and selectors. Whether each one compiles must be the same (119 are invalid); the 228 XPath and 222
|
|
134
|
+
CSS queries that compile are run on every page, on the document and from three tags of the page, and every CSS
|
|
135
|
+
selector is also matched against every tag. Selected tags and values must be identical.
|
|
136
|
+
|
|
137
|
+
All pass on Ruby 3.2 and 4.0, with one documented exception: on the page nested 600 levels deep, where the .NET result
|
|
138
|
+
depends on the order of an unstable sort.
|
|
139
|
+
|
|
140
|
+
The expected output in `testdata/conformance` is produced by `tools/conformance`, a small .NET program that references
|
|
141
|
+
the z3l.FastHtmlParser package; it is shared with the other ports.
|
|
142
|
+
|
|
143
|
+
## Benchmarks
|
|
144
|
+
|
|
145
|
+
`cd bench && bundle install && bundle exec ruby bench.rb` (Ruby 4.0.9, AMD Ryzen 7 3800X, Windows; median time per call
|
|
146
|
+
after a one-second warm-up). The RubyInstaller builds for Windows have no YJIT, so these numbers are for the plain
|
|
147
|
+
interpreter; with YJIT (Linux, macOS) the pure-Ruby parser gains more than the C parsers it is compared with. Nokogiri
|
|
148
|
+
1.19.7 parses with libxml2 2.13 (`Nokogiri::HTML4`) or with gumbo, the HTML5 parser (`Nokogiri::HTML5`); Oga 3.5 is a
|
|
149
|
+
pure-Ruby parser. Before timing, the benchmark checks that every library extracts exactly the same records.
|
|
150
|
+
|
|
151
|
+
Parse a page:
|
|
152
|
+
|
|
153
|
+
| Page | fasthtmlparser | Nokogiri HTML4 | Nokogiri HTML5 | Oga |
|
|
154
|
+
|---|---:|---:|---:|---:|
|
|
155
|
+
| quotes, 11 KB | **456 μs** | 526 μs | 802 μs | 3.14 ms |
|
|
156
|
+
| laptops, 38 KB | **1.40 ms** | 1.79 ms | 2.38 ms | 10.11 ms |
|
|
157
|
+
| books, 51 KB | 1.54 ms | **1.45 ms** | 3.23 ms | 10.28 ms |
|
|
158
|
+
| books x10, 510 KB | 15.47 ms | **14.44 ms** | 31.01 ms | 107.99 ms |
|
|
159
|
+
| books without `</li>` `</p>` | 1.97 ms | **1.56 ms** | 2.83 ms | 9.62 ms |
|
|
160
|
+
|
|
161
|
+
Scrape a page: parse and extract the records. Books: 20 × title, price, rating; quotes: 10 × text, author, tags.
|
|
162
|
+
|
|
163
|
+
| Task | fasthtmlparser `find` | CSS | XPath | Nokogiri HTML4 CSS | HTML4 XPath | HTML5 CSS | HTML5 XPath | Oga CSS | Oga XPath |
|
|
164
|
+
|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
|
165
|
+
| books | **2.26 ms** | 2.48 ms | 3.05 ms | 3.13 ms | 2.74 ms | 4.57 ms | 3.88 ms | 13.86 ms | 14.28 ms |
|
|
166
|
+
| quotes | **834 μs** | 881 μs | 1.21 ms | 1.32 ms | | 1.55 ms | | 4.66 ms | |
|
|
167
|
+
|
|
168
|
+
A pure-Ruby parser that keeps only positions parses about as fast as libxml2 and 2x faster than gumbo; extracting the
|
|
169
|
+
records is where it wins, because nothing is built that is not asked for. The full output is in `bench-results.md`.
|
|
170
|
+
|
|
171
|
+
## Development
|
|
172
|
+
|
|
173
|
+
```
|
|
174
|
+
bundle install
|
|
175
|
+
bundle exec rake test # API and conformance tests
|
|
176
|
+
ruby tools/gen.rb ../z3l.FastHtmlParser/src/z3l.FastHtmlParser # tag, attribute and entity tables, shortcuts
|
|
177
|
+
cd bench && bundle install && bundle exec ruby bench.rb # benchmarks (they load the library from ..)
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
Releases: bump `VERSION` in `lib/fasthtmlparser/version.rb`, commit, push a tag `v<version>`; `.github/workflows/publish.yml`
|
|
181
|
+
publishes the gem through rubygems.org trusted publishing (no API key stored).
|
|
182
|
+
|
|
183
|
+
- `lib/fasthtmlparser/parser.rb`: the scanner and the HTML5 optional-end-tag rules (port of `ParsingCore.cs`); it
|
|
184
|
+
writes the tags into parallel arrays of the document
|
|
185
|
+
- `lib/fasthtmlparser/attrs.rb`, `decode.rb`, `text.rb`: attributes, character references, visible text and text with
|
|
186
|
+
line breaks
|
|
187
|
+
- `lib/fasthtmlparser/html_document.rb`, `html_node.rb`, `query.rb`: search, navigation and the generated shortcuts;
|
|
188
|
+
`links.rb`: href / src
|
|
189
|
+
- `lib/fasthtmlparser/css.rb`: CSS selectors; `xpath.rb`, `xpath_compiler.rb`: XPath 1.0
|
|
190
|
+
- `lib/fasthtmlparser/tags.rb`, `attr.rb`, `entities.rb`: generated by `tools/gen.rb` from the C# sources
|
|
191
|
+
- `bench/`: benchmarks with their own Gemfile, so the library has no dependencies
|
|
192
|
+
|
|
193
|
+
## License
|
|
194
|
+
|
|
195
|
+
Source-available, not OSI open source: free for individuals and companies with annual revenue below USD 1,000,000
|
|
196
|
+
and fewer than 100 people; a one-time commercial license above that. See [LICENSE.md](LICENSE.md).
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "lib/fasthtmlparser/version"
|
|
4
|
+
|
|
5
|
+
Gem::Specification.new do |spec|
|
|
6
|
+
spec.name = "fasthtmlparser"
|
|
7
|
+
spec.version = FastHtmlParser::VERSION
|
|
8
|
+
spec.authors = ["z3l"]
|
|
9
|
+
spec.summary = "Fast HTML / XML parser for scraping, with XPath 1.0 and CSS selectors. Ruby port of z3l.FastHtmlParser."
|
|
10
|
+
spec.description = "A page becomes a flat list of tags that hold only positions in the source; text, attributes " \
|
|
11
|
+
"and entities are read when they are asked for. HTML5 optional end tags, all 2125 named " \
|
|
12
|
+
"entities, XPath 1.0, CSS selectors, links resolved against the page URL. Pure Ruby, no " \
|
|
13
|
+
"dependencies."
|
|
14
|
+
spec.homepage = "https://github.com/romanti3172/fasthtmlparser-rb"
|
|
15
|
+
spec.license = "LicenseRef-z3l-source-available"
|
|
16
|
+
spec.required_ruby_version = ">= 3.2"
|
|
17
|
+
|
|
18
|
+
spec.metadata = {
|
|
19
|
+
"source_code_uri" => spec.homepage,
|
|
20
|
+
"bug_tracker_uri" => "#{spec.homepage}/issues",
|
|
21
|
+
"rubygems_mfa_required" => "true"
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
spec.files = Dir["lib/**/*.rb"] + %w[LICENSE.md README.md fasthtmlparser.gemspec]
|
|
25
|
+
spec.require_paths = ["lib"]
|
|
26
|
+
end
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Code generated by tools/gen.rb from HtmlAttribute.cs and HtmlNames.cs; DO NOT EDIT.
|
|
4
|
+
|
|
5
|
+
module FastHtmlParser
|
|
6
|
+
# Common HTML / SVG / XML attribute names, for HtmlNode#attr, Query#attr and the shortcuts. Any other name
|
|
7
|
+
# works as a plain String or Symbol.
|
|
8
|
+
module Attr
|
|
9
|
+
ID = "id"
|
|
10
|
+
CLASS = "class"
|
|
11
|
+
STYLE = "style"
|
|
12
|
+
TITLE = "title"
|
|
13
|
+
LANG = "lang"
|
|
14
|
+
DIR = "dir"
|
|
15
|
+
HIDDEN = "hidden"
|
|
16
|
+
TABINDEX = "tabindex"
|
|
17
|
+
ROLE = "role"
|
|
18
|
+
ARIA_LABEL = "aria-label"
|
|
19
|
+
ARIA_HIDDEN = "aria-hidden"
|
|
20
|
+
HREF = "href"
|
|
21
|
+
SRC = "src"
|
|
22
|
+
SRCSET = "srcset"
|
|
23
|
+
SIZES = "sizes"
|
|
24
|
+
ALT = "alt"
|
|
25
|
+
REL = "rel"
|
|
26
|
+
TARGET = "target"
|
|
27
|
+
DOWNLOAD = "download"
|
|
28
|
+
HREFLANG = "hreflang"
|
|
29
|
+
POSTER = "poster"
|
|
30
|
+
LOADING = "loading"
|
|
31
|
+
WIDTH = "width"
|
|
32
|
+
HEIGHT = "height"
|
|
33
|
+
CONTENT = "content"
|
|
34
|
+
CHARSET = "charset"
|
|
35
|
+
HTTP_EQUIV = "http-equiv"
|
|
36
|
+
PROPERTY = "property"
|
|
37
|
+
ITEMPROP = "itemprop"
|
|
38
|
+
ITEMSCOPE = "itemscope"
|
|
39
|
+
ITEMTYPE = "itemtype"
|
|
40
|
+
NAME = "name"
|
|
41
|
+
VALUE = "value"
|
|
42
|
+
TYPE = "type"
|
|
43
|
+
PLACEHOLDER = "placeholder"
|
|
44
|
+
CHECKED = "checked"
|
|
45
|
+
SELECTED = "selected"
|
|
46
|
+
DISABLED = "disabled"
|
|
47
|
+
READONLY = "readonly"
|
|
48
|
+
REQUIRED = "required"
|
|
49
|
+
MULTIPLE = "multiple"
|
|
50
|
+
MAXLENGTH = "maxlength"
|
|
51
|
+
MIN = "min"
|
|
52
|
+
MAX = "max"
|
|
53
|
+
STEP = "step"
|
|
54
|
+
PATTERN = "pattern"
|
|
55
|
+
AUTOCOMPLETE = "autocomplete"
|
|
56
|
+
FOR = "for"
|
|
57
|
+
FORM = "form"
|
|
58
|
+
ACTION = "action"
|
|
59
|
+
METHOD = "method"
|
|
60
|
+
ENCTYPE = "enctype"
|
|
61
|
+
LABEL = "label"
|
|
62
|
+
COLSPAN = "colspan"
|
|
63
|
+
ROWSPAN = "rowspan"
|
|
64
|
+
DATETIME = "datetime"
|
|
65
|
+
ACCESSKEY = "accesskey"
|
|
66
|
+
AUTOCAPITALIZE = "autocapitalize"
|
|
67
|
+
AUTOFOCUS = "autofocus"
|
|
68
|
+
CONTENTEDITABLE = "contenteditable"
|
|
69
|
+
DRAGGABLE = "draggable"
|
|
70
|
+
ENTERKEYHINT = "enterkeyhint"
|
|
71
|
+
INERT = "inert"
|
|
72
|
+
INPUTMODE = "inputmode"
|
|
73
|
+
IS = "is"
|
|
74
|
+
NONCE = "nonce"
|
|
75
|
+
PART = "part"
|
|
76
|
+
SLOT = "slot"
|
|
77
|
+
SPELLCHECK = "spellcheck"
|
|
78
|
+
TRANSLATE = "translate"
|
|
79
|
+
POPOVER = "popover"
|
|
80
|
+
ITEMID = "itemid"
|
|
81
|
+
ITEMREF = "itemref"
|
|
82
|
+
ARIA_DESCRIBEDBY = "aria-describedby"
|
|
83
|
+
ARIA_LABELLEDBY = "aria-labelledby"
|
|
84
|
+
ARIA_EXPANDED = "aria-expanded"
|
|
85
|
+
ARIA_CONTROLS = "aria-controls"
|
|
86
|
+
ARIA_CURRENT = "aria-current"
|
|
87
|
+
ARIA_HASPOPUP = "aria-haspopup"
|
|
88
|
+
ARIA_SELECTED = "aria-selected"
|
|
89
|
+
ARIA_DISABLED = "aria-disabled"
|
|
90
|
+
ARIA_CHECKED = "aria-checked"
|
|
91
|
+
ARIA_LIVE = "aria-live"
|
|
92
|
+
DATA_SRC = "data-src"
|
|
93
|
+
DATA_SRCSET = "data-srcset"
|
|
94
|
+
DATA_LAZY_SRC = "data-lazy-src"
|
|
95
|
+
DATA_ORIGINAL = "data-original"
|
|
96
|
+
DATA_BG = "data-bg"
|
|
97
|
+
DATA_BACKGROUND = "data-background"
|
|
98
|
+
DATA_IMAGE = "data-image"
|
|
99
|
+
DATA_HREF = "data-href"
|
|
100
|
+
DATA_URL = "data-url"
|
|
101
|
+
DATA_LINK = "data-link"
|
|
102
|
+
DATA_ID = "data-id"
|
|
103
|
+
DATA_VALUE = "data-value"
|
|
104
|
+
DATA_NAME = "data-name"
|
|
105
|
+
DATA_TYPE = "data-type"
|
|
106
|
+
DATA_TITLE = "data-title"
|
|
107
|
+
DATA_TEXT = "data-text"
|
|
108
|
+
DATA_PHONE = "data-phone"
|
|
109
|
+
DATA_PRICE = "data-price"
|
|
110
|
+
DATA_TARGET = "data-target"
|
|
111
|
+
DATA_TOGGLE = "data-toggle"
|
|
112
|
+
DATA_CONTENT = "data-content"
|
|
113
|
+
ONCLICK = "onclick"
|
|
114
|
+
ONLOAD = "onload"
|
|
115
|
+
ONERROR = "onerror"
|
|
116
|
+
ONCHANGE = "onchange"
|
|
117
|
+
ONINPUT = "oninput"
|
|
118
|
+
ONSUBMIT = "onsubmit"
|
|
119
|
+
ONFOCUS = "onfocus"
|
|
120
|
+
ONBLUR = "onblur"
|
|
121
|
+
ONKEYDOWN = "onkeydown"
|
|
122
|
+
ONKEYUP = "onkeyup"
|
|
123
|
+
ONMOUSEOVER = "onmouseover"
|
|
124
|
+
ONMOUSEOUT = "onmouseout"
|
|
125
|
+
PING = "ping"
|
|
126
|
+
REFERRERPOLICY = "referrerpolicy"
|
|
127
|
+
MEDIA = "media"
|
|
128
|
+
CROSSORIGIN = "crossorigin"
|
|
129
|
+
INTEGRITY = "integrity"
|
|
130
|
+
AS = "as"
|
|
131
|
+
BLOCKING = "blocking"
|
|
132
|
+
FETCHPRIORITY = "fetchpriority"
|
|
133
|
+
DECODING = "decoding"
|
|
134
|
+
USEMAP = "usemap"
|
|
135
|
+
ISMAP = "ismap"
|
|
136
|
+
AUTOPLAY = "autoplay"
|
|
137
|
+
CONTROLS = "controls"
|
|
138
|
+
LOOP = "loop"
|
|
139
|
+
MUTED = "muted"
|
|
140
|
+
PLAYSINLINE = "playsinline"
|
|
141
|
+
PRELOAD = "preload"
|
|
142
|
+
KIND = "kind"
|
|
143
|
+
SRCLANG = "srclang"
|
|
144
|
+
DEFAULT = "default"
|
|
145
|
+
DATA = "data"
|
|
146
|
+
ALLOW = "allow"
|
|
147
|
+
ALLOWFULLSCREEN = "allowfullscreen"
|
|
148
|
+
SANDBOX = "sandbox"
|
|
149
|
+
SRCDOC = "srcdoc"
|
|
150
|
+
FRAMEBORDER = "frameborder"
|
|
151
|
+
SCROLLING = "scrolling"
|
|
152
|
+
COORDS = "coords"
|
|
153
|
+
SHAPE = "shape"
|
|
154
|
+
ASYNC = "async"
|
|
155
|
+
DEFER = "defer"
|
|
156
|
+
NOMODULE = "nomodule"
|
|
157
|
+
CITE = "cite"
|
|
158
|
+
MANIFEST = "manifest"
|
|
159
|
+
SCHEME = "scheme"
|
|
160
|
+
VOCAB = "vocab"
|
|
161
|
+
TYPEOF = "typeof"
|
|
162
|
+
RESOURCE = "resource"
|
|
163
|
+
PREFIX = "prefix"
|
|
164
|
+
ABOUT = "about"
|
|
165
|
+
DATATYPE = "datatype"
|
|
166
|
+
ACCEPT = "accept"
|
|
167
|
+
ACCEPT_CHARSET = "accept-charset"
|
|
168
|
+
NOVALIDATE = "novalidate"
|
|
169
|
+
FORMACTION = "formaction"
|
|
170
|
+
FORMMETHOD = "formmethod"
|
|
171
|
+
FORMENCTYPE = "formenctype"
|
|
172
|
+
FORMTARGET = "formtarget"
|
|
173
|
+
FORMNOVALIDATE = "formnovalidate"
|
|
174
|
+
LIST = "list"
|
|
175
|
+
DIRNAME = "dirname"
|
|
176
|
+
SIZE = "size"
|
|
177
|
+
COLS = "cols"
|
|
178
|
+
ROWS = "rows"
|
|
179
|
+
WRAP = "wrap"
|
|
180
|
+
MINLENGTH = "minlength"
|
|
181
|
+
HIGH = "high"
|
|
182
|
+
LOW = "low"
|
|
183
|
+
OPTIMUM = "optimum"
|
|
184
|
+
OPEN = "open"
|
|
185
|
+
START = "start"
|
|
186
|
+
REVERSED = "reversed"
|
|
187
|
+
HEADERS = "headers"
|
|
188
|
+
SCOPE = "scope"
|
|
189
|
+
SPAN = "span"
|
|
190
|
+
ABBR = "abbr"
|
|
191
|
+
ALIGN = "align"
|
|
192
|
+
VALIGN = "valign"
|
|
193
|
+
BGCOLOR = "bgcolor"
|
|
194
|
+
BORDER = "border"
|
|
195
|
+
CELLPADDING = "cellpadding"
|
|
196
|
+
CELLSPACING = "cellspacing"
|
|
197
|
+
COLOR = "color"
|
|
198
|
+
FACE = "face"
|
|
199
|
+
BACKGROUND = "background"
|
|
200
|
+
NOWRAP = "nowrap"
|
|
201
|
+
LANGUAGE = "language"
|
|
202
|
+
VIEW_BOX = "viewbox"
|
|
203
|
+
PRESERVE_ASPECT_RATIO = "preserveaspectratio"
|
|
204
|
+
D = "d"
|
|
205
|
+
FILL = "fill"
|
|
206
|
+
FILL_RULE = "fill-rule"
|
|
207
|
+
FILL_OPACITY = "fill-opacity"
|
|
208
|
+
STROKE = "stroke"
|
|
209
|
+
STROKE_WIDTH = "stroke-width"
|
|
210
|
+
STROKE_LINECAP = "stroke-linecap"
|
|
211
|
+
STROKE_LINEJOIN = "stroke-linejoin"
|
|
212
|
+
OPACITY = "opacity"
|
|
213
|
+
TRANSFORM = "transform"
|
|
214
|
+
POINTS = "points"
|
|
215
|
+
X = "x"
|
|
216
|
+
Y = "y"
|
|
217
|
+
X1 = "x1"
|
|
218
|
+
Y1 = "y1"
|
|
219
|
+
X2 = "x2"
|
|
220
|
+
Y2 = "y2"
|
|
221
|
+
CX = "cx"
|
|
222
|
+
CY = "cy"
|
|
223
|
+
R = "r"
|
|
224
|
+
RX = "rx"
|
|
225
|
+
RY = "ry"
|
|
226
|
+
DX = "dx"
|
|
227
|
+
DY = "dy"
|
|
228
|
+
OFFSET = "offset"
|
|
229
|
+
STOP_COLOR = "stop-color"
|
|
230
|
+
CLIP_PATH = "clip-path"
|
|
231
|
+
FONT_SIZE = "font-size"
|
|
232
|
+
FONT_FAMILY = "font-family"
|
|
233
|
+
TEXT_ANCHOR = "text-anchor"
|
|
234
|
+
XMLNS = "xmlns"
|
|
235
|
+
XMLNS_XLINK = "xmlns:xlink"
|
|
236
|
+
XLINK_HREF = "xlink:href"
|
|
237
|
+
XML_LANG = "xml:lang"
|
|
238
|
+
XML_SPACE = "xml:space"
|
|
239
|
+
VERSION = "version"
|
|
240
|
+
ENCODING = "encoding"
|
|
241
|
+
STANDALONE = "standalone"
|
|
242
|
+
end
|
|
243
|
+
end
|