nokogiri 1.11.4 → 1.13.2
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of nokogiri might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Gemfile +2 -0
- data/LICENSE-DEPENDENCIES.md +243 -22
- data/LICENSE.md +1 -1
- data/README.md +14 -11
- data/bin/nokogiri +63 -50
- data/dependencies.yml +11 -62
- data/ext/nokogiri/depend +35 -34
- data/ext/nokogiri/extconf.rb +235 -126
- data/ext/nokogiri/gumbo.c +584 -0
- data/ext/nokogiri/{html_document.c → html4_document.c} +8 -8
- data/ext/nokogiri/{html_element_description.c → html4_element_description.c} +21 -19
- data/ext/nokogiri/{html_entity_lookup.c → html4_entity_lookup.c} +7 -7
- data/ext/nokogiri/{html_sax_parser_context.c → html4_sax_parser_context.c} +6 -5
- data/ext/nokogiri/{html_sax_push_parser.c → html4_sax_push_parser.c} +4 -4
- data/ext/nokogiri/libxml2_backwards_compat.c +30 -30
- data/ext/nokogiri/nokogiri.c +70 -38
- data/ext/nokogiri/nokogiri.h +19 -9
- data/ext/nokogiri/xml_document.c +49 -49
- data/ext/nokogiri/xml_document_fragment.c +0 -2
- data/ext/nokogiri/xml_dtd.c +2 -2
- data/ext/nokogiri/xml_element_content.c +2 -0
- data/ext/nokogiri/xml_encoding_handler.c +31 -12
- data/ext/nokogiri/xml_namespace.c +4 -2
- data/ext/nokogiri/xml_node.c +759 -439
- data/ext/nokogiri/xml_node_set.c +20 -20
- data/ext/nokogiri/xml_reader.c +39 -11
- data/ext/nokogiri/xml_sax_parser.c +6 -6
- data/ext/nokogiri/xml_sax_parser_context.c +2 -0
- data/ext/nokogiri/xml_schema.c +2 -0
- data/ext/nokogiri/xml_xpath_context.c +109 -84
- data/ext/nokogiri/xslt_stylesheet.c +109 -10
- data/gumbo-parser/CHANGES.md +63 -0
- data/gumbo-parser/Makefile +101 -0
- data/gumbo-parser/THANKS +27 -0
- data/gumbo-parser/src/Makefile +34 -0
- data/gumbo-parser/src/README.md +41 -0
- data/gumbo-parser/src/ascii.c +75 -0
- data/gumbo-parser/src/ascii.h +115 -0
- data/gumbo-parser/src/attribute.c +42 -0
- data/gumbo-parser/src/attribute.h +17 -0
- data/gumbo-parser/src/char_ref.c +22225 -0
- data/gumbo-parser/src/char_ref.h +29 -0
- data/gumbo-parser/src/char_ref.rl +2154 -0
- data/gumbo-parser/src/error.c +626 -0
- data/gumbo-parser/src/error.h +148 -0
- data/gumbo-parser/src/foreign_attrs.c +104 -0
- data/gumbo-parser/src/foreign_attrs.gperf +27 -0
- data/gumbo-parser/src/gumbo.h +943 -0
- data/gumbo-parser/src/insertion_mode.h +33 -0
- data/gumbo-parser/src/macros.h +91 -0
- data/gumbo-parser/src/parser.c +4875 -0
- data/gumbo-parser/src/parser.h +41 -0
- data/gumbo-parser/src/replacement.h +33 -0
- data/gumbo-parser/src/string_buffer.c +103 -0
- data/gumbo-parser/src/string_buffer.h +68 -0
- data/gumbo-parser/src/string_piece.c +48 -0
- data/gumbo-parser/src/svg_attrs.c +174 -0
- data/gumbo-parser/src/svg_attrs.gperf +77 -0
- data/gumbo-parser/src/svg_tags.c +137 -0
- data/gumbo-parser/src/svg_tags.gperf +55 -0
- data/gumbo-parser/src/tag.c +222 -0
- data/gumbo-parser/src/tag_lookup.c +382 -0
- data/gumbo-parser/src/tag_lookup.gperf +169 -0
- data/gumbo-parser/src/tag_lookup.h +13 -0
- data/gumbo-parser/src/token_buffer.c +79 -0
- data/gumbo-parser/src/token_buffer.h +71 -0
- data/gumbo-parser/src/token_type.h +17 -0
- data/gumbo-parser/src/tokenizer.c +3463 -0
- data/gumbo-parser/src/tokenizer.h +112 -0
- data/gumbo-parser/src/tokenizer_states.h +339 -0
- data/gumbo-parser/src/utf8.c +245 -0
- data/gumbo-parser/src/utf8.h +164 -0
- data/gumbo-parser/src/util.c +68 -0
- data/gumbo-parser/src/util.h +30 -0
- data/gumbo-parser/src/vector.c +111 -0
- data/gumbo-parser/src/vector.h +45 -0
- data/lib/nokogiri/class_resolver.rb +67 -0
- data/lib/nokogiri/css/node.rb +9 -8
- data/lib/nokogiri/css/parser.rb +361 -342
- data/lib/nokogiri/css/parser.y +250 -245
- data/lib/nokogiri/css/parser_extras.rb +20 -20
- data/lib/nokogiri/css/syntax_error.rb +2 -1
- data/lib/nokogiri/css/tokenizer.rb +4 -3
- data/lib/nokogiri/css/tokenizer.rex +3 -2
- data/lib/nokogiri/css/xpath_visitor.rb +179 -82
- data/lib/nokogiri/css.rb +49 -17
- data/lib/nokogiri/decorators/slop.rb +8 -7
- data/lib/nokogiri/extension.rb +8 -3
- data/lib/nokogiri/gumbo.rb +15 -0
- data/lib/nokogiri/html.rb +37 -27
- data/lib/nokogiri/{html → html4}/builder.rb +3 -2
- data/lib/nokogiri/{html → html4}/document.rb +88 -79
- data/lib/nokogiri/{html → html4}/document_fragment.rb +13 -9
- data/lib/nokogiri/{html → html4}/element_description.rb +2 -1
- data/lib/nokogiri/html4/element_description_defaults.rb +578 -0
- data/lib/nokogiri/{html → html4}/entity_lookup.rb +3 -2
- data/lib/nokogiri/{html → html4}/sax/parser.rb +13 -15
- data/lib/nokogiri/html4/sax/parser_context.rb +20 -0
- data/lib/nokogiri/{html → html4}/sax/push_parser.rb +11 -11
- data/lib/nokogiri/html4.rb +46 -0
- data/lib/nokogiri/html5/document.rb +88 -0
- data/lib/nokogiri/html5/document_fragment.rb +83 -0
- data/lib/nokogiri/html5/node.rb +96 -0
- data/lib/nokogiri/html5.rb +477 -0
- data/lib/nokogiri/jruby/dependencies.rb +10 -9
- data/lib/nokogiri/syntax_error.rb +1 -0
- data/lib/nokogiri/version/constant.rb +2 -1
- data/lib/nokogiri/version/info.rb +30 -14
- data/lib/nokogiri/version.rb +1 -0
- data/lib/nokogiri/xml/attr.rb +5 -3
- data/lib/nokogiri/xml/attribute_decl.rb +2 -1
- data/lib/nokogiri/xml/builder.rb +69 -31
- data/lib/nokogiri/xml/cdata.rb +2 -1
- data/lib/nokogiri/xml/character_data.rb +1 -0
- data/lib/nokogiri/xml/document.rb +178 -96
- data/lib/nokogiri/xml/document_fragment.rb +41 -38
- data/lib/nokogiri/xml/dtd.rb +3 -2
- data/lib/nokogiri/xml/element_content.rb +1 -0
- data/lib/nokogiri/xml/element_decl.rb +2 -1
- data/lib/nokogiri/xml/entity_decl.rb +3 -2
- data/lib/nokogiri/xml/entity_reference.rb +1 -0
- data/lib/nokogiri/xml/namespace.rb +2 -0
- data/lib/nokogiri/xml/node/save_options.rb +7 -4
- data/lib/nokogiri/xml/node.rb +516 -351
- data/lib/nokogiri/xml/node_set.rb +46 -54
- data/lib/nokogiri/xml/notation.rb +12 -0
- data/lib/nokogiri/xml/parse_options.rb +12 -6
- data/lib/nokogiri/xml/pp/character_data.rb +8 -6
- data/lib/nokogiri/xml/pp/node.rb +24 -26
- data/lib/nokogiri/xml/pp.rb +3 -2
- data/lib/nokogiri/xml/processing_instruction.rb +2 -1
- data/lib/nokogiri/xml/reader.rb +17 -19
- data/lib/nokogiri/xml/relax_ng.rb +1 -0
- data/lib/nokogiri/xml/sax/document.rb +44 -49
- data/lib/nokogiri/xml/sax/parser.rb +36 -34
- data/lib/nokogiri/xml/sax/parser_context.rb +7 -3
- data/lib/nokogiri/xml/sax/push_parser.rb +5 -5
- data/lib/nokogiri/xml/sax.rb +5 -4
- data/lib/nokogiri/xml/schema.rb +7 -6
- data/lib/nokogiri/xml/searchable.rb +93 -62
- data/lib/nokogiri/xml/syntax_error.rb +4 -4
- data/lib/nokogiri/xml/text.rb +1 -0
- data/lib/nokogiri/xml/xpath/syntax_error.rb +2 -1
- data/lib/nokogiri/xml/xpath.rb +13 -1
- data/lib/nokogiri/xml/xpath_context.rb +2 -3
- data/lib/nokogiri/xml.rb +36 -37
- data/lib/nokogiri/xslt/stylesheet.rb +2 -1
- data/lib/nokogiri/xslt.rb +28 -20
- data/lib/nokogiri.rb +48 -43
- data/lib/xsd/xmlparser/nokogiri.rb +25 -24
- data/patches/libxml2/0004-use-glibc-strlen.patch +3 -3
- data/patches/libxml2/0006-update-automake-files-for-arm64.patch +2443 -1914
- data/patches/libxml2/0008-htmlParseComment-handle-abruptly-closed-comments.patch +61 -0
- data/patches/libxml2/0009-allow-wildcard-namespaces.patch +77 -0
- data/patches/libxslt/0001-update-automake-files-for-arm64.patch +2445 -1919
- data/ports/archives/libxml2-2.9.13.tar.xz +0 -0
- data/ports/archives/libxslt-1.1.35.tar.xz +0 -0
- metadata +198 -83
- data/lib/nokogiri/html/element_description_defaults.rb +0 -672
- data/lib/nokogiri/html/sax/parser_context.rb +0 -17
- data/patches/libxslt/0002-Fix-xml2-config-check-in-configure-script.patch +0 -19
- data/ports/archives/libxml2-2.9.12.tar.gz +0 -0
- data/ports/archives/libxslt-1.1.34.tar.gz +0 -0
@@ -0,0 +1,477 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
#
|
5
|
+
# Copyright 2013-2021 Sam Ruby, Stephen Checkoway
|
6
|
+
#
|
7
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
8
|
+
# you may not use this file except in compliance with the License.
|
9
|
+
# You may obtain a copy of the License at
|
10
|
+
#
|
11
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
12
|
+
#
|
13
|
+
# Unless required by applicable law or agreed to in writing, software
|
14
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
15
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
16
|
+
# See the License for the specific language governing permissions and
|
17
|
+
# limitations under the License.
|
18
|
+
#
|
19
|
+
|
20
|
+
require_relative "html5/document"
|
21
|
+
require_relative "html5/document_fragment"
|
22
|
+
require_relative "html5/node"
|
23
|
+
|
24
|
+
module Nokogiri
|
25
|
+
# Since v1.12.0
|
26
|
+
#
|
27
|
+
# ⚠ HTML5 functionality is not available when running JRuby.
|
28
|
+
#
|
29
|
+
# Parse an HTML5 document. Convenience method for {Nokogiri::HTML5::Document.parse}
|
30
|
+
def self.HTML5(input, url = nil, encoding = nil, **options, &block)
|
31
|
+
Nokogiri::HTML5::Document.parse(input, url, encoding, **options, &block)
|
32
|
+
end
|
33
|
+
|
34
|
+
# == Usage
|
35
|
+
#
|
36
|
+
# ⚠ HTML5 functionality is not available when running JRuby.
|
37
|
+
#
|
38
|
+
# Parse an HTML5 document:
|
39
|
+
#
|
40
|
+
# doc = Nokogiri.HTML5(string)
|
41
|
+
#
|
42
|
+
# Parse an HTML5 fragment:
|
43
|
+
#
|
44
|
+
# fragment = Nokogiri::HTML5.fragment(string)
|
45
|
+
#
|
46
|
+
# == Parsing options
|
47
|
+
#
|
48
|
+
# The document and fragment parsing methods support options that are different from Nokogiri's.
|
49
|
+
#
|
50
|
+
# - <tt>Nokogiri.HTML5(html, url = nil, encoding = nil, options = {})</tt>
|
51
|
+
# - <tt>Nokogiri::HTML5.parse(html, url = nil, encoding = nil, options = {})</tt>
|
52
|
+
# - <tt>Nokogiri::HTML5::Document.parse(html, url = nil, encoding = nil, options = {})</tt>
|
53
|
+
# - <tt>Nokogiri::HTML5.fragment(html, encoding = nil, options = {})</tt>
|
54
|
+
# - <tt>Nokogiri::HTML5::DocumentFragment.parse(html, encoding = nil, options = {})</tt>
|
55
|
+
#
|
56
|
+
# The three currently supported options are +:max_errors+, +:max_tree_depth+ and
|
57
|
+
# +:max_attributes+, described below.
|
58
|
+
#
|
59
|
+
# === Error reporting
|
60
|
+
#
|
61
|
+
# Nokogiri contains an experimental HTML5 parse error reporting facility. By default, no parse
|
62
|
+
# errors are reported but this can be configured by passing the +:max_errors+ option to
|
63
|
+
# {HTML5.parse} or {HTML5.fragment}.
|
64
|
+
#
|
65
|
+
# For example, this script:
|
66
|
+
#
|
67
|
+
# doc = Nokogiri::HTML5.parse('<span/>Hi there!</span foo=bar />', max_errors: 10)
|
68
|
+
# doc.errors.each do |err|
|
69
|
+
# puts(err)
|
70
|
+
# end
|
71
|
+
#
|
72
|
+
# Emits:
|
73
|
+
#
|
74
|
+
# 1:1: ERROR: Expected a doctype token
|
75
|
+
# <span/>Hi there!</span foo=bar />
|
76
|
+
# ^
|
77
|
+
# 1:1: ERROR: Start tag of nonvoid HTML element ends with '/>', use '>'.
|
78
|
+
# <span/>Hi there!</span foo=bar />
|
79
|
+
# ^
|
80
|
+
# 1:17: ERROR: End tag ends with '/>', use '>'.
|
81
|
+
# <span/>Hi there!</span foo=bar />
|
82
|
+
# ^
|
83
|
+
# 1:17: ERROR: End tag contains attributes.
|
84
|
+
# <span/>Hi there!</span foo=bar />
|
85
|
+
# ^
|
86
|
+
#
|
87
|
+
# Using <tt>max_errors: -1</tt> results in an unlimited number of errors being returned.
|
88
|
+
#
|
89
|
+
# The errors returned by {HTML5::Document#errors} are instances of {Nokogiri::XML::SyntaxError}.
|
90
|
+
#
|
91
|
+
# The {https://html.spec.whatwg.org/multipage/parsing.html#parse-errors HTML standard} defines a
|
92
|
+
# number of standard parse error codes. These error codes only cover the "tokenization" stage of
|
93
|
+
# parsing HTML. The parse errors in the "tree construction" stage do not have standardized error
|
94
|
+
# codes (yet).
|
95
|
+
#
|
96
|
+
# As a convenience to Nokogiri users, the defined error codes are available via
|
97
|
+
# {Nokogiri::XML::SyntaxError#str1} method.
|
98
|
+
#
|
99
|
+
# doc = Nokogiri::HTML5.parse('<span/>Hi there!</span foo=bar />', max_errors: 10)
|
100
|
+
# doc.errors.each do |err|
|
101
|
+
# puts("#{err.line}:#{err.column}: #{err.str1}")
|
102
|
+
# end
|
103
|
+
# # => 1:1: generic-parser
|
104
|
+
# # 1:1: non-void-html-element-start-tag-with-trailing-solidus
|
105
|
+
# # 1:17: end-tag-with-trailing-solidus
|
106
|
+
# # 1:17: end-tag-with-attributes
|
107
|
+
#
|
108
|
+
# Note that the first error is +generic-parser+ because it's an error from the tree construction
|
109
|
+
# stage and doesn't have a standardized error code.
|
110
|
+
#
|
111
|
+
# For the purposes of semantic versioning, the error messages, error locations, and error codes
|
112
|
+
# are not part of Nokogiri's public API. That is, these are subject to change without Nokogiri's
|
113
|
+
# major version number changing. These may be stabilized in the future.
|
114
|
+
#
|
115
|
+
# === Maximum tree depth
|
116
|
+
#
|
117
|
+
# The maximum depth of the DOM tree parsed by the various parsing methods is configurable by the
|
118
|
+
# +:max_tree_depth+ option. If the depth of the tree would exceed this limit, then an
|
119
|
+
# {::ArgumentError} is thrown.
|
120
|
+
#
|
121
|
+
# This limit (which defaults to <tt>Nokogiri::Gumbo::DEFAULT_MAX_TREE_DEPTH = 400</tt>) can be
|
122
|
+
# removed by giving the option <tt>max_tree_depth: -1</tt>.
|
123
|
+
#
|
124
|
+
# html = '<!DOCTYPE html>' + '<div>' * 1000
|
125
|
+
# doc = Nokogiri.HTML5(html)
|
126
|
+
# # raises ArgumentError: Document tree depth limit exceeded
|
127
|
+
# doc = Nokogiri.HTML5(html, max_tree_depth: -1)
|
128
|
+
#
|
129
|
+
# === Attribute limit per element
|
130
|
+
#
|
131
|
+
# The maximum number of attributes per DOM element is configurable by the +:max_attributes+
|
132
|
+
# option. If a given element would exceed this limit, then an {::ArgumentError} is thrown.
|
133
|
+
#
|
134
|
+
# This limit (which defaults to <tt>Nokogiri::Gumbo::DEFAULT_MAX_ATTRIBUTES = 400</tt>) can be
|
135
|
+
# removed by giving the option <tt>max_attributes: -1</tt>.
|
136
|
+
#
|
137
|
+
# html = '<!DOCTYPE html><div ' + (1..1000).map { |x| "attr-#{x}" }.join(' ') + '>'
|
138
|
+
# # "<!DOCTYPE html><div attr-1 attr-2 attr-3 ... attr-1000>"
|
139
|
+
# doc = Nokogiri.HTML5(html)
|
140
|
+
# # raises ArgumentError: Attributes per element limit exceeded
|
141
|
+
# doc = Nokogiri.HTML5(html, max_attributes: -1)
|
142
|
+
#
|
143
|
+
# == HTML Serialization
|
144
|
+
#
|
145
|
+
# After parsing HTML, it may be serialized using any of the {Nokogiri::XML::Node} serialization
|
146
|
+
# methods. In particular, {XML::Node#serialize}, {XML::Node#to_html}, and {XML::Node#to_s} will
|
147
|
+
# serialize a given node and its children. (This is the equivalent of JavaScript's
|
148
|
+
# +Element.outerHTML+.) Similarly, {XML::Node#inner_html} will serialize the children of a given
|
149
|
+
# node. (This is the equivalent of JavaScript's +Element.innerHTML+.)
|
150
|
+
#
|
151
|
+
# doc = Nokogiri::HTML5("<!DOCTYPE html><span>Hello world!</span>")
|
152
|
+
# puts doc.serialize
|
153
|
+
# # => <!DOCTYPE html><html><head></head><body><span>Hello world!</span></body></html>
|
154
|
+
#
|
155
|
+
# Due to quirks in how HTML is parsed and serialized, it's possible for a DOM tree to be
|
156
|
+
# serialized and then re-parsed, resulting in a different DOM. Mostly, this happens with DOMs
|
157
|
+
# produced from invalid HTML. Unfortunately, even valid HTML may not survive serialization and
|
158
|
+
# re-parsing.
|
159
|
+
#
|
160
|
+
# In particular, a newline at the start of +pre+, +listing+, and +textarea+ elements is ignored by
|
161
|
+
# the parser.
|
162
|
+
#
|
163
|
+
# doc = Nokogiri::HTML5(<<-EOF)
|
164
|
+
# <!DOCTYPE html>
|
165
|
+
# <pre>
|
166
|
+
# Content</pre>
|
167
|
+
# EOF
|
168
|
+
# puts doc.at('/html/body/pre').serialize
|
169
|
+
# # => <pre>Content</pre>
|
170
|
+
#
|
171
|
+
# In this case, the original HTML is semantically equivalent to the serialized version. If the
|
172
|
+
# +pre+, +listing+, or +textarea+ content starts with two newlines, the first newline will be
|
173
|
+
# stripped on the first parse and the second newline will be stripped on the second, leading to
|
174
|
+
# semantically different DOMs. Passing the parameter <tt>preserve_newline: true</tt> will cause
|
175
|
+
# two or more newlines to be preserved. (A single leading newline will still be removed.)
|
176
|
+
#
|
177
|
+
# doc = Nokogiri::HTML5(<<-EOF)
|
178
|
+
# <!DOCTYPE html>
|
179
|
+
# <listing>
|
180
|
+
#
|
181
|
+
# Content</listing>
|
182
|
+
# EOF
|
183
|
+
# puts doc.at('/html/body/listing').serialize(preserve_newline: true)
|
184
|
+
# # => <listing>
|
185
|
+
# #
|
186
|
+
# # Content</listing>
|
187
|
+
#
|
188
|
+
# == Encodings
|
189
|
+
#
|
190
|
+
# Nokogiri always parses HTML5 using {https://en.wikipedia.org/wiki/UTF-8 UTF-8}; however, the
|
191
|
+
# encoding of the input can be explicitly selected via the optional +encoding+ parameter. This is
|
192
|
+
# most useful when the input comes not from a string but from an IO object.
|
193
|
+
#
|
194
|
+
# When serializing a document or node, the encoding of the output string can be specified via the
|
195
|
+
# +:encoding+ options. Characters that cannot be encoded in the selected encoding will be encoded
|
196
|
+
# as {https://en.wikipedia.org/wiki/List_of_XML_and_HTML_character_entity_references HTML numeric
|
197
|
+
# entities}.
|
198
|
+
#
|
199
|
+
# frag = Nokogiri::HTML5.fragment('<span>아는 길도 물어가라</span>')
|
200
|
+
# html = frag.serialize(encoding: 'US-ASCII')
|
201
|
+
# puts html
|
202
|
+
# # => <span>아는 길도 물어가라</span>
|
203
|
+
# frag = Nokogiri::HTML5.fragment(html)
|
204
|
+
# puts frag.serialize
|
205
|
+
# # => <span>아는 길도 물어가라</span>
|
206
|
+
#
|
207
|
+
# (There's a {https://bugs.ruby-lang.org/issues/15033 bug} in all current versions of Ruby that
|
208
|
+
# can cause the entity encoding to fail. Of the mandated supported encodings for HTML, the only
|
209
|
+
# encoding I'm aware of that has this bug is <tt>'ISO-2022-JP'</tt>. We recommend avoiding this
|
210
|
+
# encoding.)
|
211
|
+
#
|
212
|
+
# == Notes
|
213
|
+
#
|
214
|
+
# * The {Nokogiri::HTML5.fragment} function takes a string and parses it
|
215
|
+
# as a HTML5 document. The +<html>+, +<head>+, and +<body>+ elements are
|
216
|
+
# removed from this document, and any children of these elements that remain
|
217
|
+
# are returned as a {Nokogiri::HTML5::DocumentFragment}.
|
218
|
+
#
|
219
|
+
# * The {Nokogiri::HTML5.parse} function takes a string and passes it to the
|
220
|
+
# <code>gumbo_parse_with_options</code> method, using the default options.
|
221
|
+
# The resulting Gumbo parse tree is then walked.
|
222
|
+
#
|
223
|
+
# * Instead of uppercase element names, lowercase element names are produced.
|
224
|
+
#
|
225
|
+
# * Instead of returning +unknown+ as the element name for unknown tags, the
|
226
|
+
# original tag name is returned verbatim.
|
227
|
+
#
|
228
|
+
# Since v1.12.0
|
229
|
+
module HTML5
|
230
|
+
# HTML uses the XHTML namespace.
|
231
|
+
HTML_NAMESPACE = "http://www.w3.org/1999/xhtml"
|
232
|
+
MATHML_NAMESPACE = "http://www.w3.org/1998/Math/MathML"
|
233
|
+
SVG_NAMESPACE = "http://www.w3.org/2000/svg"
|
234
|
+
XLINK_NAMESPACE = "http://www.w3.org/1999/xlink"
|
235
|
+
XML_NAMESPACE = "http://www.w3.org/XML/1998/namespace"
|
236
|
+
XMLNS_NAMESPACE = "http://www.w3.org/2000/xmlns/"
|
237
|
+
|
238
|
+
# Parse an HTML 5 document. Convenience method for {Nokogiri::HTML5::Document.parse}
|
239
|
+
def self.parse(string, url = nil, encoding = nil, **options, &block)
|
240
|
+
Document.parse(string, url, encoding, **options, &block)
|
241
|
+
end
|
242
|
+
|
243
|
+
# Parse a fragment from +string+. Convenience method for
|
244
|
+
# {Nokogiri::HTML5::DocumentFragment.parse}.
|
245
|
+
def self.fragment(string, encoding = nil, **options)
|
246
|
+
DocumentFragment.parse(string, encoding, options)
|
247
|
+
end
|
248
|
+
|
249
|
+
# Fetch and parse a HTML document from the web, following redirects,
|
250
|
+
# handling https, and determining the character encoding using HTML5
|
251
|
+
# rules. +uri+ may be a +String+ or a +URI+. +options+ contains
|
252
|
+
# http headers and special options. Everything which is not a
|
253
|
+
# special option is considered a header. Special options include:
|
254
|
+
# * :follow_limit => number of redirects which are followed
|
255
|
+
# * :basic_auth => [username, password]
|
256
|
+
def self.get(uri, options = {})
|
257
|
+
# TODO: deprecate
|
258
|
+
warn("Nokogiri::HTML5.get is deprecated and will be removed in a future version of Nokogiri.",
|
259
|
+
uplevel: 1, category: :deprecated)
|
260
|
+
get_impl(uri, options)
|
261
|
+
end
|
262
|
+
|
263
|
+
private
|
264
|
+
|
265
|
+
def self.get_impl(uri, options = {})
|
266
|
+
headers = options.clone
|
267
|
+
headers = { follow_limit: headers } if Numeric === headers # deprecated
|
268
|
+
limit = headers[:follow_limit] ? headers.delete(:follow_limit).to_i : 10
|
269
|
+
|
270
|
+
require "net/http"
|
271
|
+
uri = URI(uri) unless URI === uri
|
272
|
+
|
273
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
274
|
+
|
275
|
+
# TLS / SSL support
|
276
|
+
http.use_ssl = true if uri.scheme == "https"
|
277
|
+
|
278
|
+
# Pass through Net::HTTP override values, which currently include:
|
279
|
+
# :ca_file, :ca_path, :cert, :cert_store, :ciphers,
|
280
|
+
# :close_on_empty_response, :continue_timeout, :key, :open_timeout,
|
281
|
+
# :read_timeout, :ssl_timeout, :ssl_version, :use_ssl,
|
282
|
+
# :verify_callback, :verify_depth, :verify_mode
|
283
|
+
options.each do |key, _value|
|
284
|
+
http.send("#{key}=", headers.delete(key)) if http.respond_to?("#{key}=")
|
285
|
+
end
|
286
|
+
|
287
|
+
request = Net::HTTP::Get.new(uri.request_uri)
|
288
|
+
|
289
|
+
# basic authentication
|
290
|
+
auth = headers.delete(:basic_auth)
|
291
|
+
auth ||= [uri.user, uri.password] if uri.user && uri.password
|
292
|
+
request.basic_auth(auth.first, auth.last) if auth
|
293
|
+
|
294
|
+
# remaining options are treated as headers
|
295
|
+
headers.each { |key, value| request[key.to_s] = value.to_s }
|
296
|
+
|
297
|
+
response = http.request(request)
|
298
|
+
|
299
|
+
case response
|
300
|
+
when Net::HTTPSuccess
|
301
|
+
doc = parse(reencode(response.body, response["content-type"]), options)
|
302
|
+
doc.instance_variable_set("@response", response)
|
303
|
+
doc.class.send(:attr_reader, :response)
|
304
|
+
doc
|
305
|
+
when Net::HTTPRedirection
|
306
|
+
response.value if limit <= 1
|
307
|
+
location = URI.join(uri, response["location"])
|
308
|
+
get_impl(location, options.merge(follow_limit: limit - 1))
|
309
|
+
else
|
310
|
+
response.value
|
311
|
+
end
|
312
|
+
end
|
313
|
+
|
314
|
+
def self.read_and_encode(string, encoding)
|
315
|
+
# Read the string with the given encoding.
|
316
|
+
if string.respond_to?(:read)
|
317
|
+
string = if encoding.nil?
|
318
|
+
string.read
|
319
|
+
else
|
320
|
+
string.read(encoding: encoding)
|
321
|
+
end
|
322
|
+
else
|
323
|
+
# Otherwise the string has the given encoding.
|
324
|
+
string = string.to_s
|
325
|
+
if encoding
|
326
|
+
string = string.dup
|
327
|
+
string.force_encoding(encoding)
|
328
|
+
end
|
329
|
+
end
|
330
|
+
|
331
|
+
# convert to UTF-8
|
332
|
+
if string.encoding != Encoding::UTF_8
|
333
|
+
string = reencode(string)
|
334
|
+
end
|
335
|
+
string
|
336
|
+
end
|
337
|
+
|
338
|
+
# Charset sniffing is a complex and controversial topic that understandably isn't done _by
|
339
|
+
# default_ by the Ruby Net::HTTP library. This being said, it is a very real problem for
|
340
|
+
# consumers of HTML as the default for HTML is iso-8859-1, most "good" producers use utf-8, and
|
341
|
+
# the Gumbo parser *only* supports utf-8.
|
342
|
+
#
|
343
|
+
# Accordingly, Nokogiri::HTML4::Document.parse provides limited encoding detection. Following
|
344
|
+
# this lead, Nokogiri::HTML5 attempts to do likewise, while attempting to more closely follow
|
345
|
+
# the HTML5 standard.
|
346
|
+
#
|
347
|
+
# http://bugs.ruby-lang.org/issues/2567
|
348
|
+
# http://www.w3.org/TR/html5/syntax.html#determining-the-character-encoding
|
349
|
+
#
|
350
|
+
def self.reencode(body, content_type = nil)
|
351
|
+
if body.encoding == Encoding::ASCII_8BIT
|
352
|
+
encoding = nil
|
353
|
+
|
354
|
+
# look for a Byte Order Mark (BOM)
|
355
|
+
initial_bytes = body[0..2].bytes
|
356
|
+
if initial_bytes[0..2] == [0xEF, 0xBB, 0xBF]
|
357
|
+
encoding = Encoding::UTF_8
|
358
|
+
elsif initial_bytes[0..1] == [0xFE, 0xFF]
|
359
|
+
encoding = Encoding::UTF_16BE
|
360
|
+
elsif initial_bytes[0..1] == [0xFF, 0xFE]
|
361
|
+
encoding = Encoding::UTF_16LE
|
362
|
+
end
|
363
|
+
|
364
|
+
# look for a charset in a content-encoding header
|
365
|
+
if content_type
|
366
|
+
encoding ||= content_type[/charset=["']?(.*?)($|["';\s])/i, 1]
|
367
|
+
end
|
368
|
+
|
369
|
+
# look for a charset in a meta tag in the first 1024 bytes
|
370
|
+
unless encoding
|
371
|
+
data = body[0..1023].gsub(/<!--.*?(-->|\Z)/m, "")
|
372
|
+
data.scan(/<meta.*?>/m).each do |meta|
|
373
|
+
encoding ||= meta[/charset=["']?([^>]*?)($|["'\s>])/im, 1]
|
374
|
+
end
|
375
|
+
end
|
376
|
+
|
377
|
+
# if all else fails, default to the official default encoding for HTML
|
378
|
+
encoding ||= Encoding::ISO_8859_1
|
379
|
+
|
380
|
+
# change the encoding to match the detected or inferred encoding
|
381
|
+
body = body.dup
|
382
|
+
begin
|
383
|
+
body.force_encoding(encoding)
|
384
|
+
rescue ArgumentError
|
385
|
+
body.force_encoding(Encoding::ISO_8859_1)
|
386
|
+
end
|
387
|
+
end
|
388
|
+
|
389
|
+
body.encode(Encoding::UTF_8)
|
390
|
+
end
|
391
|
+
|
392
|
+
def self.serialize_node_internal(current_node, io, encoding, options)
|
393
|
+
case current_node.type
|
394
|
+
when XML::Node::ELEMENT_NODE
|
395
|
+
ns = current_node.namespace
|
396
|
+
ns_uri = ns.nil? ? nil : ns.href
|
397
|
+
# XXX(sfc): attach namespaces to all nodes, even html?
|
398
|
+
tagname = if ns_uri.nil? || ns_uri == HTML_NAMESPACE || ns_uri == MATHML_NAMESPACE || ns_uri == SVG_NAMESPACE
|
399
|
+
current_node.name
|
400
|
+
else
|
401
|
+
"#{ns.prefix}:#{current_node.name}"
|
402
|
+
end
|
403
|
+
io << "<" << tagname
|
404
|
+
current_node.attribute_nodes.each do |attr|
|
405
|
+
attr_ns = attr.namespace
|
406
|
+
if attr_ns.nil?
|
407
|
+
attr_name = attr.name
|
408
|
+
else
|
409
|
+
ns_uri = attr_ns.href
|
410
|
+
attr_name = if ns_uri == XML_NAMESPACE
|
411
|
+
"xml:" + attr.name.sub(/^[^:]*:/, "")
|
412
|
+
elsif ns_uri == XMLNS_NAMESPACE && attr.name.sub(/^[^:]*:/, "") == "xmlns"
|
413
|
+
"xmlns"
|
414
|
+
elsif ns_uri == XMLNS_NAMESPACE
|
415
|
+
"xmlns:" + attr.name.sub(/^[^:]*:/, "")
|
416
|
+
elsif ns_uri == XLINK_NAMESPACE
|
417
|
+
"xlink:" + attr.name.sub(/^[^:]*:/, "")
|
418
|
+
else
|
419
|
+
"#{attr_ns.prefix}:#{attr.name}"
|
420
|
+
end
|
421
|
+
end
|
422
|
+
io << " " << attr_name << '="' << escape_text(attr.content, encoding, true) << '"'
|
423
|
+
end
|
424
|
+
io << ">"
|
425
|
+
unless ["area", "base", "basefont", "bgsound", "br", "col", "embed", "frame", "hr", "img", "input", "keygen", "link", "meta", "param", "source", "track", "wbr"].include?(current_node.name)
|
426
|
+
io << "\n" if options[:preserve_newline] && prepend_newline?(current_node)
|
427
|
+
current_node.children.each do |child|
|
428
|
+
# XXX(sfc): Templates handled specially?
|
429
|
+
serialize_node_internal(child, io, encoding, options)
|
430
|
+
end
|
431
|
+
io << "</" << tagname << ">"
|
432
|
+
end
|
433
|
+
when XML::Node::TEXT_NODE
|
434
|
+
parent = current_node.parent
|
435
|
+
io << if parent.element? && ["style", "script", "xmp", "iframe", "noembed", "noframes", "plaintext", "noscript"].include?(parent.name)
|
436
|
+
current_node.content
|
437
|
+
else
|
438
|
+
escape_text(current_node.content, encoding, false)
|
439
|
+
end
|
440
|
+
when XML::Node::CDATA_SECTION_NODE
|
441
|
+
io << "<![CDATA[" << current_node.content << "]]>"
|
442
|
+
when XML::Node::COMMENT_NODE
|
443
|
+
io << "<!--" << current_node.content << "-->"
|
444
|
+
when XML::Node::PI_NODE
|
445
|
+
io << "<?" << current_node.content << ">"
|
446
|
+
when XML::Node::DOCUMENT_TYPE_NODE, XML::Node::DTD_NODE
|
447
|
+
io << "<!DOCTYPE " << current_node.name << ">"
|
448
|
+
when XML::Node::HTML_DOCUMENT_NODE, XML::Node::DOCUMENT_FRAG_NODE
|
449
|
+
current_node.children.each do |child|
|
450
|
+
serialize_node_internal(child, io, encoding, options)
|
451
|
+
end
|
452
|
+
else
|
453
|
+
raise "Unexpected node '#{current_node.name}' of type #{current_node.type}"
|
454
|
+
end
|
455
|
+
end
|
456
|
+
|
457
|
+
def self.escape_text(text, encoding, attribute_mode)
|
458
|
+
text = if attribute_mode
|
459
|
+
text.gsub(/[&\u00a0"]/,
|
460
|
+
"&" => "&", "\u00a0" => " ", '"' => """)
|
461
|
+
else
|
462
|
+
text.gsub(/[&\u00a0<>]/,
|
463
|
+
"&" => "&", "\u00a0" => " ", "<" => "<", ">" => ">")
|
464
|
+
end
|
465
|
+
# Not part of the standard
|
466
|
+
text.encode(encoding, fallback: lambda { |c| "&\#x#{c.ord.to_s(16)};" })
|
467
|
+
end
|
468
|
+
|
469
|
+
def self.prepend_newline?(node)
|
470
|
+
return false unless ["pre", "textarea", "listing"].include?(node.name) && !node.children.empty?
|
471
|
+
first_child = node.children[0]
|
472
|
+
first_child.text? && first_child.content.start_with?("\n")
|
473
|
+
end
|
474
|
+
end
|
475
|
+
end
|
476
|
+
|
477
|
+
require_relative "gumbo"
|
@@ -1,4 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
# The line below caused a problem on non-GAE rack environment.
|
3
4
|
# unless defined?(JRuby::Rack::VERSION) || defined?(AppEngine::ApiProxy)
|
4
5
|
#
|
@@ -8,13 +9,13 @@
|
|
8
9
|
# should skip loading xml jars. This is because those are in WEB-INF/lib and
|
9
10
|
# already set in the classpath.
|
10
11
|
unless $LOAD_PATH.to_s.include?("appengine-rack")
|
11
|
-
require
|
12
|
-
require
|
13
|
-
require
|
14
|
-
require
|
15
|
-
require
|
16
|
-
require
|
17
|
-
require
|
18
|
-
require
|
19
|
-
require
|
12
|
+
require "stringio"
|
13
|
+
require "isorelax.jar"
|
14
|
+
require "jing.jar"
|
15
|
+
require "nekohtml.jar"
|
16
|
+
require "nekodtd.jar"
|
17
|
+
require "xercesImpl.jar"
|
18
|
+
require "serializer.jar"
|
19
|
+
require "xalan.jar"
|
20
|
+
require "xml-apis.jar"
|
20
21
|
end
|
@@ -1,4 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
require "singleton"
|
3
4
|
require "shellwords"
|
4
5
|
|
@@ -52,6 +53,10 @@ module Nokogiri
|
|
52
53
|
defined?(Nokogiri::LIBXML_ICONV_ENABLED) && Nokogiri::LIBXML_ICONV_ENABLED
|
53
54
|
end
|
54
55
|
|
56
|
+
def libxslt_has_datetime?
|
57
|
+
defined?(Nokogiri::LIBXSLT_DATETIME_ENABLED) && Nokogiri::LIBXSLT_DATETIME_ENABLED
|
58
|
+
end
|
59
|
+
|
55
60
|
def libxml2_using_packaged?
|
56
61
|
libxml2? && Nokogiri::PACKAGED_LIBRARIES
|
57
62
|
end
|
@@ -98,8 +103,8 @@ module Nokogiri
|
|
98
103
|
ldflags = []
|
99
104
|
|
100
105
|
if libxml2_using_packaged?
|
101
|
-
cppflags << "-I#{File.join(header_directory,
|
102
|
-
cppflags << "-I#{File.join(header_directory,
|
106
|
+
cppflags << "-I#{File.join(header_directory, "include").shellescape}"
|
107
|
+
cppflags << "-I#{File.join(header_directory, "include/libxml2").shellescape}"
|
103
108
|
|
104
109
|
if windows?
|
105
110
|
# on windows, nokogumbo needs to link against nokogiri.so to resolve symbols. see #2167
|
@@ -137,6 +142,7 @@ module Nokogiri
|
|
137
142
|
else
|
138
143
|
libxml["source"] = "system"
|
139
144
|
end
|
145
|
+
libxml["memory_management"] = Nokogiri::LIBXML_MEMORY_MANAGEMENT
|
140
146
|
libxml["iconv_enabled"] = libxml2_has_iconv?
|
141
147
|
libxml["compiled"] = compiled_libxml_version.to_s
|
142
148
|
libxml["loaded"] = loaded_libxml_version.to_s
|
@@ -150,6 +156,7 @@ module Nokogiri
|
|
150
156
|
else
|
151
157
|
libxslt["source"] = "system"
|
152
158
|
end
|
159
|
+
libxslt["datetime_enabled"] = libxslt_has_datetime?
|
153
160
|
libxslt["compiled"] = compiled_libxslt_version.to_s
|
154
161
|
libxslt["loaded"] = loaded_libxslt_version.to_s
|
155
162
|
end
|
@@ -170,13 +177,9 @@ module Nokogiri
|
|
170
177
|
end
|
171
178
|
|
172
179
|
def to_markdown
|
173
|
-
begin
|
174
|
-
require "psych"
|
175
|
-
rescue LoadError
|
176
|
-
end
|
177
180
|
require "yaml"
|
178
181
|
"# Nokogiri (#{Nokogiri::VERSION})\n" +
|
179
|
-
|
182
|
+
YAML.dump(to_hash).each_line.map { |line| " #{line}" }.join
|
180
183
|
end
|
181
184
|
|
182
185
|
instance.warnings.each do |warning|
|
@@ -184,22 +187,35 @@ module Nokogiri
|
|
184
187
|
end
|
185
188
|
end
|
186
189
|
|
187
|
-
|
190
|
+
# :nodoc:
|
191
|
+
def self.uses_libxml?(requirement = nil)
|
188
192
|
return false unless VersionInfo.instance.libxml2?
|
189
193
|
return true unless requirement
|
190
194
|
Gem::Requirement.new(requirement).satisfied_by?(VersionInfo.instance.loaded_libxml_version)
|
191
195
|
end
|
192
196
|
|
193
|
-
|
197
|
+
# :nodoc:
|
198
|
+
def self.uses_gumbo?
|
199
|
+
uses_libxml? # TODO: replace with Gumbo functionality
|
200
|
+
end
|
201
|
+
|
202
|
+
# :nodoc:
|
203
|
+
def self.jruby?
|
194
204
|
VersionInfo.instance.jruby?
|
195
205
|
end
|
196
206
|
|
197
|
-
#
|
198
|
-
|
199
|
-
|
207
|
+
# :nodoc:
|
208
|
+
def self.libxml2_patches
|
209
|
+
if VersionInfo.instance.libxml2_using_packaged?
|
210
|
+
Nokogiri::VERSION_INFO["libxml"]["patches"]
|
211
|
+
else
|
212
|
+
[]
|
213
|
+
end
|
200
214
|
end
|
201
|
-
require "nokogiri/extension"
|
202
215
|
|
203
|
-
|
216
|
+
require_relative "../jruby/dependencies" if Nokogiri.jruby?
|
217
|
+
require_relative "../extension"
|
218
|
+
|
219
|
+
# Detailed version info about Nokogiri and the installed extension dependencies.
|
204
220
|
VERSION_INFO = VersionInfo.instance.to_hash
|
205
221
|
end
|
data/lib/nokogiri/version.rb
CHANGED
data/lib/nokogiri/xml/attr.rb
CHANGED
@@ -1,12 +1,14 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
module Nokogiri
|
3
4
|
module XML
|
4
5
|
class Attr < Node
|
5
|
-
|
6
|
-
|
7
|
-
|
6
|
+
alias_method :value, :content
|
7
|
+
alias_method :to_s, :content
|
8
|
+
alias_method :content=, :value=
|
8
9
|
|
9
10
|
private
|
11
|
+
|
10
12
|
def inspect_attributes
|
11
13
|
[:name, :namespace, :value]
|
12
14
|
end
|
@@ -1,4 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
module Nokogiri
|
3
4
|
module XML
|
4
5
|
###
|
@@ -12,7 +13,7 @@ module Nokogiri
|
|
12
13
|
undef_method :line if method_defined?(:line)
|
13
14
|
|
14
15
|
def inspect
|
15
|
-
"#<#{self.class.name}:#{
|
16
|
+
"#<#{self.class.name}:#{format("0x%x", object_id)} #{to_s.inspect}>"
|
16
17
|
end
|
17
18
|
end
|
18
19
|
end
|