nokogiri 1.13.6 → 1.14.2
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of nokogiri might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Gemfile +39 -0
- data/LICENSE-DEPENDENCIES.md +830 -509
- data/LICENSE.md +1 -1
- data/README.md +18 -11
- data/dependencies.yml +33 -15
- data/ext/nokogiri/extconf.rb +100 -24
- data/ext/nokogiri/gumbo.c +21 -11
- data/ext/nokogiri/html4_document.c +2 -2
- data/ext/nokogiri/html4_element_description.c +1 -1
- data/ext/nokogiri/html4_entity_lookup.c +2 -2
- data/ext/nokogiri/html4_sax_parser_context.c +1 -6
- data/ext/nokogiri/html4_sax_push_parser.c +1 -1
- data/ext/nokogiri/nokogiri.c +38 -51
- data/ext/nokogiri/nokogiri.h +26 -14
- data/ext/nokogiri/test_global_handlers.c +1 -1
- data/ext/nokogiri/xml_attr.c +3 -3
- data/ext/nokogiri/xml_attribute_decl.c +5 -5
- data/ext/nokogiri/xml_cdata.c +3 -3
- data/ext/nokogiri/xml_comment.c +1 -1
- data/ext/nokogiri/xml_document.c +23 -14
- data/ext/nokogiri/xml_document_fragment.c +1 -1
- data/ext/nokogiri/xml_dtd.c +9 -9
- data/ext/nokogiri/xml_element_content.c +3 -3
- data/ext/nokogiri/xml_element_decl.c +5 -5
- data/ext/nokogiri/xml_encoding_handler.c +3 -3
- data/ext/nokogiri/xml_entity_decl.c +6 -6
- data/ext/nokogiri/xml_entity_reference.c +1 -1
- data/ext/nokogiri/xml_namespace.c +80 -14
- data/ext/nokogiri/xml_node.c +363 -82
- data/ext/nokogiri/xml_node_set.c +4 -6
- data/ext/nokogiri/xml_processing_instruction.c +1 -1
- data/ext/nokogiri/xml_reader.c +97 -22
- data/ext/nokogiri/xml_relax_ng.c +1 -3
- data/ext/nokogiri/xml_sax_parser.c +23 -17
- data/ext/nokogiri/xml_sax_parser_context.c +1 -6
- data/ext/nokogiri/xml_sax_push_parser.c +1 -3
- data/ext/nokogiri/xml_schema.c +4 -6
- data/ext/nokogiri/xml_syntax_error.c +1 -1
- data/ext/nokogiri/xml_text.c +2 -2
- data/ext/nokogiri/xml_xpath_context.c +91 -84
- data/ext/nokogiri/xslt_stylesheet.c +15 -14
- data/gumbo-parser/Makefile +10 -0
- data/gumbo-parser/src/attribute.h +1 -1
- data/gumbo-parser/src/error.c +2 -2
- data/gumbo-parser/src/error.h +1 -1
- data/gumbo-parser/src/foreign_attrs.c +2 -2
- data/gumbo-parser/src/{gumbo.h → nokogiri_gumbo.h} +1 -0
- data/gumbo-parser/src/parser.c +8 -5
- data/gumbo-parser/src/replacement.h +1 -1
- data/gumbo-parser/src/string_buffer.h +1 -1
- data/gumbo-parser/src/string_piece.c +1 -1
- data/gumbo-parser/src/svg_attrs.c +2 -2
- data/gumbo-parser/src/svg_tags.c +2 -2
- data/gumbo-parser/src/tag.c +2 -1
- data/gumbo-parser/src/tag_lookup.c +7 -7
- data/gumbo-parser/src/tag_lookup.gperf +1 -0
- data/gumbo-parser/src/tag_lookup.h +1 -1
- data/gumbo-parser/src/token_buffer.h +1 -1
- data/gumbo-parser/src/tokenizer.c +1 -1
- data/gumbo-parser/src/tokenizer.h +1 -1
- data/gumbo-parser/src/utf8.c +1 -1
- data/gumbo-parser/src/utf8.h +1 -1
- data/gumbo-parser/src/util.c +1 -3
- data/gumbo-parser/src/util.h +4 -0
- data/gumbo-parser/src/vector.h +1 -1
- data/lib/nokogiri/css/node.rb +2 -2
- data/lib/nokogiri/css/xpath_visitor.rb +5 -3
- data/lib/nokogiri/css.rb +6 -0
- data/lib/nokogiri/decorators/slop.rb +1 -1
- data/lib/nokogiri/encoding_handler.rb +57 -0
- data/lib/nokogiri/extension.rb +3 -2
- data/lib/nokogiri/html4/document.rb +2 -121
- data/lib/nokogiri/html4/element_description_defaults.rb +6 -12
- data/lib/nokogiri/html4/encoding_reader.rb +121 -0
- data/lib/nokogiri/html4.rb +1 -0
- data/lib/nokogiri/html5/document.rb +113 -36
- data/lib/nokogiri/html5/document_fragment.rb +9 -2
- data/lib/nokogiri/html5/node.rb +3 -5
- data/lib/nokogiri/html5.rb +127 -216
- data/lib/nokogiri/jruby/dependencies.rb +1 -19
- data/lib/nokogiri/jruby/nokogiri_jars.rb +43 -0
- data/lib/nokogiri/version/constant.rb +1 -1
- data/lib/nokogiri/version/info.rb +11 -10
- data/lib/nokogiri/xml/attr.rb +49 -0
- data/lib/nokogiri/xml/builder.rb +1 -1
- data/lib/nokogiri/xml/document.rb +103 -55
- data/lib/nokogiri/xml/document_fragment.rb +49 -6
- data/lib/nokogiri/xml/namespace.rb +42 -0
- data/lib/nokogiri/xml/node/save_options.rb +6 -4
- data/lib/nokogiri/xml/node.rb +190 -35
- data/lib/nokogiri/xml/node_set.rb +88 -9
- data/lib/nokogiri/xml/parse_options.rb +129 -50
- data/lib/nokogiri/xml/pp/node.rb +6 -4
- data/lib/nokogiri/xml/processing_instruction.rb +2 -1
- data/lib/nokogiri/xml/reader.rb +6 -8
- data/lib/nokogiri/xml/sax/parser.rb +2 -3
- data/lib/nokogiri/xslt.rb +1 -1
- data/lib/nokogiri.rb +3 -11
- data/lib/xsd/xmlparser/nokogiri.rb +3 -1
- data/ports/archives/libxml2-2.10.3.tar.xz +0 -0
- data/ports/archives/libxslt-1.1.37.tar.xz +0 -0
- metadata +11 -242
- data/patches/libxml2/0004-use-glibc-strlen.patch +0 -53
- data/patches/libxml2/0005-avoid-isnan-isinf.patch +0 -81
- data/patches/libxml2/0006-update-automake-files-for-arm64.patch +0 -3040
- data/patches/libxml2/0008-htmlParseComment-handle-abruptly-closed-comments.patch +0 -61
- data/ports/archives/libxml2-2.9.14.tar.xz +0 -0
- data/ports/archives/libxslt-1.1.35.tar.xz +0 -0
data/gumbo-parser/src/utf8.c
CHANGED
data/gumbo-parser/src/utf8.h
CHANGED
data/gumbo-parser/src/util.c
CHANGED
@@ -19,7 +19,7 @@
|
|
19
19
|
#include <stdlib.h>
|
20
20
|
#include <string.h>
|
21
21
|
#include "util.h"
|
22
|
-
#include "
|
22
|
+
#include "nokogiri_gumbo.h"
|
23
23
|
|
24
24
|
void* gumbo_alloc(size_t size) {
|
25
25
|
void* ptr = malloc(size);
|
@@ -63,6 +63,4 @@ void gumbo_debug(const char* format, ...) {
|
|
63
63
|
va_end(args);
|
64
64
|
fflush(stdout);
|
65
65
|
}
|
66
|
-
#else
|
67
|
-
void gumbo_debug(const char* UNUSED_ARG(format), ...) {}
|
68
66
|
#endif
|
data/gumbo-parser/src/util.h
CHANGED
@@ -21,7 +21,11 @@ void* gumbo_realloc(void* ptr, size_t size) RETURNS_NONNULL;
|
|
21
21
|
void gumbo_free(void* ptr);
|
22
22
|
|
23
23
|
// Debug wrapper for printf
|
24
|
+
#ifdef GUMBO_DEBUG
|
24
25
|
void gumbo_debug(const char* format, ...) PRINTF(1);
|
26
|
+
#else
|
27
|
+
static inline void PRINTF(1) gumbo_debug(const char* UNUSED_ARG(format), ...) {};
|
28
|
+
#endif
|
25
29
|
|
26
30
|
#ifdef __cplusplus
|
27
31
|
}
|
data/gumbo-parser/src/vector.h
CHANGED
data/lib/nokogiri/css/node.rb
CHANGED
@@ -230,11 +230,11 @@ module Nokogiri
|
|
230
230
|
"descendant_selector" => "//",
|
231
231
|
"child_selector" => "/",
|
232
232
|
}.each do |k, v|
|
233
|
-
class_eval
|
233
|
+
class_eval <<~RUBY, __FILE__, __LINE__ + 1
|
234
234
|
def visit_#{k} node
|
235
235
|
"\#{node.value.first.accept(self) if node.value.first}#{v}\#{node.value.last.accept(self)}"
|
236
236
|
end
|
237
|
-
|
237
|
+
RUBY
|
238
238
|
end
|
239
239
|
|
240
240
|
def visit_conditional_selector(node)
|
@@ -278,7 +278,9 @@ module Nokogiri
|
|
278
278
|
end
|
279
279
|
|
280
280
|
def nth(node, options = {})
|
281
|
-
|
281
|
+
unless node.value.size == 4
|
282
|
+
raise(ArgumentError, "expected an+b node to contain 4 tokens, but is #{node.value.inspect}")
|
283
|
+
end
|
282
284
|
|
283
285
|
a, b = read_a_and_positive_b(node.value)
|
284
286
|
position = if options[:child]
|
data/lib/nokogiri/css.rb
CHANGED
@@ -40,9 +40,15 @@ module Nokogiri
|
|
40
40
|
# 💡 Note that translated queries are cached for performance concerns.
|
41
41
|
#
|
42
42
|
def xpath_for(selector, options = {})
|
43
|
+
raise TypeError, "no implicit conversion of #{selector.inspect} to String" unless selector.respond_to?(:to_str)
|
44
|
+
|
45
|
+
selector = selector.to_str
|
46
|
+
raise Nokogiri::CSS::SyntaxError, "empty CSS selector" if selector.empty?
|
47
|
+
|
43
48
|
prefix = options.fetch(:prefix, Nokogiri::XML::XPath::GLOBAL_SEARCH_PREFIX)
|
44
49
|
visitor = options.fetch(:visitor) { Nokogiri::CSS::XPathVisitor.new }
|
45
50
|
ns = options.fetch(:ns, {})
|
51
|
+
|
46
52
|
Parser.new(ns).xpath_for(selector, prefix, visitor)
|
47
53
|
end
|
48
54
|
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
module Nokogiri
|
5
|
+
class EncodingHandler
|
6
|
+
# Popular encoding aliases not known by all iconv implementations that Nokogiri should support.
|
7
|
+
USEFUL_ALIASES = {
|
8
|
+
# alias_name => true_name
|
9
|
+
"NOKOGIRI-SENTINEL" => "UTF-8", # indicating the Nokogiri has installed aliases
|
10
|
+
"Windows-31J" => "CP932", # Windows-31J is the IANA registered name of CP932.
|
11
|
+
"UTF-8" => "UTF-8", # for JRuby tests, this is a no-op in CRuby
|
12
|
+
}
|
13
|
+
|
14
|
+
class << self
|
15
|
+
def install_default_aliases
|
16
|
+
USEFUL_ALIASES.each do |alias_name, name|
|
17
|
+
EncodingHandler.alias(name, alias_name) if EncodingHandler[alias_name].nil?
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
# :stopdoc:
|
23
|
+
if Nokogiri.jruby?
|
24
|
+
class << self
|
25
|
+
def [](name)
|
26
|
+
storage.key?(name) ? new(storage[name]) : nil
|
27
|
+
end
|
28
|
+
|
29
|
+
def alias(name, alias_name)
|
30
|
+
storage[alias_name] = name
|
31
|
+
end
|
32
|
+
|
33
|
+
def delete(name)
|
34
|
+
storage.delete(name)
|
35
|
+
end
|
36
|
+
|
37
|
+
def clear_aliases!
|
38
|
+
storage.clear
|
39
|
+
end
|
40
|
+
|
41
|
+
private
|
42
|
+
|
43
|
+
def storage
|
44
|
+
@storage ||= {}
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def initialize(name)
|
49
|
+
@name = name
|
50
|
+
end
|
51
|
+
|
52
|
+
attr_reader :name
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
Nokogiri::EncodingHandler.install_default_aliases
|
data/lib/nokogiri/extension.rb
CHANGED
@@ -3,13 +3,14 @@
|
|
3
3
|
# load the C or Java extension
|
4
4
|
begin
|
5
5
|
# native precompiled gems package shared libraries in <gem_dir>/lib/nokogiri/<ruby_version>
|
6
|
-
|
6
|
+
RUBY_VERSION =~ /(\d+\.\d+)/
|
7
7
|
require_relative "#{Regexp.last_match(1)}/nokogiri"
|
8
8
|
rescue LoadError => e
|
9
9
|
if /GLIBC/.match?(e.message)
|
10
10
|
warn(<<~EOM)
|
11
11
|
|
12
|
-
ERROR: It looks like you're trying to use Nokogiri as a precompiled native gem on a system
|
12
|
+
ERROR: It looks like you're trying to use Nokogiri as a precompiled native gem on a system
|
13
|
+
with an unsupported version of glibc.
|
13
14
|
|
14
15
|
#{e.message}
|
15
16
|
|
@@ -176,7 +176,7 @@ module Nokogiri
|
|
176
176
|
url ||= string_or_io.respond_to?(:path) ? string_or_io.path : nil
|
177
177
|
|
178
178
|
if string_or_io.respond_to?(:encoding)
|
179
|
-
unless string_or_io.encoding
|
179
|
+
unless string_or_io.encoding == Encoding::ASCII_8BIT
|
180
180
|
encoding ||= string_or_io.encoding.name
|
181
181
|
end
|
182
182
|
end
|
@@ -189,21 +189,10 @@ module Nokogiri
|
|
189
189
|
end
|
190
190
|
|
191
191
|
unless encoding
|
192
|
-
# Libxml2's parser has poor support for encoding
|
193
|
-
# detection. First, it does not recognize the HTML5
|
194
|
-
# style meta charset declaration. Secondly, even if it
|
195
|
-
# successfully detects an encoding hint, it does not
|
196
|
-
# re-decode or re-parse the preceding part which may be
|
197
|
-
# garbled.
|
198
|
-
#
|
199
|
-
# EncodingReader aims to perform advanced encoding
|
200
|
-
# detection beyond what Libxml2 does, and to emulate
|
201
|
-
# rewinding of a stream and make Libxml2 redo parsing
|
202
|
-
# from the start when an encoding hint is found.
|
203
192
|
string_or_io = EncodingReader.new(string_or_io)
|
204
193
|
begin
|
205
194
|
return read_io(string_or_io, url, encoding, options.to_i)
|
206
|
-
rescue EncodingFound => e
|
195
|
+
rescue EncodingReader::EncodingFound => e
|
207
196
|
encoding = e.found_encoding
|
208
197
|
end
|
209
198
|
end
|
@@ -220,114 +209,6 @@ module Nokogiri
|
|
220
209
|
read_memory(string_or_io, url, encoding, options.to_i)
|
221
210
|
end
|
222
211
|
end
|
223
|
-
|
224
|
-
class EncodingFound < StandardError # :nodoc: all
|
225
|
-
attr_reader :found_encoding
|
226
|
-
|
227
|
-
def initialize(encoding)
|
228
|
-
@found_encoding = encoding
|
229
|
-
super(format("encoding found: %s", encoding))
|
230
|
-
end
|
231
|
-
end
|
232
|
-
|
233
|
-
# :nodoc: all
|
234
|
-
class EncodingReader
|
235
|
-
class SAXHandler < Nokogiri::XML::SAX::Document
|
236
|
-
attr_reader :encoding
|
237
|
-
|
238
|
-
def initialize
|
239
|
-
@encoding = nil
|
240
|
-
super()
|
241
|
-
end
|
242
|
-
|
243
|
-
def start_element(name, attrs = [])
|
244
|
-
return unless name == "meta"
|
245
|
-
|
246
|
-
attr = Hash[attrs]
|
247
|
-
(charset = attr["charset"]) &&
|
248
|
-
(@encoding = charset)
|
249
|
-
(http_equiv = attr["http-equiv"]) &&
|
250
|
-
http_equiv.match(/\AContent-Type\z/i) &&
|
251
|
-
(content = attr["content"]) &&
|
252
|
-
(m = content.match(/;\s*charset\s*=\s*([\w-]+)/)) &&
|
253
|
-
(@encoding = m[1])
|
254
|
-
end
|
255
|
-
end
|
256
|
-
|
257
|
-
class JumpSAXHandler < SAXHandler
|
258
|
-
def initialize(jumptag)
|
259
|
-
@jumptag = jumptag
|
260
|
-
super()
|
261
|
-
end
|
262
|
-
|
263
|
-
def start_element(name, attrs = [])
|
264
|
-
super
|
265
|
-
throw(@jumptag, @encoding) if @encoding
|
266
|
-
throw(@jumptag, nil) if /\A(?:div|h1|img|p|br)\z/.match?(name)
|
267
|
-
end
|
268
|
-
end
|
269
|
-
|
270
|
-
def self.detect_encoding(chunk)
|
271
|
-
(m = chunk.match(/\A(<\?xml[ \t\r\n][^>]*>)/)) &&
|
272
|
-
(return Nokogiri.XML(m[1]).encoding)
|
273
|
-
|
274
|
-
if Nokogiri.jruby?
|
275
|
-
(m = chunk.match(/(<meta\s)(.*)(charset\s*=\s*([\w-]+))(.*)/i)) &&
|
276
|
-
(return m[4])
|
277
|
-
catch(:encoding_found) do
|
278
|
-
Nokogiri::HTML4::SAX::Parser.new(JumpSAXHandler.new(:encoding_found)).parse(chunk)
|
279
|
-
nil
|
280
|
-
end
|
281
|
-
else
|
282
|
-
handler = SAXHandler.new
|
283
|
-
parser = Nokogiri::HTML4::SAX::PushParser.new(handler)
|
284
|
-
begin
|
285
|
-
parser << chunk
|
286
|
-
rescue
|
287
|
-
Nokogiri::SyntaxError
|
288
|
-
end
|
289
|
-
handler.encoding
|
290
|
-
end
|
291
|
-
end
|
292
|
-
|
293
|
-
def initialize(io)
|
294
|
-
@io = io
|
295
|
-
@firstchunk = nil
|
296
|
-
@encoding_found = nil
|
297
|
-
end
|
298
|
-
|
299
|
-
# This method is used by the C extension so that
|
300
|
-
# Nokogiri::HTML4::Document#read_io() does not leak memory when
|
301
|
-
# EncodingFound is raised.
|
302
|
-
attr_reader :encoding_found
|
303
|
-
|
304
|
-
def read(len)
|
305
|
-
# no support for a call without len
|
306
|
-
|
307
|
-
unless @firstchunk
|
308
|
-
(@firstchunk = @io.read(len)) || (return nil)
|
309
|
-
|
310
|
-
# This implementation expects that the first call from
|
311
|
-
# htmlReadIO() is made with a length long enough (~1KB) to
|
312
|
-
# achieve advanced encoding detection.
|
313
|
-
if (encoding = EncodingReader.detect_encoding(@firstchunk))
|
314
|
-
# The first chunk is stored for the next read in retry.
|
315
|
-
raise @encoding_found = EncodingFound.new(encoding)
|
316
|
-
end
|
317
|
-
end
|
318
|
-
@encoding_found = nil
|
319
|
-
|
320
|
-
ret = @firstchunk.slice!(0, len)
|
321
|
-
if (len -= ret.length) > 0
|
322
|
-
(rest = @io.read(len)) && ret << (rest)
|
323
|
-
end
|
324
|
-
if ret.empty?
|
325
|
-
nil
|
326
|
-
else
|
327
|
-
ret
|
328
|
-
end
|
329
|
-
end
|
330
|
-
end
|
331
212
|
end
|
332
213
|
end
|
333
214
|
end
|
@@ -25,43 +25,37 @@ module Nokogiri
|
|
25
25
|
|
26
26
|
unless method_defined?(:implied_start_tag?)
|
27
27
|
def implied_start_tag?
|
28
|
-
|
29
|
-
d ? d.startTag : nil
|
28
|
+
default_desc&.startTag
|
30
29
|
end
|
31
30
|
end
|
32
31
|
|
33
32
|
unless method_defined?(:implied_end_tag?)
|
34
33
|
def implied_end_tag?
|
35
|
-
|
36
|
-
d ? d.endTag : nil
|
34
|
+
default_desc&.endTag
|
37
35
|
end
|
38
36
|
end
|
39
37
|
|
40
38
|
unless method_defined?(:save_end_tag?)
|
41
39
|
def save_end_tag?
|
42
|
-
|
43
|
-
d ? d.saveEndTag : nil
|
40
|
+
default_desc&.saveEndTag
|
44
41
|
end
|
45
42
|
end
|
46
43
|
|
47
44
|
unless method_defined?(:deprecated?)
|
48
45
|
def deprecated?
|
49
|
-
|
50
|
-
d ? d.depr : nil
|
46
|
+
default_desc&.depr
|
51
47
|
end
|
52
48
|
end
|
53
49
|
|
54
50
|
unless method_defined?(:description)
|
55
51
|
def description
|
56
|
-
|
57
|
-
d ? d.desc : nil
|
52
|
+
default_desc&.desc
|
58
53
|
end
|
59
54
|
end
|
60
55
|
|
61
56
|
unless method_defined?(:default_sub_element)
|
62
57
|
def default_sub_element
|
63
|
-
|
64
|
-
d ? d.defaultsubelt : nil
|
58
|
+
default_desc&.defaultsubelt
|
65
59
|
end
|
66
60
|
end
|
67
61
|
|
@@ -0,0 +1,121 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Nokogiri
|
4
|
+
module HTML4
|
5
|
+
# Libxml2's parser has poor support for encoding detection. First, it does not recognize the
|
6
|
+
# HTML5 style meta charset declaration. Secondly, even if it successfully detects an encoding
|
7
|
+
# hint, it does not re-decode or re-parse the preceding part which may be garbled.
|
8
|
+
#
|
9
|
+
# EncodingReader aims to perform advanced encoding detection beyond what Libxml2 does, and to
|
10
|
+
# emulate rewinding of a stream and make Libxml2 redo parsing from the start when an encoding
|
11
|
+
# hint is found.
|
12
|
+
|
13
|
+
# :nodoc: all
|
14
|
+
class EncodingReader
|
15
|
+
class EncodingFound < StandardError
|
16
|
+
attr_reader :found_encoding
|
17
|
+
|
18
|
+
def initialize(encoding)
|
19
|
+
@found_encoding = encoding
|
20
|
+
super(format("encoding found: %s", encoding))
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
class SAXHandler < Nokogiri::XML::SAX::Document
|
25
|
+
attr_reader :encoding
|
26
|
+
|
27
|
+
def initialize
|
28
|
+
@encoding = nil
|
29
|
+
super()
|
30
|
+
end
|
31
|
+
|
32
|
+
def start_element(name, attrs = [])
|
33
|
+
return unless name == "meta"
|
34
|
+
|
35
|
+
attr = Hash[attrs]
|
36
|
+
(charset = attr["charset"]) &&
|
37
|
+
(@encoding = charset)
|
38
|
+
(http_equiv = attr["http-equiv"]) &&
|
39
|
+
http_equiv.match(/\AContent-Type\z/i) &&
|
40
|
+
(content = attr["content"]) &&
|
41
|
+
(m = content.match(/;\s*charset\s*=\s*([\w-]+)/)) &&
|
42
|
+
(@encoding = m[1])
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
class JumpSAXHandler < SAXHandler
|
47
|
+
def initialize(jumptag)
|
48
|
+
@jumptag = jumptag
|
49
|
+
super()
|
50
|
+
end
|
51
|
+
|
52
|
+
def start_element(name, attrs = [])
|
53
|
+
super
|
54
|
+
throw(@jumptag, @encoding) if @encoding
|
55
|
+
throw(@jumptag, nil) if /\A(?:div|h1|img|p|br)\z/.match?(name)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def self.detect_encoding(chunk)
|
60
|
+
(m = chunk.match(/\A(<\?xml[ \t\r\n][^>]*>)/)) &&
|
61
|
+
(return Nokogiri.XML(m[1]).encoding)
|
62
|
+
|
63
|
+
if Nokogiri.jruby?
|
64
|
+
(m = chunk.match(/(<meta\s)(.*)(charset\s*=\s*([\w-]+))(.*)/i)) &&
|
65
|
+
(return m[4])
|
66
|
+
catch(:encoding_found) do
|
67
|
+
Nokogiri::HTML4::SAX::Parser.new(JumpSAXHandler.new(:encoding_found)).parse(chunk)
|
68
|
+
nil
|
69
|
+
end
|
70
|
+
else
|
71
|
+
handler = SAXHandler.new
|
72
|
+
parser = Nokogiri::HTML4::SAX::PushParser.new(handler)
|
73
|
+
begin
|
74
|
+
parser << chunk
|
75
|
+
rescue
|
76
|
+
Nokogiri::SyntaxError
|
77
|
+
end
|
78
|
+
handler.encoding
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
def initialize(io)
|
83
|
+
@io = io
|
84
|
+
@firstchunk = nil
|
85
|
+
@encoding_found = nil
|
86
|
+
end
|
87
|
+
|
88
|
+
# This method is used by the C extension so that
|
89
|
+
# Nokogiri::HTML4::Document#read_io() does not leak memory when
|
90
|
+
# EncodingFound is raised.
|
91
|
+
attr_reader :encoding_found
|
92
|
+
|
93
|
+
def read(len)
|
94
|
+
# no support for a call without len
|
95
|
+
|
96
|
+
unless @firstchunk
|
97
|
+
(@firstchunk = @io.read(len)) || (return nil)
|
98
|
+
|
99
|
+
# This implementation expects that the first call from
|
100
|
+
# htmlReadIO() is made with a length long enough (~1KB) to
|
101
|
+
# achieve advanced encoding detection.
|
102
|
+
if (encoding = EncodingReader.detect_encoding(@firstchunk))
|
103
|
+
# The first chunk is stored for the next read in retry.
|
104
|
+
raise @encoding_found = EncodingFound.new(encoding)
|
105
|
+
end
|
106
|
+
end
|
107
|
+
@encoding_found = nil
|
108
|
+
|
109
|
+
ret = @firstchunk.slice!(0, len)
|
110
|
+
if (len -= ret.length) > 0
|
111
|
+
(rest = @io.read(len)) && ret << (rest)
|
112
|
+
end
|
113
|
+
if ret.empty?
|
114
|
+
nil
|
115
|
+
else
|
116
|
+
ret
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
data/lib/nokogiri/html4.rb
CHANGED
@@ -39,6 +39,7 @@ end
|
|
39
39
|
require_relative "html4/entity_lookup"
|
40
40
|
require_relative "html4/document"
|
41
41
|
require_relative "html4/document_fragment"
|
42
|
+
require_relative "html4/encoding_reader"
|
42
43
|
require_relative "html4/sax/parser_context"
|
43
44
|
require_relative "html4/sax/parser"
|
44
45
|
require_relative "html4/sax/push_parser"
|