nokogumbo 1.5.0 → 2.0.0.pre.alpha

Sign up to get free protection for your applications and to get access to all the features.
Files changed (55) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +56 -0
  3. data/README.md +146 -22
  4. data/ext/nokogumbo/extconf.rb +116 -0
  5. data/ext/{nokogumboc → nokogumbo}/nokogumbo.c +174 -71
  6. data/gumbo-parser/src/ascii.c +33 -0
  7. data/gumbo-parser/src/ascii.h +31 -0
  8. data/gumbo-parser/src/attribute.c +26 -28
  9. data/gumbo-parser/src/attribute.h +3 -23
  10. data/gumbo-parser/src/char_ref.c +135 -2351
  11. data/gumbo-parser/src/char_ref.h +13 -29
  12. data/gumbo-parser/src/error.c +215 -133
  13. data/gumbo-parser/src/error.h +34 -49
  14. data/gumbo-parser/src/foreign_attrs.c +104 -0
  15. data/gumbo-parser/src/gumbo.h +506 -304
  16. data/gumbo-parser/src/insertion_mode.h +4 -28
  17. data/gumbo-parser/src/macros.h +91 -0
  18. data/gumbo-parser/src/parser.c +1989 -1431
  19. data/gumbo-parser/src/parser.h +6 -22
  20. data/gumbo-parser/src/replacement.h +33 -0
  21. data/gumbo-parser/src/string_buffer.c +43 -50
  22. data/gumbo-parser/src/string_buffer.h +24 -40
  23. data/gumbo-parser/src/string_piece.c +39 -39
  24. data/gumbo-parser/src/svg_attrs.c +174 -0
  25. data/gumbo-parser/src/svg_tags.c +137 -0
  26. data/gumbo-parser/src/tag.c +186 -59
  27. data/gumbo-parser/src/tag_lookup.c +382 -0
  28. data/gumbo-parser/src/tag_lookup.h +13 -0
  29. data/gumbo-parser/src/token_type.h +1 -25
  30. data/gumbo-parser/src/tokenizer.c +899 -495
  31. data/gumbo-parser/src/tokenizer.h +37 -37
  32. data/gumbo-parser/src/tokenizer_states.h +6 -22
  33. data/gumbo-parser/src/utf8.c +103 -86
  34. data/gumbo-parser/src/utf8.h +37 -41
  35. data/gumbo-parser/src/util.c +48 -38
  36. data/gumbo-parser/src/util.h +10 -40
  37. data/gumbo-parser/src/vector.c +45 -57
  38. data/gumbo-parser/src/vector.h +17 -39
  39. data/lib/nokogumbo.rb +10 -174
  40. data/lib/nokogumbo/html5.rb +250 -0
  41. data/lib/nokogumbo/html5/document.rb +37 -0
  42. data/lib/nokogumbo/html5/document_fragment.rb +46 -0
  43. data/lib/nokogumbo/version.rb +3 -0
  44. data/lib/nokogumbo/xml/node.rb +57 -0
  45. metadata +32 -19
  46. data/ext/nokogumboc/extconf.rb +0 -60
  47. data/gumbo-parser/src/char_ref.rl +0 -2554
  48. data/gumbo-parser/src/string_piece.h +0 -38
  49. data/gumbo-parser/src/tag.in +0 -150
  50. data/gumbo-parser/src/tag_enum.h +0 -153
  51. data/gumbo-parser/src/tag_gperf.h +0 -105
  52. data/gumbo-parser/src/tag_sizes.h +0 -4
  53. data/gumbo-parser/src/tag_strings.h +0 -153
  54. data/gumbo-parser/visualc/include/strings.h +0 -4
  55. data/test-nokogumbo.rb +0 -190
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 96fa61565f78d5491e0b6d5b505cf936524745eb848b8b6584fc15e20c7ae35b
4
- data.tar.gz: e5416f71bbe90323f04b8aad4dc48b28947e43a9eb46f446f8ca1444f519a07b
3
+ metadata.gz: e0d434c0749d7922ba8f084c15ed7219ccbf0e07b715368ae846bc38e64aad17
4
+ data.tar.gz: 2770648e3e9e82d0ffb1877f1c06edc537688cf6a8405bc52dbdf5a6bb69bc1a
5
5
  SHA512:
6
- metadata.gz: 676bf3585d38cd4ad5c72b8b3afd4952e248c747683ae1072dd43f6ce1ccd279177e4d0c75a9821ed76d32806333128152231349d8d113ae5d81279580b13004
7
- data.tar.gz: 3459078d96977399e75551c4a3ee5623091f48569984b771e540ec111125f5af91e39a8d78cbd3ce9280326b1b9395dc4a0b0d7f0a72294876682cb9fe35e3d9
6
+ metadata.gz: e6c3de49495bf55ccaa250e2a3275b6796b0f0565da2a930e3333d2a153f2a16312eb77cb28ca3e03c17720127c2ecc27a1f71cfd6acfd15407295c29973e9fb
7
+ data.tar.gz: e8ce6c80cb2327d2327f03c7e829156c1f0074ba4d6fce2b0d59305b80112b8fd5edc0932fad1fca13cb5f4bb6f2652fe52a2f090110aa76d06e1afbdebc334f
data/CHANGELOG.md ADDED
@@ -0,0 +1,56 @@
1
+ # Changelog
2
+
3
+ All notable changes to Nokogumbo will be documented in this file.
4
+
5
+ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
6
+ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
7
+
8
+ ## [Unreleased]
9
+ ### Added
10
+ - Experimental support for errors (it was supported in 1.5.0 but
11
+ undocumented).
12
+ - Added proper HTML5 serialization.
13
+ - Added option `:max_tree_depth` to control the maximum parse tree depth.
14
+
15
+ ### Changed
16
+ - Integrated [Gumbo parser](https://github.com/google/gumbo-parser) into
17
+ Nokogumbo. A system version will not be used.
18
+ - The undocumented (but publicly mentioned) `:max_parse_errors` renamed to `:max_errors`;
19
+ `:max_parse_errors` is deprecated and will go away
20
+ - The various `#parse` and `#fragment` (and `Nokogiri.HTML5`) methods return
21
+ `Nokogiri::HTML5::Document` and `Nokogiri::HTML5::DocumentFragment` classes
22
+ rather than `Nokogiri::HTML::Document` and
23
+ `Nokogiri::HTML::DocumentFragment`.
24
+ - Changed the top-level API to more closely match Nokogiri's while maintaining
25
+ backwards compatibility. The new APIs are
26
+ * `Nokogiri::HTML5(html, url = nil, encoding = nil, **options, &block)`
27
+ * `Nokogiri::HTML5.parse(html, url = nil, encoding = nil, **options, &block)`
28
+ * `Nokogiri::HTML5::Document.parse(html, url = nil, encoding = nil, **options, &block)`
29
+ * `Nokogiri::HTML5.fragment(html, encoding = nil, **options)`
30
+ * `Nokogiri::HTML5::DocumentFragment.parse(html, encoding = nil, **options)`
31
+ In all cases, `html` can be a string or an `IO` object (something that
32
+ responds to `#read`). The `url` parameter is entirely for error reporting,
33
+ as in Nokogiri. The `encoding` parameter only signals what encoding `html`
34
+ should have on input; the output `Document` or `DocumentFragment` will be in
35
+ UTF-8. Currently, the only options supported is `:max_errors` which controls
36
+ the maximum number of reported by `#errors`.
37
+
38
+ ### Deprecated
39
+ - `:max_parse_errors`; use `:max_errors`
40
+
41
+ ### Removed
42
+
43
+ ### Fixed
44
+ - Fixed documents failing to serialize (via `to_html`) if they contain certain
45
+ `meta` elements that set the `charset`.
46
+ - Documents are now properly marked as UTF-8 after parsing.
47
+ - Fixed `Nokogiri::HTML5.fragment` reporting an error due to a missing
48
+ `<!DOCTYPE html>`.
49
+ - Fixed crash when input contains U+0000 NULL bytes and error reporting is
50
+ enabled.
51
+
52
+ ### Security
53
+ - The most recent, released version of Gumbo has a [potential security
54
+ issue](https://github.com/google/gumbo-parser/pull/375) that could result in
55
+ a cross-site scripting vulnerability. This has been fixed by integrating
56
+ Gumbo into Nokogumbo.
data/README.md CHANGED
@@ -1,5 +1,4 @@
1
- Nokogumbo - a Nokogiri interface to the Gumbo HTML5 parser.
2
- ===========
1
+ # Nokogumbo - a Nokogiri interface to the Gumbo HTML5 parser.
3
2
 
4
3
  Nokogumbo provides the ability for a Ruby program to invoke the
5
4
  [Gumbo HTML5 parser](https://github.com/google/gumbo-parser#readme)
@@ -8,12 +7,11 @@ and to access the result as a
8
7
 
9
8
  [![Build Status](https://travis-ci.org/rubys/nokogumbo.svg)](https://travis-ci.org/rubys/nokogumbo)
10
9
 
11
- Usage
12
- -----
10
+ ## Usage
13
11
 
14
12
  ```ruby
15
13
  require 'nokogumbo'
16
- doc = Nokogiri::HTML5(string)
14
+ doc = Nokogiri.HTML5(string)
17
15
  ```
18
16
 
19
17
  An experimental _fragment_ method is also provided. While not HTML5
@@ -32,21 +30,150 @@ require 'nokogumbo'
32
30
  doc = Nokogiri::HTML5.get(uri)
33
31
  ```
34
32
 
35
- Example
36
- -----
33
+ ## Parsing options
34
+ The document and fragment parsing methods,
35
+ - `Nokogiri.HTML5(html, url = nil, encoding = nil, options = {})`
36
+ - `Nokogiri::HTML5.parse(html, url = nil, encoding = nil, options = {})`
37
+ - `Nokogiri::HTML5::Document.parse(html, url = nil, encoding = nil, options = {})`
38
+ - `Nokogiri::HTML5.fragment(html, encoding = nil, options = {})`
39
+ - `Nokogiri::HTML5::DocumentFragment.parse(html, encoding = nil, options = {})`
40
+ support options that are different from Nokogiri's.
41
+
42
+ The two currently supported options are `:max_errors` and `:max_tree_depth`,
43
+ described below.
44
+
45
+ ### Error reporting
46
+ Nokogumbo contains an experimental parse error reporting facility. By default,
47
+ no parse errors are reported but this can be configured by passing the
48
+ `:max_errors` option to `::parse` or `::fragment`.
49
+
37
50
  ```ruby
38
51
  require 'nokogumbo'
39
- puts Nokogiri::HTML5.get('http://nokogiri.org').search('ol li')[2].text
52
+ doc = Nokogiri::HTML5.parse('Hi there!<body>', max_errors: 10)
53
+ doc.errors.each do |err|
54
+ puts err
55
+ end
56
+ ```
57
+
58
+ This prints the following.
59
+ ```
60
+ 1:1: ERROR: @1:1: The doctype must be the first token in the document.
61
+ Hi there!<body>
62
+ ^
63
+ 1:10: ERROR: @1:10: That tag isn't allowed here Currently open tags: html, body..
64
+ Hi there!<body>
65
+ ^
66
+ ```
67
+
68
+ Using `max_errors: -1` results in an unlimited number of errors being
69
+ returned.
70
+
71
+ The errors returned by `#errors` are instances of
72
+ [`Nokogiri::XML::SyntaxError`](https://www.rubydoc.info/github/sparklemotion/nokogiri/Nokogiri/XML/SyntaxError).
73
+
74
+ ### Maximum tree depth
75
+ The maximum depth of the DOM tree parsed by the various parsing methods is
76
+ configurable by the `:max_tree_depth` option. If the depth of the tree would
77
+ exceed this limit, then an
78
+ [ArgumentError](https://ruby-doc.org/core-2.5.0/ArgumentError.html) is thrown.
79
+
80
+ This limit (which defaults to `Nokogumbo::DEFAULT_MAX_TREE_DEPTH = 400`) can
81
+ be removed by giving the option `max_tree_depth: -1`.
82
+
83
+ ``` ruby
84
+ html = '<!DOCTYPE html>' + '<div>' * 1000
85
+ doc = Nokogiri.HTML5(html)
86
+ # raises ArgumentError: Document tree depth limit exceeded
87
+ doc = Nokogiri.HTML5(html, max_tree_depth: -1)
40
88
  ```
41
89
 
42
- Use `.to_html` instead of `.to_s` when parsing and serializing multiple times
90
+ ## HTML Serialization
91
+
92
+ After parsing HTML, it may be serialized using any of the Nokogiri
93
+ [serialization
94
+ methods](https://www.rubydoc.info/gems/nokogiri/Nokogiri/XML/Node). In
95
+ particular, `#serialize`, `#to_html`, and `#to_s` will serialize a given node
96
+ and its children. (This is the equivalent of JavaScript's
97
+ `Element.outerHTML`.) Similarly, `#inner_html` will serialize the children of
98
+ a given node. (This is the equivalent of JavaScript's `Element.innerHTML`.)
99
+
100
+ ``` ruby
101
+ doc = Nokogiri::HTML5("<!DOCTYPE html><span>Hello world!</span>")
102
+ puts doc.serialize
103
+ # Prints: <!DOCTYPE html><html><head></head><body><span>Hello world!</span></body></html>
104
+ ```
105
+
106
+ Due to quirks in how HTML is parsed and serialized, it's possible for a DOM
107
+ tree to be serialized and then re-parsed, resulting in a different DOM.
108
+ Mostly, this happens with DOMs produced from invalid HTML. Unfortunately, even
109
+ valid HTML may not survive serialization and re-parsing.
110
+
111
+ In particular, a newline at the start of `pre`, `listing`, and `textarea`
112
+ elements is ignored by the parser.
113
+
114
+ ``` ruby
115
+ doc = Nokogiri::HTML5(<<-EOF)
116
+ <!DOCTYPE html>
117
+ <pre>
118
+ Content</pre>
119
+ EOF
120
+ puts doc.at('/html/body/pre').serialize
121
+ # Prints: <pre>Content</pre>
43
122
  ```
123
+
124
+ In this case, the original HTML is semantically equivalent to the serialized
125
+ version. If the `pre`, `listing`, or `textarea` content starts with two
126
+ newlines, the first newline will be stripped on the first parse and the second
127
+ newline will be stripped on the second, leading to semantically different
128
+ DOMs. Passing the parameter `preserve_newline: true` will cause two or more
129
+ newlines to be preserved. (A single leading newline will still be removed.)
130
+
131
+ ``` ruby
132
+ doc = Nokogiri::HTML5(<<-EOF)
133
+ <!DOCTYPE html>
134
+ <listing>
135
+
136
+ Content</listing>
137
+ EOF
138
+ puts doc.at('/html/body/listing').serialize(preserve_newline: true)
139
+ # Prints: <listing>
140
+ #
141
+ # Content</listing>
142
+ ```
143
+
144
+ ## Encodings
145
+ Nokogumbo always parses HTML using
146
+ [UTF-8](https://en.wikipedia.org/wiki/UTF-8); however, the encoding of the
147
+ input can be explicitly selected via the optional `encoding` parameter. This
148
+ is most useful when the input comes not from a string but from an IO object.
149
+
150
+ When serializing a document or node, the encoding of the output string can be
151
+ specified via the `:encoding` options. Characters that cannot be encoded in
152
+ the selected encoding will be encoded as [HTML numeric
153
+ entities](https://en.wikipedia.org/wiki/List_of_XML_and_HTML_character_entity_references).
154
+
155
+ ``` ruby
156
+ frag = Nokogiri::HTML5.fragment('<span>아는 길도 물어가라</span>')
157
+ html = frag.serialize(encoding: 'US-ASCII')
158
+ puts html
159
+ # Prints: <span>&#xc544;&#xb294; &#xae38;&#xb3c4; &#xbb3c;&#xc5b4;&#xac00;&#xb77c;</span>
160
+ frag = Nokogiri::HTML5.fragment(html)
161
+ puts frag.serialize
162
+ # Prints: <span>아는 길도 물어가라</span>
163
+ ```
164
+
165
+ (There's a [bug](https://bugs.ruby-lang.org/issues/15033) in all current
166
+ versions of Ruby that can cause the entity encoding to fail. Of the mandated
167
+ supported encodings for HTML, the only encoding I'm aware of that has this bug
168
+ is `'ISO-2022-JP'`. I recommend avoiding this encoding.)
169
+
170
+ ## Examples
171
+ ```ruby
44
172
  require 'nokogumbo'
45
- Nokogiri::HTML5.parse(Nokogiri::HTML5.parse('<div></div> a').to_html).to_html
173
+ puts Nokogiri::HTML5.get('http://nokogiri.org').search('ol li')[2].text
46
174
  ```
47
175
 
48
- Notes
49
- -----
176
+ ## Notes
50
177
 
51
178
  * The `Nokogiri::HTML5.fragment` function takes a string and parses it
52
179
  as a HTML5 document. The `<html>`, `<head>`, and `<body>` elements are
@@ -74,20 +201,17 @@ rules defined in the HTML5 specification for doing so.
74
201
  * Instead of returning `unknown` as the element name for unknown tags, the
75
202
  original tag name is returned verbatim.
76
203
 
77
- * If the Gumbo HTML5 parser is not already installed, the source for the
78
- parser will be downloaded and compiled into the Gem itself.
79
-
80
- Installation
81
- ============
204
+ # Installation
82
205
 
83
- git clone --recursive https://github.com/rubys/nokogumbo.git
206
+ git clone https://github.com/rubys/nokogumbo.git
84
207
  cd nokogumbo
85
208
  bundle install
86
209
  rake gem
87
210
  gem install pkg/nokogumbo*.gem
88
211
 
89
- Related efforts
90
- ============
212
+ # Related efforts
91
213
 
92
- * [ruby-gumbo](https://github.com/nevir/ruby-gumbo#readme) - a ruby binding
93
- for the Gumbo HTML5 parser.
214
+ * [ruby-gumbo](https://github.com/nevir/ruby-gumbo#readme) -- a ruby binding
215
+ for the Gumbo HTML5 parser.
216
+ * [lua-gumbo](https://gitlab.com/craigbarnes/lua-gumbo) -- a lua binding for
217
+ the Gumbo HTML5 parser.
data/ext/nokogumbo/extconf.rb ADDED
@@ -0,0 +1,116 @@
1
+ require 'fileutils'
2
+ require 'mkmf'
3
+ require 'nokogiri'
4
+
5
+ $CFLAGS += " -std=c99"
6
+ $LDFLAGS.gsub!('-Wl,--no-undefined', '')
7
+ $warnflags = CONFIG['warnflags'] = '-Wall'
8
+
9
+ NG_SPEC = Gem::Specification.find_by_name('nokogiri', "= #{Nokogiri::VERSION}")
10
+
11
+ def download_headers
12
+ begin
13
+ require 'yaml'
14
+
15
+ dependencies = YAML.load_file(File.join(NG_SPEC.gem_dir, 'dependencies.yml'))
16
+ version = dependencies['libxml2']['version']
17
+ host = RbConfig::CONFIG["host_alias"].empty? ? RbConfig::CONFIG["host"] : RbConfig::CONFIG["host_alias"]
18
+ path = File.join('ports', host, 'libxml2', version, 'include/libxml2')
19
+ return path if File.directory?(path)
20
+
21
+ # Make sure we're using the same version Nokogiri uses
22
+ dep_index = NG_SPEC.dependencies.index { |dep| dep.name == 'mini_portile2' and dep.type == :runtime }
23
+ return nil if dep_index.nil?
24
+ requirement = NG_SPEC.dependencies[dep_index].requirement.to_s
25
+
26
+ require 'rubygems'
27
+ gem 'mini_portile2', requirement
28
+ require 'mini_portile2'
29
+ p = MiniPortile::new('libxml2', version).tap do |r|
30
+ r.host = RbConfig::CONFIG["host_alias"].empty? ? RbConfig::CONFIG["host"] : RbConfig::CONFIG["host_alias"]
31
+ r.files = [{
32
+ url: "http://xmlsoft.org/sources/libxml2-#{r.version}.tar.gz",
33
+ sha256: dependencies['libxml2']['sha256']
34
+ }]
35
+ r.configure_options += [
36
+ "--without-python",
37
+ "--without-readline",
38
+ "--with-c14n",
39
+ "--with-debug",
40
+ "--with-threads"
41
+ ]
42
+ end
43
+ p.download unless p.downloaded?
44
+ p.extract
45
+ p.configure unless p.configured?
46
+ system('make', '-C', "tmp/#{p.host}/ports/libxml2/#{version}/libxml2-#{version}/include/libxml", 'install-xmlincHEADERS')
47
+ path
48
+ rescue
49
+ puts 'failed to download/install headers'
50
+ nil
51
+ end
52
+ end
53
+
54
+ required = arg_config('--with-libxml2')
55
+ prohibited = arg_config('--without-libxml2')
56
+ if required and prohibited
57
+ abort "cannot use both --with-libxml2 and --without-libxml2"
58
+ end
59
+
60
+ have_libxml2 = false
61
+ have_ng = false
62
+
63
+ if !prohibited
64
+ if Nokogiri::VERSION_INFO.include?('libxml') and
65
+ Nokogiri::VERSION_INFO['libxml']['source'] == 'packaged'
66
+ # Nokogiri has libxml2 built in. Find the headers.
67
+ libxml2_path = File.join(Nokogiri::VERSION_INFO['libxml']['libxml2_path'],
68
+ 'include/libxml2')
69
+ if find_header('libxml/tree.h', libxml2_path)
70
+ have_libxml2 = true
71
+ else
72
+ # Unfortunately, some versions of Nokogiri delete these files.
73
+ # https://github.com/sparklemotion/nokogiri/pull/1788
74
+ # Try to download them
75
+ libxml2_path = download_headers
76
+ unless libxml2_path.nil?
77
+ have_libxml2 = find_header('libxml/tree.h', libxml2_path)
78
+ end
79
+ end
80
+ else
81
+ # Nokogiri is compiled with system headers.
82
+ # Hack to work around broken mkmf on macOS
83
+ # (https://bugs.ruby-lang.org/issues/14992 fixed now)
84
+ if RbConfig::MAKEFILE_CONFIG['LIBPATHENV'] == 'DYLD_LIBRARY_PATH'
85
+ RbConfig::MAKEFILE_CONFIG['LIBPATHENV'] = 'DYLD_FALLBACK_LIBRARY_PATH'
86
+ end
87
+
88
+ pkg_config('libxml-2.0')
89
+ have_libxml2 = have_library('xml2', 'xmlNewDoc')
90
+ end
91
+ if required and !have_libxml2
92
+ abort "libxml2 required but could not be located"
93
+ end
94
+
95
+ if have_libxml2
96
+ # Find nokogiri.h
97
+ have_ng = find_header('nokogiri.h', File.join(NG_SPEC.gem_dir, 'ext/nokogiri'))
98
+ end
99
+ end
100
+
101
+ if have_libxml2 and have_ng
102
+ $CFLAGS += " -DNGLIB=1"
103
+ end
104
+
105
+ # Symlink gumbo-parser source files.
106
+ ext_dir = File.dirname(__FILE__)
107
+ gumbo_src = File.join(ext_dir, 'gumbo_src')
108
+
109
+ Dir.chdir(ext_dir) do
110
+ $srcs = Dir['*.c', '../../gumbo-parser/src/*.c']
111
+ end
112
+ $INCFLAGS << ' -I$(srcdir)/../../gumbo-parser/src'
113
+ $VPATH << '$(srcdir)/../../gumbo-parser/src'
114
+
115
+ create_makefile('nokogumbo/nokogumbo')
116
+ # vim: set sw=2 sts=2 ts=8 et:
data/ext/{nokogumboc → nokogumbo}/nokogumbo.c RENAMED
@@ -2,7 +2,7 @@
2
2
  // nokogumbo.c defines the following:
3
3
  //
4
4
  // class Nokogumbo
5
- // def parse(utf8_string) # returns Nokogiri::HTML::Document
5
+ // def parse(utf8_string) # returns Nokogiri::HTML5::Document
6
6
  // end
7
7
  //
8
8
  // Processing starts by calling gumbo_parse_with_options. The resulting
@@ -18,26 +18,29 @@
18
18
  // methods are called instead, producing the equivalent functionality.
19
19
  //
20
20
 
21
+ #include <assert.h>
21
22
  #include <ruby.h>
22
23
  #include "gumbo.h"
23
24
  #include "error.h"
24
- #include "parser.h"
25
25
 
26
26
  // class constants
27
27
  static VALUE Document;
28
- static VALUE XMLSyntaxError;
29
28
 
30
29
  #ifdef NGLIB
31
30
  #include <nokogiri.h>
31
+ #include <xml_syntax_error.h>
32
32
  #include <libxml/tree.h>
33
+ #include <libxml/HTMLtree.h>
33
34
 
34
35
  #define NIL NULL
35
36
  #define CONST_CAST (xmlChar const*)
36
37
  #else
37
- #define NIL 0
38
+ #define NIL Qnil
38
39
  #define CONST_CAST
39
40
 
40
41
  // more class constants
42
+ static VALUE cNokogiriXmlSyntaxError;
43
+
41
44
  static VALUE Element;
42
45
  static VALUE Text;
43
46
  static VALUE CDATA;
@@ -45,11 +48,15 @@ static VALUE Comment;
45
48
 
46
49
  // interned symbols
47
50
  static VALUE new;
51
+ static VALUE attribute;
48
52
  static VALUE set_attribute;
53
+ static VALUE remove_attribute;
49
54
  static VALUE add_child;
50
55
  static VALUE internal_subset;
51
56
  static VALUE remove_;
52
57
  static VALUE create_internal_subset;
58
+ static VALUE key_;
59
+ static VALUE node_name_;
53
60
 
54
61
  // map libxml2 types to Ruby VALUE
55
62
  #define xmlNodePtr VALUE
@@ -58,12 +65,10 @@ static VALUE create_internal_subset;
58
65
  // redefine libxml2 API as Ruby function calls
59
66
  #define xmlNewDocNode(doc, ns, name, content) \
60
67
  rb_funcall(Element, new, 2, rb_str_new2(name), doc)
61
- #define xmlNewProp(element, name, value) \
62
- rb_funcall(element, set_attribute, 2, rb_str_new2(name), rb_str_new2(value))
63
68
  #define xmlNewDocText(doc, text) \
64
69
  rb_funcall(Text, new, 2, rb_str_new2(text), doc)
65
70
  #define xmlNewCDataBlock(doc, content, length) \
66
- rb_funcall(CDATA, new, 2, rb_str_new(content, length), doc)
71
+ rb_funcall(CDATA, new, 2, doc, rb_str_new(content, length))
67
72
  #define xmlNewDocComment(doc, text) \
68
73
  rb_funcall(Comment, new, 2, doc, rb_str_new2(text))
69
74
  #define xmlAddChild(element, node) \
@@ -77,11 +82,76 @@ static VALUE create_internal_subset;
77
82
  #define Nokogiri_wrap_xml_document(klass, doc) \
78
83
  doc
79
84
 
80
- // remove internal subset from newly created documents
81
- static VALUE xmlNewDoc(char* version) {
82
- VALUE doc = rb_funcall(Document, new, 0);
83
- rb_funcall(rb_funcall(doc, internal_subset, 0), remove_, 0);
84
- return doc;
85
+ static VALUE find_dummy_key(VALUE collection) {
86
+ VALUE r_dummy = Qnil;
87
+ char dummy[5] = "a";
88
+ size_t len = 1;
89
+ while (len < sizeof dummy) {
90
+ r_dummy = rb_str_new(dummy, len);
91
+ if (rb_funcall(collection, key_, 1, r_dummy) == Qfalse)
92
+ return r_dummy;
93
+ for (size_t i = 0; ; ++i) {
94
+ if (dummy[i] == 0) {
95
+ dummy[i] = 'a';
96
+ ++len;
97
+ break;
98
+ }
99
+ if (dummy[i] == 'z')
100
+ dummy[i] = 'a';
101
+ else {
102
+ ++dummy[i];
103
+ break;
104
+ }
105
+ }
106
+ }
107
+ // This collection has 475254 elements?? Give up.
108
+ return Qnil;
109
+ }
110
+
111
+ static xmlNodePtr xmlNewProp(xmlNodePtr node, const char *name, const char *value) {
112
+ // Nokogiri::XML::Node#set_attribute calls xmlSetProp(node, name, value)
113
+ // which behaves roughly as
114
+ // if name is a QName prefix:local
115
+ // if node->doc has a namespace ns corresponding to prefix
116
+ // return xmlSetNsProp(node, ns, local, value)
117
+ // return xmlSetNsProp(node, NULL, name, value)
118
+ //
119
+ // If the prefix is "xml", then the namespace lookup will create it.
120
+ //
121
+ // By contrast, xmlNewProp does not do this parsing and creates an attribute
122
+ // with the name and value exactly as given. This is the behavior that we
123
+ // want.
124
+ //
125
+ // Thus, for attribute names like "xml:lang", #set_attribute will create an
126
+ // attribute with namespace "xml" and name "lang". This is incorrect for
127
+ // html elements (but correct for foreign elements).
128
+ //
129
+ // Work around this by inserting a dummy attribute and then changing the
130
+ // name, if needed.
131
+
132
+ // Can't use strchr since it's locale-sensitive.
133
+ size_t len = strlen(name);
134
+ VALUE r_name = rb_str_new(name, len);
135
+ if (memchr(name, ':', len) == NULL) {
136
+ // No colon.
137
+ return rb_funcall(node, set_attribute, 2, r_name, rb_str_new2(value));
138
+ }
139
+ // Find a dummy attribute string that doesn't already exist.
140
+ VALUE dummy = find_dummy_key(node);
141
+ if (dummy == Qnil)
142
+ return Qnil;
143
+ // Add the dummy attribute.
144
+ VALUE r_value = rb_funcall(node, set_attribute, 2, dummy, rb_str_new2(value));
145
+ if (r_value == Qnil)
146
+ return Qnil;
147
+ // Remove thet old attribute, if it exists.
148
+ rb_funcall(node, remove_attribute, 1, r_name);
149
+ // Rename the dummy
150
+ VALUE attr = rb_funcall(node, attribute, 1, dummy);
151
+ if (attr == Qnil)
152
+ return Qnil;
153
+ rb_funcall(attr, node_name_, 1, r_name);
154
+ return attr;
85
155
  }
86
156
  #endif
87
157
 
@@ -90,30 +160,15 @@ static xmlNodePtr walk_tree(xmlDocPtr document, GumboNode *node);
90
160
 
91
161
  // Build a xmlNodePtr for a given GumboElement (recursively)
92
162
  static xmlNodePtr walk_element(xmlDocPtr document, GumboElement *node) {
93
- // determine tag name for a given node
94
- xmlNodePtr element;
95
- if (node->tag != GUMBO_TAG_UNKNOWN) {
96
- element = xmlNewDocNode(document, NIL,
97
- CONST_CAST gumbo_normalized_tagname(node->tag), NIL);
98
- } else {
99
- GumboStringPiece tag = node->original_tag;
100
- gumbo_tag_from_original_text(&tag);
101
- #ifdef _MSC_VER
102
- char* name = alloca(tag.length+1);
103
- #else
104
- char name[tag.length+1];
105
- #endif
106
- strncpy(name, tag.data, tag.length);
107
- name[tag.length] = '\0';
108
- element = xmlNewDocNode(document, NIL, CONST_CAST name, NIL);
109
- }
163
+ // create the given element
164
+ xmlNodePtr element = xmlNewDocNode(document, NIL, CONST_CAST node->name, NIL);
110
165
 
111
166
  // add in the attributes
112
167
  GumboVector* attrs = &node->attributes;
113
168
  char *name = NULL;
114
- int namelen = 0;
115
- char *ns;
116
- for (int i=0; i < attrs->length; i++) {
169
+ size_t namelen = 0;
170
+ const char *ns;
171
+ for (size_t i=0; i < attrs->length; i++) {
117
172
  GumboAttribute *attr = attrs->data[i];
118
173
 
119
174
  switch (attr->attr_namespace) {
@@ -156,7 +211,7 @@ static xmlNodePtr walk_element(xmlDocPtr document, GumboElement *node) {
156
211
 
157
212
  // add in the children
158
213
  GumboVector* children = &node->children;
159
- for (int i=0; i < children->length; i++) {
214
+ for (size_t i=0; i < children->length; i++) {
160
215
  xmlNodePtr node = walk_tree(document, children->data[i]);
161
216
  if (node) xmlAddChild(element, node);
162
217
  }
@@ -176,37 +231,89 @@ static xmlNodePtr walk_tree(xmlDocPtr document, GumboNode *node) {
176
231
  return xmlNewDocText(document, CONST_CAST node->v.text.text);
177
232
  case GUMBO_NODE_CDATA:
178
233
  return xmlNewCDataBlock(document,
179
- CONST_CAST node->v.text.original_text.data,
180
- (int) node->v.text.original_text.length);
234
+ CONST_CAST node->v.text.text,
235
+ (int) strlen(node->v.text.text));
181
236
  case GUMBO_NODE_COMMENT:
182
237
  return xmlNewDocComment(document, CONST_CAST node->v.text.text);
183
238
  }
184
239
  }
185
240
 
241
+ // URI = system id
242
+ // external id = public id
243
+ #if NGLIB
244
+ static htmlDocPtr new_html_doc(const char *dtd_name, const char *system, const char *public)
245
+ {
246
+ // These two libxml2 functions take the public and system ids in
247
+ // opposite orders.
248
+ htmlDocPtr doc = htmlNewDocNoDtD(/* URI */ NULL, /* ExternalID */NULL);
249
+ assert(doc);
250
+ if (dtd_name)
251
+ xmlCreateIntSubset(doc, CONST_CAST dtd_name, CONST_CAST public, CONST_CAST system);
252
+ return doc;
253
+ }
254
+ #else
255
+ // remove internal subset from newly created documents
256
+ static VALUE new_html_doc(const char *dtd_name, const char *system, const char *public) {
257
+ VALUE doc;
258
+ // If system and public are both NULL, Document#new is going to set default
259
+ // values for them so we're going to have to remove the internal subset
260
+ // which seems to leak memory in Nokogiri, so leak as little as possible.
261
+ if (system == NULL && public == NULL) {
262
+ doc = rb_funcall(Document, new, 2, /* URI */ Qnil, /* external_id */ rb_str_new("", 0));
263
+ rb_funcall(rb_funcall(doc, internal_subset, 0), remove_, 0);
264
+ if (dtd_name) {
265
+ // We need to create an internal subset now.
266
+ rb_funcall(doc, create_internal_subset, 3, rb_str_new2(dtd_name), Qnil, Qnil);
267
+ }
268
+ } else {
269
+ assert(dtd_name);
270
+ // Rather than removing and creating the internal subset as we did above,
271
+ // just create and then rename one.
272
+ VALUE r_system = system ? rb_str_new2(system) : Qnil;
273
+ VALUE r_public = public ? rb_str_new2(public) : Qnil;
274
+ doc = rb_funcall(Document, new, 2, r_system, r_public);
275
+ rb_funcall(rb_funcall(doc, internal_subset, 0), node_name_, 1, rb_str_new2(dtd_name));
276
+ }
277
+ return doc;
278
+ }
279
+ #endif
280
+
186
281
  // Parse a string using gumbo_parse into a Nokogiri document
187
- static VALUE parse(VALUE self, VALUE string, VALUE max_parse_errors) {
188
- GumboOptions options;
189
- memcpy(&options, &kGumboDefaultOptions, sizeof options);
190
- options.max_errors = NUM2INT(max_parse_errors);
282
+ static VALUE parse(VALUE self, VALUE string, VALUE url, VALUE max_errors, VALUE max_depth) {
283
+ GumboOptions options = kGumboDefaultOptions;
284
+ options.max_errors = NUM2INT(max_errors);
285
+ options.max_tree_depth = NUM2INT(max_depth);
191
286
 
192
287
  const char *input = RSTRING_PTR(string);
193
288
  size_t input_len = RSTRING_LEN(string);
194
289
  GumboOutput *output = gumbo_parse_with_options(&options, input, input_len);
195
- xmlDocPtr doc = xmlNewDoc(CONST_CAST "1.0");
196
- #ifdef NGLIB
197
- doc->type = XML_HTML_DOCUMENT_NODE;
198
- #endif
290
+
291
+ const char *status_string = gumbo_status_to_string(output->status);
292
+ switch (output->status) {
293
+ case GUMBO_STATUS_OK:
294
+ break;
295
+ case GUMBO_STATUS_TREE_TOO_DEEP:
296
+ gumbo_destroy_output(output);
297
+ rb_raise(rb_eArgError, "%s", status_string);
298
+ case GUMBO_STATUS_OUT_OF_MEMORY:
299
+ gumbo_destroy_output(output);
300
+ rb_raise(rb_eNoMemError, "%s", status_string);
301
+ }
302
+
303
+ xmlDocPtr doc;
199
304
  if (output->document->v.document.has_doctype) {
200
305
  const char *name = output->document->v.document.name;
201
306
  const char *public = output->document->v.document.public_identifier;
202
307
  const char *system = output->document->v.document.system_identifier;
203
- xmlCreateIntSubset(doc, CONST_CAST name,
204
- (public[0] ? CONST_CAST public : NIL),
205
- (system[0] ? CONST_CAST system : NIL));
308
+ public = public[0] ? public : NULL;
309
+ system = system[0] ? system : NULL;
310
+ doc = new_html_doc(name, system, public);
311
+ } else {
312
+ doc = new_html_doc(NULL, NULL, NULL);
206
313
  }
207
314
 
208
315
  GumboVector *children = &output->document->v.document.children;
209
- for (int i=0; i < children->length; i++) {
316
+ for (size_t i=0; i < children->length; i++) {
210
317
  GumboNode *child = children->data[i];
211
318
  xmlNodePtr node = walk_tree(doc, child);
212
319
  if (node) {
@@ -222,28 +329,20 @@ static VALUE parse(VALUE self, VALUE string, VALUE max_parse_errors) {
222
329
  // Add parse errors to rdoc.
223
330
  if (output->errors.length) {
224
331
  GumboVector *errors = &output->errors;
225
- GumboParser parser = { ._options = &options };
226
332
  GumboStringBuffer msg;
227
333
  VALUE rerrors = rb_ary_new2(errors->length);
228
334
 
229
- gumbo_string_buffer_init(&parser, &msg);
230
- for (int i=0; i < errors->length; i++) {
335
+ gumbo_string_buffer_init(&msg);
336
+ for (size_t i=0; i < errors->length; i++) {
231
337
  GumboError *err = errors->data[i];
232
- gumbo_string_buffer_clear(&parser, &msg);
233
- // Work around bug in gumbo_caret_diagnostic_to_string.
234
- // See https://github.com/google/gumbo-parser/pull/371
235
- // The bug occurs when the error starts with a newline (unless it's the
236
- // first character in the input--but that shouldn't cause an error in
237
- // the first place.
238
- if (*err->original_text == '\n' && err->original_text != input)
239
- --err->original_text;
240
- gumbo_caret_diagnostic_to_string(&parser, err, input, &msg);
338
+ gumbo_string_buffer_clear(&msg);
339
+ gumbo_caret_diagnostic_to_string(err, input, input_len, &msg);
241
340
  VALUE err_str = rb_str_new(msg.data, msg.length);
242
- VALUE syntax_error = rb_class_new_instance(1, &err_str, XMLSyntaxError);
341
+ VALUE syntax_error = rb_class_new_instance(1, &err_str, cNokogiriXmlSyntaxError);
243
342
  rb_iv_set(syntax_error, "@domain", INT2NUM(1)); // XML_FROM_PARSER
244
343
  rb_iv_set(syntax_error, "@code", INT2NUM(1)); // XML_ERR_INTERNAL_ERROR
245
344
  rb_iv_set(syntax_error, "@level", INT2NUM(2)); // XML_ERR_ERROR
246
- rb_iv_set(syntax_error, "@file", Qnil);
345
+ rb_iv_set(syntax_error, "@file", url);
247
346
  rb_iv_set(syntax_error, "@line", INT2NUM(err->position.line));
248
347
  rb_iv_set(syntax_error, "@str1", Qnil);
249
348
  rb_iv_set(syntax_error, "@str2", Qnil);
@@ -253,28 +352,28 @@ static VALUE parse(VALUE self, VALUE string, VALUE max_parse_errors) {
253
352
  rb_ary_push(rerrors, syntax_error);
254
353
  }
255
354
  rb_iv_set(rdoc, "@errors", rerrors);
256
- gumbo_string_buffer_destroy(&parser, &msg);
355
+ gumbo_string_buffer_destroy(&msg);
257
356
  }
258
357
 
259
- gumbo_destroy_output(&options, output);
358
+ gumbo_destroy_output(output);
260
359
 
261
360
  return rdoc;
262
361
  }
263
362
 
264
363
  // Initialize the Nokogumbo class and fetch constants we will use later
265
- void Init_nokogumboc() {
364
+ void Init_nokogumbo() {
266
365
  rb_funcall(rb_mKernel, rb_intern("gem"), 1, rb_str_new2("nokogiri"));
267
366
  rb_require("nokogiri");
268
367
 
269
368
  // class constants
270
369
  VALUE Nokogiri = rb_const_get(rb_cObject, rb_intern("Nokogiri"));
271
- VALUE HTML = rb_const_get(Nokogiri, rb_intern("HTML"));
272
- Document = rb_const_get(HTML, rb_intern("Document"));
273
- VALUE XML = rb_const_get(Nokogiri, rb_intern("XML"));
274
- XMLSyntaxError = rb_const_get(XML, rb_intern("SyntaxError"));
370
+ VALUE HTML5 = rb_const_get(Nokogiri, rb_intern("HTML5"));
371
+ Document = rb_const_get(HTML5, rb_intern("Document"));
275
372
 
276
373
  #ifndef NGLIB
277
374
  // more class constants
375
+ VALUE XML = rb_const_get(Nokogiri, rb_intern("XML"));
376
+ cNokogiriXmlSyntaxError = rb_const_get(XML, rb_intern("SyntaxError"));
278
377
  Element = rb_const_get(XML, rb_intern("Element"));
279
378
  Text = rb_const_get(XML, rb_intern("Text"));
280
379
  CDATA = rb_const_get(XML, rb_intern("CDATA"));
@@ -282,14 +381,18 @@ void Init_nokogumboc() {
282
381
 
283
382
  // interned symbols
284
383
  new = rb_intern("new");
384
+ attribute = rb_intern("attribute");
285
385
  set_attribute = rb_intern("set_attribute");
386
+ remove_attribute = rb_intern("remove_attribute");
286
387
  add_child = rb_intern("add_child_node_and_reparent_attrs");
287
388
  internal_subset = rb_intern("internal_subset");
288
389
  remove_ = rb_intern("remove");
289
390
  create_internal_subset = rb_intern("create_internal_subset");
391
+ key_ = rb_intern("key?");
392
+ node_name_ = rb_intern("node_name=");
290
393
  #endif
291
394
 
292
- // define Nokogumbo class with a singleton parse method
293
- VALUE Gumbo = rb_define_class("Nokogumbo", rb_cObject);
294
- rb_define_singleton_method(Gumbo, "parse", parse, 2);
395
+ // define Nokogumbo module with a parse method
396
+ VALUE Gumbo = rb_define_module("Nokogumbo");
397
+ rb_define_singleton_method(Gumbo, "parse", parse, 4);
295
398
  }