nokogiri 1.8.5 → 1.13.6

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of nokogiri might be problematic. Click here for more details.

Files changed (356) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +3 -21
  3. data/LICENSE-DEPENDENCIES.md +1159 -868
  4. data/LICENSE.md +5 -28
  5. data/README.md +196 -90
  6. data/bin/nokogiri +63 -50
  7. data/dependencies.yml +13 -59
  8. data/ext/nokogiri/depend +38 -358
  9. data/ext/nokogiri/extconf.rb +750 -420
  10. data/ext/nokogiri/gumbo.c +584 -0
  11. data/ext/nokogiri/html4_document.c +166 -0
  12. data/ext/nokogiri/html4_element_description.c +294 -0
  13. data/ext/nokogiri/html4_entity_lookup.c +37 -0
  14. data/ext/nokogiri/html4_sax_parser_context.c +119 -0
  15. data/ext/nokogiri/html4_sax_push_parser.c +95 -0
  16. data/ext/nokogiri/libxml2_backwards_compat.c +121 -0
  17. data/ext/nokogiri/nokogiri.c +228 -91
  18. data/ext/nokogiri/nokogiri.h +191 -89
  19. data/ext/nokogiri/test_global_handlers.c +40 -0
  20. data/ext/nokogiri/xml_attr.c +41 -36
  21. data/ext/nokogiri/xml_attribute_decl.c +18 -18
  22. data/ext/nokogiri/xml_cdata.c +13 -18
  23. data/ext/nokogiri/xml_comment.c +19 -26
  24. data/ext/nokogiri/xml_document.c +291 -216
  25. data/ext/nokogiri/xml_document_fragment.c +12 -16
  26. data/ext/nokogiri/xml_dtd.c +56 -50
  27. data/ext/nokogiri/xml_element_content.c +31 -26
  28. data/ext/nokogiri/xml_element_decl.c +22 -22
  29. data/ext/nokogiri/xml_encoding_handler.c +43 -18
  30. data/ext/nokogiri/xml_entity_decl.c +32 -30
  31. data/ext/nokogiri/xml_entity_reference.c +16 -18
  32. data/ext/nokogiri/xml_namespace.c +61 -52
  33. data/ext/nokogiri/xml_node.c +1044 -616
  34. data/ext/nokogiri/xml_node_set.c +174 -162
  35. data/ext/nokogiri/xml_processing_instruction.c +17 -19
  36. data/ext/nokogiri/xml_reader.c +226 -175
  37. data/ext/nokogiri/xml_relax_ng.c +52 -28
  38. data/ext/nokogiri/xml_sax_parser.c +112 -112
  39. data/ext/nokogiri/xml_sax_parser_context.c +112 -86
  40. data/ext/nokogiri/xml_sax_push_parser.c +36 -27
  41. data/ext/nokogiri/xml_schema.c +112 -33
  42. data/ext/nokogiri/xml_syntax_error.c +42 -21
  43. data/ext/nokogiri/xml_text.c +13 -17
  44. data/ext/nokogiri/xml_xpath_context.c +223 -115
  45. data/ext/nokogiri/xslt_stylesheet.c +265 -173
  46. data/gumbo-parser/CHANGES.md +63 -0
  47. data/gumbo-parser/Makefile +101 -0
  48. data/gumbo-parser/THANKS +27 -0
  49. data/gumbo-parser/src/Makefile +34 -0
  50. data/gumbo-parser/src/README.md +41 -0
  51. data/gumbo-parser/src/ascii.c +75 -0
  52. data/gumbo-parser/src/ascii.h +115 -0
  53. data/gumbo-parser/src/attribute.c +42 -0
  54. data/gumbo-parser/src/attribute.h +17 -0
  55. data/gumbo-parser/src/char_ref.c +22225 -0
  56. data/gumbo-parser/src/char_ref.h +29 -0
  57. data/gumbo-parser/src/char_ref.rl +2154 -0
  58. data/gumbo-parser/src/error.c +626 -0
  59. data/gumbo-parser/src/error.h +148 -0
  60. data/gumbo-parser/src/foreign_attrs.c +104 -0
  61. data/gumbo-parser/src/foreign_attrs.gperf +27 -0
  62. data/gumbo-parser/src/gumbo.h +943 -0
  63. data/gumbo-parser/src/insertion_mode.h +33 -0
  64. data/gumbo-parser/src/macros.h +91 -0
  65. data/gumbo-parser/src/parser.c +4875 -0
  66. data/gumbo-parser/src/parser.h +41 -0
  67. data/gumbo-parser/src/replacement.h +33 -0
  68. data/gumbo-parser/src/string_buffer.c +103 -0
  69. data/gumbo-parser/src/string_buffer.h +68 -0
  70. data/gumbo-parser/src/string_piece.c +48 -0
  71. data/gumbo-parser/src/svg_attrs.c +174 -0
  72. data/gumbo-parser/src/svg_attrs.gperf +77 -0
  73. data/gumbo-parser/src/svg_tags.c +137 -0
  74. data/gumbo-parser/src/svg_tags.gperf +55 -0
  75. data/gumbo-parser/src/tag.c +222 -0
  76. data/gumbo-parser/src/tag_lookup.c +382 -0
  77. data/gumbo-parser/src/tag_lookup.gperf +169 -0
  78. data/gumbo-parser/src/tag_lookup.h +13 -0
  79. data/gumbo-parser/src/token_buffer.c +79 -0
  80. data/gumbo-parser/src/token_buffer.h +71 -0
  81. data/gumbo-parser/src/token_type.h +17 -0
  82. data/gumbo-parser/src/tokenizer.c +3463 -0
  83. data/gumbo-parser/src/tokenizer.h +112 -0
  84. data/gumbo-parser/src/tokenizer_states.h +339 -0
  85. data/gumbo-parser/src/utf8.c +245 -0
  86. data/gumbo-parser/src/utf8.h +164 -0
  87. data/gumbo-parser/src/util.c +68 -0
  88. data/gumbo-parser/src/util.h +30 -0
  89. data/gumbo-parser/src/vector.c +111 -0
  90. data/gumbo-parser/src/vector.h +45 -0
  91. data/lib/nokogiri/class_resolver.rb +67 -0
  92. data/lib/nokogiri/css/node.rb +10 -8
  93. data/lib/nokogiri/css/parser.rb +397 -377
  94. data/lib/nokogiri/css/parser.y +250 -245
  95. data/lib/nokogiri/css/parser_extras.rb +54 -49
  96. data/lib/nokogiri/css/syntax_error.rb +3 -1
  97. data/lib/nokogiri/css/tokenizer.rb +107 -104
  98. data/lib/nokogiri/css/tokenizer.rex +3 -2
  99. data/lib/nokogiri/css/xpath_visitor.rb +218 -91
  100. data/lib/nokogiri/css.rb +50 -17
  101. data/lib/nokogiri/decorators/slop.rb +9 -7
  102. data/lib/nokogiri/extension.rb +31 -0
  103. data/lib/nokogiri/gumbo.rb +15 -0
  104. data/lib/nokogiri/html.rb +38 -27
  105. data/lib/nokogiri/{html → html4}/builder.rb +4 -2
  106. data/lib/nokogiri/{html → html4}/document.rb +103 -105
  107. data/lib/nokogiri/html4/document_fragment.rb +54 -0
  108. data/lib/nokogiri/{html → html4}/element_description.rb +3 -1
  109. data/lib/nokogiri/html4/element_description_defaults.rb +578 -0
  110. data/lib/nokogiri/{html → html4}/entity_lookup.rb +4 -2
  111. data/lib/nokogiri/{html → html4}/sax/parser.rb +17 -16
  112. data/lib/nokogiri/html4/sax/parser_context.rb +20 -0
  113. data/lib/nokogiri/{html → html4}/sax/push_parser.rb +12 -11
  114. data/lib/nokogiri/html4.rb +46 -0
  115. data/lib/nokogiri/html5/document.rb +91 -0
  116. data/lib/nokogiri/html5/document_fragment.rb +83 -0
  117. data/lib/nokogiri/html5/node.rb +100 -0
  118. data/lib/nokogiri/html5.rb +478 -0
  119. data/lib/nokogiri/jruby/dependencies.rb +21 -0
  120. data/lib/nokogiri/syntax_error.rb +2 -0
  121. data/lib/nokogiri/version/constant.rb +6 -0
  122. data/lib/nokogiri/version/info.rb +222 -0
  123. data/lib/nokogiri/version.rb +3 -108
  124. data/lib/nokogiri/xml/attr.rb +6 -3
  125. data/lib/nokogiri/xml/attribute_decl.rb +3 -1
  126. data/lib/nokogiri/xml/builder.rb +97 -53
  127. data/lib/nokogiri/xml/cdata.rb +3 -1
  128. data/lib/nokogiri/xml/character_data.rb +2 -0
  129. data/lib/nokogiri/xml/document.rb +224 -86
  130. data/lib/nokogiri/xml/document_fragment.rb +57 -44
  131. data/lib/nokogiri/xml/dtd.rb +4 -2
  132. data/lib/nokogiri/xml/element_content.rb +2 -0
  133. data/lib/nokogiri/xml/element_decl.rb +3 -1
  134. data/lib/nokogiri/xml/entity_decl.rb +4 -2
  135. data/lib/nokogiri/xml/entity_reference.rb +2 -0
  136. data/lib/nokogiri/xml/namespace.rb +3 -0
  137. data/lib/nokogiri/xml/node/save_options.rb +10 -5
  138. data/lib/nokogiri/xml/node.rb +895 -377
  139. data/lib/nokogiri/xml/node_set.rb +92 -65
  140. data/lib/nokogiri/xml/notation.rb +13 -0
  141. data/lib/nokogiri/xml/parse_options.rb +22 -8
  142. data/lib/nokogiri/xml/pp/character_data.rb +9 -6
  143. data/lib/nokogiri/xml/pp/node.rb +25 -26
  144. data/lib/nokogiri/xml/pp.rb +4 -2
  145. data/lib/nokogiri/xml/processing_instruction.rb +3 -1
  146. data/lib/nokogiri/xml/reader.rb +23 -28
  147. data/lib/nokogiri/xml/relax_ng.rb +8 -2
  148. data/lib/nokogiri/xml/sax/document.rb +45 -49
  149. data/lib/nokogiri/xml/sax/parser.rb +38 -34
  150. data/lib/nokogiri/xml/sax/parser_context.rb +8 -3
  151. data/lib/nokogiri/xml/sax/push_parser.rb +6 -5
  152. data/lib/nokogiri/xml/sax.rb +6 -4
  153. data/lib/nokogiri/xml/schema.rb +19 -9
  154. data/lib/nokogiri/xml/searchable.rb +112 -72
  155. data/lib/nokogiri/xml/syntax_error.rb +6 -4
  156. data/lib/nokogiri/xml/text.rb +2 -0
  157. data/lib/nokogiri/xml/xpath/syntax_error.rb +4 -2
  158. data/lib/nokogiri/xml/xpath.rb +15 -4
  159. data/lib/nokogiri/xml/xpath_context.rb +3 -3
  160. data/lib/nokogiri/xml.rb +38 -37
  161. data/lib/nokogiri/xslt/stylesheet.rb +3 -1
  162. data/lib/nokogiri/xslt.rb +29 -20
  163. data/lib/nokogiri.rb +49 -65
  164. data/lib/xsd/xmlparser/nokogiri.rb +26 -24
  165. data/patches/libxml2/0001-Remove-script-macro-support.patch +40 -0
  166. data/patches/libxml2/0002-Update-entities-to-remove-handling-of-ssi.patch +44 -0
  167. data/patches/libxml2/0003-libxml2.la-is-in-top_builddir.patch +25 -0
  168. data/patches/libxml2/0004-use-glibc-strlen.patch +53 -0
  169. data/patches/libxml2/0005-avoid-isnan-isinf.patch +81 -0
  170. data/patches/libxml2/0006-update-automake-files-for-arm64.patch +3040 -0
  171. data/patches/libxml2/0008-htmlParseComment-handle-abruptly-closed-comments.patch +61 -0
  172. data/patches/libxml2/0009-allow-wildcard-namespaces.patch +77 -0
  173. data/patches/libxslt/0001-update-automake-files-for-arm64.patch +3037 -0
  174. data/ports/archives/libxml2-2.9.14.tar.xz +0 -0
  175. data/ports/archives/libxslt-1.1.35.tar.xz +0 -0
  176. metadata +220 -266
  177. data/.autotest +0 -22
  178. data/.cross_rubies +0 -8
  179. data/.editorconfig +0 -17
  180. data/.gemtest +0 -0
  181. data/.travis.yml +0 -63
  182. data/CHANGELOG.md +0 -1368
  183. data/CONTRIBUTING.md +0 -42
  184. data/C_CODING_STYLE.rdoc +0 -33
  185. data/Gemfile-libxml-ruby +0 -3
  186. data/Manifest.txt +0 -370
  187. data/ROADMAP.md +0 -111
  188. data/Rakefile +0 -348
  189. data/SECURITY.md +0 -19
  190. data/STANDARD_RESPONSES.md +0 -47
  191. data/Y_U_NO_GEMSPEC.md +0 -155
  192. data/appveyor.yml +0 -29
  193. data/build_all +0 -44
  194. data/ext/nokogiri/html_document.c +0 -170
  195. data/ext/nokogiri/html_document.h +0 -10
  196. data/ext/nokogiri/html_element_description.c +0 -279
  197. data/ext/nokogiri/html_element_description.h +0 -10
  198. data/ext/nokogiri/html_entity_lookup.c +0 -32
  199. data/ext/nokogiri/html_entity_lookup.h +0 -8
  200. data/ext/nokogiri/html_sax_parser_context.c +0 -116
  201. data/ext/nokogiri/html_sax_parser_context.h +0 -11
  202. data/ext/nokogiri/html_sax_push_parser.c +0 -87
  203. data/ext/nokogiri/html_sax_push_parser.h +0 -9
  204. data/ext/nokogiri/xml_attr.h +0 -9
  205. data/ext/nokogiri/xml_attribute_decl.h +0 -9
  206. data/ext/nokogiri/xml_cdata.h +0 -9
  207. data/ext/nokogiri/xml_comment.h +0 -9
  208. data/ext/nokogiri/xml_document.h +0 -23
  209. data/ext/nokogiri/xml_document_fragment.h +0 -10
  210. data/ext/nokogiri/xml_dtd.h +0 -10
  211. data/ext/nokogiri/xml_element_content.h +0 -10
  212. data/ext/nokogiri/xml_element_decl.h +0 -9
  213. data/ext/nokogiri/xml_encoding_handler.h +0 -8
  214. data/ext/nokogiri/xml_entity_decl.h +0 -10
  215. data/ext/nokogiri/xml_entity_reference.h +0 -9
  216. data/ext/nokogiri/xml_io.c +0 -61
  217. data/ext/nokogiri/xml_io.h +0 -11
  218. data/ext/nokogiri/xml_libxml2_hacks.c +0 -112
  219. data/ext/nokogiri/xml_libxml2_hacks.h +0 -12
  220. data/ext/nokogiri/xml_namespace.h +0 -15
  221. data/ext/nokogiri/xml_node.h +0 -13
  222. data/ext/nokogiri/xml_node_set.h +0 -12
  223. data/ext/nokogiri/xml_processing_instruction.h +0 -9
  224. data/ext/nokogiri/xml_reader.h +0 -10
  225. data/ext/nokogiri/xml_relax_ng.h +0 -9
  226. data/ext/nokogiri/xml_sax_parser.h +0 -39
  227. data/ext/nokogiri/xml_sax_parser_context.h +0 -10
  228. data/ext/nokogiri/xml_sax_push_parser.h +0 -9
  229. data/ext/nokogiri/xml_schema.h +0 -9
  230. data/ext/nokogiri/xml_syntax_error.h +0 -13
  231. data/ext/nokogiri/xml_text.h +0 -9
  232. data/ext/nokogiri/xml_xpath_context.h +0 -10
  233. data/ext/nokogiri/xslt_stylesheet.h +0 -14
  234. data/lib/nokogiri/html/document_fragment.rb +0 -49
  235. data/lib/nokogiri/html/element_description_defaults.rb +0 -671
  236. data/lib/nokogiri/html/sax/parser_context.rb +0 -16
  237. data/patches/libxml2/0001-Revert-Do-not-URI-escape-in-server-side-includes.patch +0 -78
  238. data/patches/libxml2/0002-Fix-nullptr-deref-with-XPath-logic-ops.patch +0 -54
  239. data/patches/libxml2/0003-Fix-infinite-loop-in-LZMA-decompression.patch +0 -50
  240. data/patches/sort-patches-by-date +0 -25
  241. data/ports/archives/libxml2-2.9.8.tar.gz +0 -0
  242. data/ports/archives/libxslt-1.1.32.tar.gz +0 -0
  243. data/suppressions/README.txt +0 -1
  244. data/suppressions/nokogiri_ruby-2.supp +0 -10
  245. data/tasks/test.rb +0 -100
  246. data/test/css/test_nthiness.rb +0 -226
  247. data/test/css/test_parser.rb +0 -386
  248. data/test/css/test_tokenizer.rb +0 -215
  249. data/test/css/test_xpath_visitor.rb +0 -96
  250. data/test/decorators/test_slop.rb +0 -23
  251. data/test/files/2ch.html +0 -108
  252. data/test/files/GH_1042.html +0 -18
  253. data/test/files/address_book.rlx +0 -12
  254. data/test/files/address_book.xml +0 -10
  255. data/test/files/atom.xml +0 -344
  256. data/test/files/bar/bar.xsd +0 -4
  257. data/test/files/bogus.xml +0 -0
  258. data/test/files/dont_hurt_em_why.xml +0 -422
  259. data/test/files/encoding.html +0 -82
  260. data/test/files/encoding.xhtml +0 -84
  261. data/test/files/exslt.xml +0 -8
  262. data/test/files/exslt.xslt +0 -35
  263. data/test/files/foo/foo.xsd +0 -4
  264. data/test/files/metacharset.html +0 -10
  265. data/test/files/namespace_pressure_test.xml +0 -1684
  266. data/test/files/noencoding.html +0 -47
  267. data/test/files/po.xml +0 -32
  268. data/test/files/po.xsd +0 -66
  269. data/test/files/saml/saml20assertion_schema.xsd +0 -283
  270. data/test/files/saml/saml20protocol_schema.xsd +0 -302
  271. data/test/files/saml/xenc_schema.xsd +0 -146
  272. data/test/files/saml/xmldsig_schema.xsd +0 -318
  273. data/test/files/shift_jis.html +0 -10
  274. data/test/files/shift_jis.xml +0 -5
  275. data/test/files/shift_jis_no_charset.html +0 -9
  276. data/test/files/slow-xpath.xml +0 -25509
  277. data/test/files/snuggles.xml +0 -3
  278. data/test/files/staff.dtd +0 -10
  279. data/test/files/staff.xml +0 -59
  280. data/test/files/staff.xslt +0 -32
  281. data/test/files/test_document_url/bar.xml +0 -2
  282. data/test/files/test_document_url/document.dtd +0 -4
  283. data/test/files/test_document_url/document.xml +0 -6
  284. data/test/files/tlm.html +0 -851
  285. data/test/files/to_be_xincluded.xml +0 -2
  286. data/test/files/valid_bar.xml +0 -2
  287. data/test/files/xinclude.xml +0 -4
  288. data/test/helper.rb +0 -271
  289. data/test/html/sax/test_parser.rb +0 -168
  290. data/test/html/sax/test_parser_context.rb +0 -46
  291. data/test/html/sax/test_parser_text.rb +0 -163
  292. data/test/html/sax/test_push_parser.rb +0 -87
  293. data/test/html/test_attributes.rb +0 -85
  294. data/test/html/test_builder.rb +0 -164
  295. data/test/html/test_document.rb +0 -712
  296. data/test/html/test_document_encoding.rb +0 -143
  297. data/test/html/test_document_fragment.rb +0 -310
  298. data/test/html/test_element_description.rb +0 -105
  299. data/test/html/test_named_characters.rb +0 -14
  300. data/test/html/test_node.rb +0 -212
  301. data/test/html/test_node_encoding.rb +0 -91
  302. data/test/namespaces/test_additional_namespaces_in_builder_doc.rb +0 -14
  303. data/test/namespaces/test_namespaces_aliased_default.rb +0 -24
  304. data/test/namespaces/test_namespaces_in_builder_doc.rb +0 -75
  305. data/test/namespaces/test_namespaces_in_cloned_doc.rb +0 -31
  306. data/test/namespaces/test_namespaces_in_created_doc.rb +0 -75
  307. data/test/namespaces/test_namespaces_in_parsed_doc.rb +0 -80
  308. data/test/namespaces/test_namespaces_preservation.rb +0 -31
  309. data/test/test_convert_xpath.rb +0 -135
  310. data/test/test_css_cache.rb +0 -47
  311. data/test/test_encoding_handler.rb +0 -48
  312. data/test/test_memory_leak.rb +0 -156
  313. data/test/test_nokogiri.rb +0 -138
  314. data/test/test_soap4r_sax.rb +0 -52
  315. data/test/test_xslt_transforms.rb +0 -314
  316. data/test/xml/node/test_save_options.rb +0 -28
  317. data/test/xml/node/test_subclass.rb +0 -44
  318. data/test/xml/sax/test_parser.rb +0 -402
  319. data/test/xml/sax/test_parser_context.rb +0 -115
  320. data/test/xml/sax/test_parser_text.rb +0 -202
  321. data/test/xml/sax/test_push_parser.rb +0 -265
  322. data/test/xml/test_attr.rb +0 -74
  323. data/test/xml/test_attribute_decl.rb +0 -86
  324. data/test/xml/test_builder.rb +0 -341
  325. data/test/xml/test_c14n.rb +0 -180
  326. data/test/xml/test_cdata.rb +0 -54
  327. data/test/xml/test_comment.rb +0 -40
  328. data/test/xml/test_document.rb +0 -982
  329. data/test/xml/test_document_encoding.rb +0 -31
  330. data/test/xml/test_document_fragment.rb +0 -298
  331. data/test/xml/test_dtd.rb +0 -187
  332. data/test/xml/test_dtd_encoding.rb +0 -31
  333. data/test/xml/test_element_content.rb +0 -56
  334. data/test/xml/test_element_decl.rb +0 -73
  335. data/test/xml/test_entity_decl.rb +0 -122
  336. data/test/xml/test_entity_reference.rb +0 -262
  337. data/test/xml/test_namespace.rb +0 -96
  338. data/test/xml/test_node.rb +0 -1325
  339. data/test/xml/test_node_attributes.rb +0 -115
  340. data/test/xml/test_node_encoding.rb +0 -75
  341. data/test/xml/test_node_inheritance.rb +0 -32
  342. data/test/xml/test_node_reparenting.rb +0 -592
  343. data/test/xml/test_node_set.rb +0 -809
  344. data/test/xml/test_parse_options.rb +0 -64
  345. data/test/xml/test_processing_instruction.rb +0 -30
  346. data/test/xml/test_reader.rb +0 -620
  347. data/test/xml/test_reader_encoding.rb +0 -134
  348. data/test/xml/test_relax_ng.rb +0 -60
  349. data/test/xml/test_schema.rb +0 -142
  350. data/test/xml/test_syntax_error.rb +0 -36
  351. data/test/xml/test_text.rb +0 -60
  352. data/test/xml/test_unparented_node.rb +0 -483
  353. data/test/xml/test_xinclude.rb +0 -83
  354. data/test/xml/test_xpath.rb +0 -470
  355. data/test/xslt/test_custom_functions.rb +0 -133
  356. data/test/xslt/test_exception_handling.rb +0 -37
@@ -0,0 +1,478 @@
1
+ # coding: utf-8
2
+ # frozen_string_literal: true
3
+
4
+ #
5
+ # Copyright 2013-2021 Sam Ruby, Stephen Checkoway
6
+ #
7
+ # Licensed under the Apache License, Version 2.0 (the "License");
8
+ # you may not use this file except in compliance with the License.
9
+ # You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing, software
14
+ # distributed under the License is distributed on an "AS IS" BASIS,
15
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ # See the License for the specific language governing permissions and
17
+ # limitations under the License.
18
+ #
19
+
20
+ require_relative "html5/document"
21
+ require_relative "html5/document_fragment"
22
+ require_relative "html5/node"
23
+
24
+ module Nokogiri
25
+ # Since v1.12.0
26
+ #
27
+ # ⚠ HTML5 functionality is not available when running JRuby.
28
+ #
29
+ # Parse an HTML5 document. Convenience method for {Nokogiri::HTML5::Document.parse}
30
+ def self.HTML5(input, url = nil, encoding = nil, **options, &block)
31
+ Nokogiri::HTML5::Document.parse(input, url, encoding, **options, &block)
32
+ end
33
+
34
+ # == Usage
35
+ #
36
+ # ⚠ HTML5 functionality is not available when running JRuby.
37
+ #
38
+ # Parse an HTML5 document:
39
+ #
40
+ # doc = Nokogiri.HTML5(string)
41
+ #
42
+ # Parse an HTML5 fragment:
43
+ #
44
+ # fragment = Nokogiri::HTML5.fragment(string)
45
+ #
46
+ # == Parsing options
47
+ #
48
+ # The document and fragment parsing methods support options that are different from Nokogiri's.
49
+ #
50
+ # - <tt>Nokogiri.HTML5(html, url = nil, encoding = nil, options = {})</tt>
51
+ # - <tt>Nokogiri::HTML5.parse(html, url = nil, encoding = nil, options = {})</tt>
52
+ # - <tt>Nokogiri::HTML5::Document.parse(html, url = nil, encoding = nil, options = {})</tt>
53
+ # - <tt>Nokogiri::HTML5.fragment(html, encoding = nil, options = {})</tt>
54
+ # - <tt>Nokogiri::HTML5::DocumentFragment.parse(html, encoding = nil, options = {})</tt>
55
+ #
56
+ # The three currently supported options are +:max_errors+, +:max_tree_depth+ and
57
+ # +:max_attributes+, described below.
58
+ #
59
+ # === Error reporting
60
+ #
61
+ # Nokogiri contains an experimental HTML5 parse error reporting facility. By default, no parse
62
+ # errors are reported but this can be configured by passing the +:max_errors+ option to
63
+ # {HTML5.parse} or {HTML5.fragment}.
64
+ #
65
+ # For example, this script:
66
+ #
67
+ # doc = Nokogiri::HTML5.parse('<span/>Hi there!</span foo=bar />', max_errors: 10)
68
+ # doc.errors.each do |err|
69
+ # puts(err)
70
+ # end
71
+ #
72
+ # Emits:
73
+ #
74
+ # 1:1: ERROR: Expected a doctype token
75
+ # <span/>Hi there!</span foo=bar />
76
+ # ^
77
+ # 1:1: ERROR: Start tag of nonvoid HTML element ends with '/>', use '>'.
78
+ # <span/>Hi there!</span foo=bar />
79
+ # ^
80
+ # 1:17: ERROR: End tag ends with '/>', use '>'.
81
+ # <span/>Hi there!</span foo=bar />
82
+ # ^
83
+ # 1:17: ERROR: End tag contains attributes.
84
+ # <span/>Hi there!</span foo=bar />
85
+ # ^
86
+ #
87
+ # Using <tt>max_errors: -1</tt> results in an unlimited number of errors being returned.
88
+ #
89
+ # The errors returned by {HTML5::Document#errors} are instances of {Nokogiri::XML::SyntaxError}.
90
+ #
91
+ # The {https://html.spec.whatwg.org/multipage/parsing.html#parse-errors HTML standard} defines a
92
+ # number of standard parse error codes. These error codes only cover the "tokenization" stage of
93
+ # parsing HTML. The parse errors in the "tree construction" stage do not have standardized error
94
+ # codes (yet).
95
+ #
96
+ # As a convenience to Nokogiri users, the defined error codes are available via
97
+ # {Nokogiri::XML::SyntaxError#str1} method.
98
+ #
99
+ # doc = Nokogiri::HTML5.parse('<span/>Hi there!</span foo=bar />', max_errors: 10)
100
+ # doc.errors.each do |err|
101
+ # puts("#{err.line}:#{err.column}: #{err.str1}")
102
+ # end
103
+ # # => 1:1: generic-parser
104
+ # # 1:1: non-void-html-element-start-tag-with-trailing-solidus
105
+ # # 1:17: end-tag-with-trailing-solidus
106
+ # # 1:17: end-tag-with-attributes
107
+ #
108
+ # Note that the first error is +generic-parser+ because it's an error from the tree construction
109
+ # stage and doesn't have a standardized error code.
110
+ #
111
+ # For the purposes of semantic versioning, the error messages, error locations, and error codes
112
+ # are not part of Nokogiri's public API. That is, these are subject to change without Nokogiri's
113
+ # major version number changing. These may be stabilized in the future.
114
+ #
115
+ # === Maximum tree depth
116
+ #
117
+ # The maximum depth of the DOM tree parsed by the various parsing methods is configurable by the
118
+ # +:max_tree_depth+ option. If the depth of the tree would exceed this limit, then an
119
+ # {::ArgumentError} is thrown.
120
+ #
121
+ # This limit (which defaults to <tt>Nokogiri::Gumbo::DEFAULT_MAX_TREE_DEPTH = 400</tt>) can be
122
+ # removed by giving the option <tt>max_tree_depth: -1</tt>.
123
+ #
124
+ # html = '<!DOCTYPE html>' + '<div>' * 1000
125
+ # doc = Nokogiri.HTML5(html)
126
+ # # raises ArgumentError: Document tree depth limit exceeded
127
+ # doc = Nokogiri.HTML5(html, max_tree_depth: -1)
128
+ #
129
+ # === Attribute limit per element
130
+ #
131
+ # The maximum number of attributes per DOM element is configurable by the +:max_attributes+
132
+ # option. If a given element would exceed this limit, then an {::ArgumentError} is thrown.
133
+ #
134
+ # This limit (which defaults to <tt>Nokogiri::Gumbo::DEFAULT_MAX_ATTRIBUTES = 400</tt>) can be
135
+ # removed by giving the option <tt>max_attributes: -1</tt>.
136
+ #
137
+ # html = '<!DOCTYPE html><div ' + (1..1000).map { |x| "attr-#{x}" }.join(' ') + '>'
138
+ # # "<!DOCTYPE html><div attr-1 attr-2 attr-3 ... attr-1000>"
139
+ # doc = Nokogiri.HTML5(html)
140
+ # # raises ArgumentError: Attributes per element limit exceeded
141
+ # doc = Nokogiri.HTML5(html, max_attributes: -1)
142
+ #
143
+ # == HTML Serialization
144
+ #
145
+ # After parsing HTML, it may be serialized using any of the {Nokogiri::XML::Node} serialization
146
+ # methods. In particular, {XML::Node#serialize}, {XML::Node#to_html}, and {XML::Node#to_s} will
147
+ # serialize a given node and its children. (This is the equivalent of JavaScript's
148
+ # +Element.outerHTML+.) Similarly, {XML::Node#inner_html} will serialize the children of a given
149
+ # node. (This is the equivalent of JavaScript's +Element.innerHTML+.)
150
+ #
151
+ # doc = Nokogiri::HTML5("<!DOCTYPE html><span>Hello world!</span>")
152
+ # puts doc.serialize
153
+ # # => <!DOCTYPE html><html><head></head><body><span>Hello world!</span></body></html>
154
+ #
155
+ # Due to quirks in how HTML is parsed and serialized, it's possible for a DOM tree to be
156
+ # serialized and then re-parsed, resulting in a different DOM. Mostly, this happens with DOMs
157
+ # produced from invalid HTML. Unfortunately, even valid HTML may not survive serialization and
158
+ # re-parsing.
159
+ #
160
+ # In particular, a newline at the start of +pre+, +listing+, and +textarea+ elements is ignored by
161
+ # the parser.
162
+ #
163
+ # doc = Nokogiri::HTML5(<<-EOF)
164
+ # <!DOCTYPE html>
165
+ # <pre>
166
+ # Content</pre>
167
+ # EOF
168
+ # puts doc.at('/html/body/pre').serialize
169
+ # # => <pre>Content</pre>
170
+ #
171
+ # In this case, the original HTML is semantically equivalent to the serialized version. If the
172
+ # +pre+, +listing+, or +textarea+ content starts with two newlines, the first newline will be
173
+ # stripped on the first parse and the second newline will be stripped on the second, leading to
174
+ # semantically different DOMs. Passing the parameter <tt>preserve_newline: true</tt> will cause
175
+ # two or more newlines to be preserved. (A single leading newline will still be removed.)
176
+ #
177
+ # doc = Nokogiri::HTML5(<<-EOF)
178
+ # <!DOCTYPE html>
179
+ # <listing>
180
+ #
181
+ # Content</listing>
182
+ # EOF
183
+ # puts doc.at('/html/body/listing').serialize(preserve_newline: true)
184
+ # # => <listing>
185
+ # #
186
+ # # Content</listing>
187
+ #
188
+ # == Encodings
189
+ #
190
+ # Nokogiri always parses HTML5 using {https://en.wikipedia.org/wiki/UTF-8 UTF-8}; however, the
191
+ # encoding of the input can be explicitly selected via the optional +encoding+ parameter. This is
192
+ # most useful when the input comes not from a string but from an IO object.
193
+ #
194
+ # When serializing a document or node, the encoding of the output string can be specified via the
195
+ # +:encoding+ options. Characters that cannot be encoded in the selected encoding will be encoded
196
+ # as {https://en.wikipedia.org/wiki/List_of_XML_and_HTML_character_entity_references HTML numeric
197
+ # entities}.
198
+ #
199
+ # frag = Nokogiri::HTML5.fragment('<span>아는 길도 물어가라</span>')
200
+ # html = frag.serialize(encoding: 'US-ASCII')
201
+ # puts html
202
+ # # => <span>&#xc544;&#xb294; &#xae38;&#xb3c4; &#xbb3c;&#xc5b4;&#xac00;&#xb77c;</span>
203
+ # frag = Nokogiri::HTML5.fragment(html)
204
+ # puts frag.serialize
205
+ # # => <span>아는 길도 물어가라</span>
206
+ #
207
+ # (There's a {https://bugs.ruby-lang.org/issues/15033 bug} in all current versions of Ruby that
208
+ # can cause the entity encoding to fail. Of the mandated supported encodings for HTML, the only
209
+ # encoding I'm aware of that has this bug is <tt>'ISO-2022-JP'</tt>. We recommend avoiding this
210
+ # encoding.)
211
+ #
212
+ # == Notes
213
+ #
214
+ # * The {Nokogiri::HTML5.fragment} function takes a string and parses it
215
+ # as a HTML5 document. The +<html>+, +<head>+, and +<body>+ elements are
216
+ # removed from this document, and any children of these elements that remain
217
+ # are returned as a {Nokogiri::HTML5::DocumentFragment}.
218
+ #
219
+ # * The {Nokogiri::HTML5.parse} function takes a string and passes it to the
220
+ # <code>gumbo_parse_with_options</code> method, using the default options.
221
+ # The resulting Gumbo parse tree is then walked.
222
+ #
223
+ # * Instead of uppercase element names, lowercase element names are produced.
224
+ #
225
+ # * Instead of returning +unknown+ as the element name for unknown tags, the
226
+ # original tag name is returned verbatim.
227
+ #
228
+ # Since v1.12.0
229
+ module HTML5
230
+ # HTML uses the XHTML namespace.
231
+ HTML_NAMESPACE = "http://www.w3.org/1999/xhtml"
232
+ MATHML_NAMESPACE = "http://www.w3.org/1998/Math/MathML"
233
+ SVG_NAMESPACE = "http://www.w3.org/2000/svg"
234
+ XLINK_NAMESPACE = "http://www.w3.org/1999/xlink"
235
+ XML_NAMESPACE = "http://www.w3.org/XML/1998/namespace"
236
+ XMLNS_NAMESPACE = "http://www.w3.org/2000/xmlns/"
237
+
238
+ # Parse an HTML 5 document. Convenience method for {Nokogiri::HTML5::Document.parse}
239
+ def self.parse(string, url = nil, encoding = nil, **options, &block)
240
+ Document.parse(string, url, encoding, **options, &block)
241
+ end
242
+
243
+ # Parse a fragment from +string+. Convenience method for
244
+ # {Nokogiri::HTML5::DocumentFragment.parse}.
245
+ def self.fragment(string, encoding = nil, **options)
246
+ DocumentFragment.parse(string, encoding, options)
247
+ end
248
+
249
+ # Fetch and parse a HTML document from the web, following redirects,
250
+ # handling https, and determining the character encoding using HTML5
251
+ # rules. +uri+ may be a +String+ or a +URI+. +options+ contains
252
+ # http headers and special options. Everything which is not a
253
+ # special option is considered a header. Special options include:
254
+ # * :follow_limit => number of redirects which are followed
255
+ # * :basic_auth => [username, password]
256
+ def self.get(uri, options = {})
257
+ # TODO: deprecate
258
+ warn("Nokogiri::HTML5.get is deprecated and will be removed in a future version of Nokogiri.",
259
+ uplevel: 1, category: :deprecated)
260
+ get_impl(uri, options)
261
+ end
262
+
263
+ private
264
+
265
+ def self.get_impl(uri, options = {})
266
+ headers = options.clone
267
+ headers = { follow_limit: headers } if Numeric === headers # deprecated
268
+ limit = headers[:follow_limit] ? headers.delete(:follow_limit).to_i : 10
269
+
270
+ require "net/http"
271
+ uri = URI(uri) unless URI === uri
272
+
273
+ http = Net::HTTP.new(uri.host, uri.port)
274
+
275
+ # TLS / SSL support
276
+ http.use_ssl = true if uri.scheme == "https"
277
+
278
+ # Pass through Net::HTTP override values, which currently include:
279
+ # :ca_file, :ca_path, :cert, :cert_store, :ciphers,
280
+ # :close_on_empty_response, :continue_timeout, :key, :open_timeout,
281
+ # :read_timeout, :ssl_timeout, :ssl_version, :use_ssl,
282
+ # :verify_callback, :verify_depth, :verify_mode
283
+ options.each do |key, _value|
284
+ http.send("#{key}=", headers.delete(key)) if http.respond_to?("#{key}=")
285
+ end
286
+
287
+ request = Net::HTTP::Get.new(uri.request_uri)
288
+
289
+ # basic authentication
290
+ auth = headers.delete(:basic_auth)
291
+ auth ||= [uri.user, uri.password] if uri.user && uri.password
292
+ request.basic_auth(auth.first, auth.last) if auth
293
+
294
+ # remaining options are treated as headers
295
+ headers.each { |key, value| request[key.to_s] = value.to_s }
296
+
297
+ response = http.request(request)
298
+
299
+ case response
300
+ when Net::HTTPSuccess
301
+ doc = parse(reencode(response.body, response["content-type"]), options)
302
+ doc.instance_variable_set("@response", response)
303
+ doc.class.send(:attr_reader, :response)
304
+ doc
305
+ when Net::HTTPRedirection
306
+ response.value if limit <= 1
307
+ location = URI.join(uri, response["location"])
308
+ get_impl(location, options.merge(follow_limit: limit - 1))
309
+ else
310
+ response.value
311
+ end
312
+ end
313
+
314
+ def self.read_and_encode(string, encoding)
315
+ # Read the string with the given encoding.
316
+ if string.respond_to?(:read)
317
+ string = if encoding.nil?
318
+ string.read
319
+ else
320
+ string.read(encoding: encoding)
321
+ end
322
+ else
323
+ # Otherwise the string has the given encoding.
324
+ string = string.to_s
325
+ if encoding
326
+ string = string.dup
327
+ string.force_encoding(encoding)
328
+ end
329
+ end
330
+
331
+ # convert to UTF-8
332
+ if string.encoding != Encoding::UTF_8
333
+ string = reencode(string)
334
+ end
335
+ string
336
+ end
337
+
338
+ # Charset sniffing is a complex and controversial topic that understandably isn't done _by
339
+ # default_ by the Ruby Net::HTTP library. This being said, it is a very real problem for
340
+ # consumers of HTML as the default for HTML is iso-8859-1, most "good" producers use utf-8, and
341
+ # the Gumbo parser *only* supports utf-8.
342
+ #
343
+ # Accordingly, Nokogiri::HTML4::Document.parse provides limited encoding detection. Following
344
+ # this lead, Nokogiri::HTML5 attempts to do likewise, while attempting to more closely follow
345
+ # the HTML5 standard.
346
+ #
347
+ # http://bugs.ruby-lang.org/issues/2567
348
+ # http://www.w3.org/TR/html5/syntax.html#determining-the-character-encoding
349
+ #
350
+ def self.reencode(body, content_type = nil)
351
+ if body.encoding == Encoding::ASCII_8BIT
352
+ encoding = nil
353
+
354
+ # look for a Byte Order Mark (BOM)
355
+ initial_bytes = body[0..2].bytes
356
+ if initial_bytes[0..2] == [0xEF, 0xBB, 0xBF]
357
+ encoding = Encoding::UTF_8
358
+ elsif initial_bytes[0..1] == [0xFE, 0xFF]
359
+ encoding = Encoding::UTF_16BE
360
+ elsif initial_bytes[0..1] == [0xFF, 0xFE]
361
+ encoding = Encoding::UTF_16LE
362
+ end
363
+
364
+ # look for a charset in a content-encoding header
365
+ if content_type
366
+ encoding ||= content_type[/charset=["']?(.*?)($|["';\s])/i, 1]
367
+ end
368
+
369
+ # look for a charset in a meta tag in the first 1024 bytes
370
+ unless encoding
371
+ data = body[0..1023].gsub(/<!--.*?(-->|\Z)/m, "")
372
+ data.scan(/<meta.*?>/m).each do |meta|
373
+ encoding ||= meta[/charset=["']?([^>]*?)($|["'\s>])/im, 1]
374
+ end
375
+ end
376
+
377
+ # if all else fails, default to the official default encoding for HTML
378
+ encoding ||= Encoding::ISO_8859_1
379
+
380
+ # change the encoding to match the detected or inferred encoding
381
+ body = body.dup
382
+ begin
383
+ body.force_encoding(encoding)
384
+ rescue ArgumentError
385
+ body.force_encoding(Encoding::ISO_8859_1)
386
+ end
387
+ end
388
+
389
+ body.encode(Encoding::UTF_8)
390
+ end
391
+
392
+ def self.serialize_node_internal(current_node, io, encoding, options)
393
+ case current_node.type
394
+ when XML::Node::ELEMENT_NODE
395
+ ns = current_node.namespace
396
+ ns_uri = ns.nil? ? nil : ns.href
397
+ # XXX(sfc): attach namespaces to all nodes, even html?
398
+ tagname = if ns_uri.nil? || ns_uri == HTML_NAMESPACE || ns_uri == MATHML_NAMESPACE || ns_uri == SVG_NAMESPACE
399
+ current_node.name
400
+ else
401
+ "#{ns.prefix}:#{current_node.name}"
402
+ end
403
+ io << "<" << tagname
404
+ current_node.attribute_nodes.each do |attr|
405
+ attr_ns = attr.namespace
406
+ if attr_ns.nil?
407
+ attr_name = attr.name
408
+ else
409
+ ns_uri = attr_ns.href
410
+ attr_name = if ns_uri == XML_NAMESPACE
411
+ "xml:" + attr.name.sub(/^[^:]*:/, "")
412
+ elsif ns_uri == XMLNS_NAMESPACE && attr.name.sub(/^[^:]*:/, "") == "xmlns"
413
+ "xmlns"
414
+ elsif ns_uri == XMLNS_NAMESPACE
415
+ "xmlns:" + attr.name.sub(/^[^:]*:/, "")
416
+ elsif ns_uri == XLINK_NAMESPACE
417
+ "xlink:" + attr.name.sub(/^[^:]*:/, "")
418
+ else
419
+ "#{attr_ns.prefix}:#{attr.name}"
420
+ end
421
+ end
422
+ io << " " << attr_name << '="' << escape_text(attr.content, encoding, true) << '"'
423
+ end
424
+ io << ">"
425
+ unless ["area", "base", "basefont", "bgsound", "br", "col", "embed", "frame", "hr", "img", "input", "keygen", "link", "meta", "param", "source", "track", "wbr"].include?(current_node.name)
426
+ io << "\n" if options[:preserve_newline] && prepend_newline?(current_node)
427
+ current_node.children.each do |child|
428
+ # XXX(sfc): Templates handled specially?
429
+ serialize_node_internal(child, io, encoding, options)
430
+ end
431
+ io << "</" << tagname << ">"
432
+ end
433
+ when XML::Node::TEXT_NODE
434
+ parent = current_node.parent
435
+ io << if parent.element? && ["style", "script", "xmp", "iframe", "noembed", "noframes", "plaintext", "noscript"].include?(parent.name)
436
+ current_node.content
437
+ else
438
+ escape_text(current_node.content, encoding, false)
439
+ end
440
+ when XML::Node::CDATA_SECTION_NODE
441
+ io << "<![CDATA[" << current_node.content << "]]>"
442
+ when XML::Node::COMMENT_NODE
443
+ io << "<!--" << current_node.content << "-->"
444
+ when XML::Node::PI_NODE
445
+ io << "<?" << current_node.content << ">"
446
+ when XML::Node::DOCUMENT_TYPE_NODE, XML::Node::DTD_NODE
447
+ io << "<!DOCTYPE " << current_node.name << ">"
448
+ when XML::Node::HTML_DOCUMENT_NODE, XML::Node::DOCUMENT_FRAG_NODE
449
+ current_node.children.each do |child|
450
+ serialize_node_internal(child, io, encoding, options)
451
+ end
452
+ else
453
+ raise "Unexpected node '#{current_node.name}' of type #{current_node.type}"
454
+ end
455
+ end
456
+
457
+ def self.escape_text(text, encoding, attribute_mode)
458
+ text = if attribute_mode
459
+ text.gsub(/[&\u00a0"]/,
460
+ "&" => "&amp;", "\u00a0" => "&nbsp;", '"' => "&quot;")
461
+ else
462
+ text.gsub(/[&\u00a0<>]/,
463
+ "&" => "&amp;", "\u00a0" => "&nbsp;", "<" => "&lt;", ">" => "&gt;")
464
+ end
465
+ # Not part of the standard
466
+ text.encode(encoding, fallback: lambda { |c| "&\#x#{c.ord.to_s(16)};" })
467
+ end
468
+
469
+ def self.prepend_newline?(node)
470
+ return false unless ["pre", "textarea", "listing"].include?(node.name) && !node.children.empty?
471
+
472
+ first_child = node.children[0]
473
+ first_child.text? && first_child.content.start_with?("\n")
474
+ end
475
+ end
476
+ end
477
+
478
+ require_relative "gumbo"
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ # The line below caused a problem on non-GAE rack environment.
4
+ # unless defined?(JRuby::Rack::VERSION) || defined?(AppEngine::ApiProxy)
5
+ #
6
+ # However, simply cutting defined?(JRuby::Rack::VERSION) off resulted in
7
+ # an unable-to-load-nokogiri problem. Thus, now, Nokogiri checks the presense
8
+ # of appengine-rack.jar in $LOAD_PATH. If Nokogiri is on GAE, Nokogiri
9
+ # should skip loading xml jars. This is because those are in WEB-INF/lib and
10
+ # already set in the classpath.
11
+ unless $LOAD_PATH.to_s.include?("appengine-rack")
12
+ require "stringio"
13
+ require "isorelax.jar"
14
+ require "jing.jar"
15
+ require "nekohtml.jar"
16
+ require "nekodtd.jar"
17
+ require "xercesImpl.jar"
18
+ require "serializer.jar"
19
+ require "xalan.jar"
20
+ require "xml-apis.jar"
21
+ end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Nokogiri
2
4
  class SyntaxError < ::StandardError
3
5
  end
@@ -0,0 +1,6 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Nokogiri
4
+ # The version of Nokogiri you are using
5
+ VERSION = "1.13.6"
6
+ end