nokogumbo 1.5.0 → 2.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +237 -26
  3. data/ext/nokogumbo/extconf.rb +144 -0
  4. data/ext/nokogumbo/nokogumbo.c +793 -0
  5. data/gumbo-parser/src/ascii.c +75 -0
  6. data/gumbo-parser/src/ascii.h +115 -0
  7. data/gumbo-parser/src/attribute.c +26 -28
  8. data/gumbo-parser/src/attribute.h +3 -23
  9. data/gumbo-parser/src/char_ref.c +5972 -6816
  10. data/gumbo-parser/src/char_ref.h +14 -45
  11. data/gumbo-parser/src/error.c +510 -163
  12. data/gumbo-parser/src/error.h +70 -147
  13. data/gumbo-parser/src/foreign_attrs.c +104 -0
  14. data/gumbo-parser/src/gumbo.h +577 -305
  15. data/gumbo-parser/src/insertion_mode.h +4 -28
  16. data/gumbo-parser/src/macros.h +91 -0
  17. data/gumbo-parser/src/parser.c +2922 -2228
  18. data/gumbo-parser/src/parser.h +6 -22
  19. data/gumbo-parser/src/replacement.h +33 -0
  20. data/gumbo-parser/src/string_buffer.c +43 -50
  21. data/gumbo-parser/src/string_buffer.h +24 -40
  22. data/gumbo-parser/src/string_piece.c +39 -39
  23. data/gumbo-parser/src/svg_attrs.c +174 -0
  24. data/gumbo-parser/src/svg_tags.c +137 -0
  25. data/gumbo-parser/src/tag.c +186 -59
  26. data/gumbo-parser/src/tag_lookup.c +382 -0
  27. data/gumbo-parser/src/tag_lookup.h +13 -0
  28. data/gumbo-parser/src/token_buffer.c +79 -0
  29. data/gumbo-parser/src/token_buffer.h +71 -0
  30. data/gumbo-parser/src/token_type.h +1 -25
  31. data/gumbo-parser/src/tokenizer.c +2128 -1562
  32. data/gumbo-parser/src/tokenizer.h +41 -52
  33. data/gumbo-parser/src/tokenizer_states.h +281 -45
  34. data/gumbo-parser/src/utf8.c +98 -123
  35. data/gumbo-parser/src/utf8.h +84 -52
  36. data/gumbo-parser/src/util.c +48 -38
  37. data/gumbo-parser/src/util.h +10 -40
  38. data/gumbo-parser/src/vector.c +45 -57
  39. data/gumbo-parser/src/vector.h +17 -39
  40. data/lib/nokogumbo.rb +18 -170
  41. data/lib/nokogumbo/html5.rb +252 -0
  42. data/lib/nokogumbo/html5/document.rb +53 -0
  43. data/lib/nokogumbo/html5/document_fragment.rb +62 -0
  44. data/lib/nokogumbo/html5/node.rb +72 -0
  45. data/lib/nokogumbo/version.rb +3 -0
  46. metadata +40 -21
  47. data/ext/nokogumboc/extconf.rb +0 -60
  48. data/ext/nokogumboc/nokogumbo.c +0 -295
  49. data/gumbo-parser/src/char_ref.rl +0 -2554
  50. data/gumbo-parser/src/string_piece.h +0 -38
  51. data/gumbo-parser/src/tag.in +0 -150
  52. data/gumbo-parser/src/tag_enum.h +0 -153
  53. data/gumbo-parser/src/tag_gperf.h +0 -105
  54. data/gumbo-parser/src/tag_sizes.h +0 -4
  55. data/gumbo-parser/src/tag_strings.h +0 -153
  56. data/gumbo-parser/visualc/include/strings.h +0 -4
  57. data/test-nokogumbo.rb +0 -190
@@ -0,0 +1,793 @@
1
+ //
2
+ // nokogumbo.c defines the following:
3
+ //
4
+ // class Nokogumbo
5
+ // def parse(utf8_string) # returns Nokogiri::HTML5::Document
6
+ // end
7
+ //
8
+ // Processing starts by calling gumbo_parse_with_options. The resulting
9
+ // document tree is then walked:
10
+ //
11
+ // * if Nokogiri and libxml2 headers are available at compile time,
12
+ // (if NGLIB) then a parallel libxml2 tree is constructed, and the
13
+ // final document is then wrapped using Nokogiri_wrap_xml_document.
14
+ // This approach reduces memory and CPU requirements as Ruby objects
15
+ // are only built when necessary.
16
+ //
17
+ // * if the necessary headers are not available at compile time, Nokogiri
18
+ // methods are called instead, producing the equivalent functionality.
19
+ //
20
+
21
+ #include <assert.h>
22
+ #include <ruby.h>
23
+ #include <ruby/version.h>
24
+
25
+ #include "gumbo.h"
26
+
27
+ // class constants
28
+ static VALUE Document;
29
+
30
+ // Interned symbols
31
+ static ID internal_subset;
32
+ static ID parent;
33
+
34
+ /* Backwards compatibility to Ruby 2.1.0 */
35
+ #if RUBY_API_VERSION_CODE < 20200
36
+ #define ONIG_ESCAPE_UCHAR_COLLISION 1
37
+ #include <ruby/encoding.h>
38
+
39
+ static VALUE rb_utf8_str_new(const char *str, long length) {
40
+ return rb_enc_str_new(str, length, rb_utf8_encoding());
41
+ }
42
+
43
+ static VALUE rb_utf8_str_new_cstr(const char *str) {
44
+ return rb_enc_str_new_cstr(str, rb_utf8_encoding());
45
+ }
46
+
47
+ static VALUE rb_utf8_str_new_static(const char *str, long length) {
48
+ return rb_enc_str_new(str, length, rb_utf8_encoding());
49
+ }
50
+ #endif
51
+
52
+ #if NGLIB
53
+ #include <nokogiri.h>
54
+ #include <libxml/tree.h>
55
+ #include <libxml/HTMLtree.h>
56
+
57
+ #define NIL NULL
58
+ #else
59
+ #define NIL Qnil
60
+
61
+ // These are defined by nokogiri.h
62
+ static VALUE cNokogiriXmlSyntaxError;
63
+ static VALUE cNokogiriXmlElement;
64
+ static VALUE cNokogiriXmlText;
65
+ static VALUE cNokogiriXmlCData;
66
+ static VALUE cNokogiriXmlComment;
67
+
68
+ // Interned symbols.
69
+ static ID new;
70
+ static ID node_name_;
71
+
72
+ // Map libxml2 types to Ruby VALUE.
73
+ typedef VALUE xmlNodePtr;
74
+ typedef VALUE xmlDocPtr;
75
+ typedef VALUE xmlNsPtr;
76
+ typedef VALUE xmlDtdPtr;
77
+ typedef char xmlChar;
78
+ #define BAD_CAST
79
+
80
+ // Redefine libxml2 API as Ruby function calls.
81
+ static xmlNodePtr xmlNewDocNode(xmlDocPtr doc, xmlNsPtr ns, const xmlChar *name, const xmlChar *content) {
82
+ assert(ns == NIL && content == NULL);
83
+ return rb_funcall(cNokogiriXmlElement, new, 2, rb_utf8_str_new_cstr(name), doc);
84
+ }
85
+
86
+ static xmlNodePtr xmlNewDocText(xmlDocPtr doc, const xmlChar *content) {
87
+ VALUE str = rb_utf8_str_new_cstr(content);
88
+ return rb_funcall(cNokogiriXmlText, new, 2, str, doc);
89
+ }
90
+
91
+ static xmlNodePtr xmlNewCDataBlock(xmlDocPtr doc, const xmlChar *content, int len) {
92
+ VALUE str = rb_utf8_str_new(content, len);
93
+ // CDATA.new takes arguments in the opposite order from Text.new.
94
+ return rb_funcall(cNokogiriXmlCData, new, 2, doc, str);
95
+ }
96
+
97
+ static xmlNodePtr xmlNewDocComment(xmlDocPtr doc, const xmlChar *content) {
98
+ VALUE str = rb_utf8_str_new_cstr(content);
99
+ return rb_funcall(cNokogiriXmlComment, new, 2, doc, str);
100
+ }
101
+
102
+ static xmlNodePtr xmlAddChild(xmlNodePtr parent, xmlNodePtr cur) {
103
+ ID add_child;
104
+ CONST_ID(add_child, "add_child");
105
+ return rb_funcall(parent, add_child, 1, cur);
106
+ }
107
+
108
+ static void xmlSetNs(xmlNodePtr node, xmlNsPtr ns) {
109
+ ID namespace_;
110
+ CONST_ID(namespace_, "namespace=");
111
+ rb_funcall(node, namespace_, 1, ns);
112
+ }
113
+
114
+ static void xmlFreeDoc(xmlDocPtr doc) { }
115
+
116
+ static VALUE Nokogiri_wrap_xml_document(VALUE klass, xmlDocPtr doc) {
117
+ return doc;
118
+ }
119
+
120
+ static VALUE find_dummy_key(VALUE collection) {
121
+ VALUE r_dummy = Qnil;
122
+ char dummy[5] = "a";
123
+ size_t len = 1;
124
+ ID key_;
125
+ CONST_ID(key_, "key?");
126
+ while (len < sizeof dummy) {
127
+ r_dummy = rb_utf8_str_new(dummy, len);
128
+ if (rb_funcall(collection, key_, 1, r_dummy) == Qfalse)
129
+ return r_dummy;
130
+ for (size_t i = 0; ; ++i) {
131
+ if (dummy[i] == 0) {
132
+ dummy[i] = 'a';
133
+ ++len;
134
+ break;
135
+ }
136
+ if (dummy[i] == 'z')
137
+ dummy[i] = 'a';
138
+ else {
139
+ ++dummy[i];
140
+ break;
141
+ }
142
+ }
143
+ }
144
+ // This collection has 475254 elements?? Give up.
145
+ rb_raise(rb_eArgError, "Failed to find a dummy key.");
146
+ }
147
+
148
+ // This should return an xmlAttrPtr, but we don't need it and it's easier to
149
+ // not get the result.
150
+ static void xmlNewNsProp (
151
+ xmlNodePtr node,
152
+ xmlNsPtr ns,
153
+ const xmlChar *name,
154
+ const xmlChar *value
155
+ ) {
156
+ ID set_attribute;
157
+ CONST_ID(set_attribute, "set_attribute");
158
+
159
+ VALUE rvalue = rb_utf8_str_new_cstr(value);
160
+
161
+ if (RTEST(ns)) {
162
+ // This is an easy case, we have a namespace so it's enough to do
163
+ // node["#{ns.prefix}:#{name}"] = value
164
+ ID prefix;
165
+ CONST_ID(prefix, "prefix");
166
+ VALUE ns_prefix = rb_funcall(ns, prefix, 0);
167
+ VALUE qname = rb_sprintf("%" PRIsVALUE ":%s", ns_prefix, name);
168
+ rb_funcall(node, set_attribute, 2, qname, rvalue);
169
+ return;
170
+ }
171
+
172
+ size_t len = strlen(name);
173
+ VALUE rname = rb_utf8_str_new(name, len);
174
+ if (memchr(name, ':', len) == NULL) {
175
+ // This is the easiest case. There's no colon so we can do
176
+ // node[name] = value.
177
+ rb_funcall(node, set_attribute, 2, rname, rvalue);
178
+ return;
179
+ }
180
+
181
+ // Nokogiri::XML::Node#set_attribute calls xmlSetProp(node, name, value)
182
+ // which behaves roughly as
183
+ // if name is a QName prefix:local
184
+ // if node->doc has a namespace ns corresponding to prefix
185
+ // return xmlSetNsProp(node, ns, local, value)
186
+ // return xmlSetNsProp(node, NULL, name, value)
187
+ //
188
+ // If the prefix is "xml", then the namespace lookup will create it.
189
+ //
190
+ // By contrast, xmlNewNsProp does not do this parsing and creates an attribute
191
+ // with the name and value exactly as given. This is the behavior that we
192
+ // want.
193
+ //
194
+ // Thus, for attribute names like "xml:lang", #set_attribute will create an
195
+ // attribute with namespace "xml" and name "lang". This is incorrect for
196
+ // html elements (but correct for foreign elements).
197
+ //
198
+ // Work around this by inserting a dummy attribute and then changing the
199
+ // name, if needed.
200
+
201
+ // Find a dummy attribute string that doesn't already exist.
202
+ VALUE dummy = find_dummy_key(node);
203
+ // Add the dummy attribute.
204
+ rb_funcall(node, set_attribute, 2, dummy, rvalue);
205
+
206
+ // Remove the old attribute, if it exists.
207
+ ID remove_attribute;
208
+ CONST_ID(remove_attribute, "remove_attribute");
209
+ rb_funcall(node, remove_attribute, 1, rname);
210
+
211
+ // Rename the dummy
212
+ ID attribute;
213
+ CONST_ID(attribute, "attribute");
214
+ VALUE attr = rb_funcall(node, attribute, 1, dummy);
215
+ rb_funcall(attr, node_name_, 1, rname);
216
+ }
217
+ #endif
218
+
219
+ // URI = system id
220
+ // external id = public id
221
+ static xmlDocPtr new_html_doc(const char *dtd_name, const char *system, const char *public)
222
+ {
223
+ #if NGLIB
224
+ // These two libxml2 functions take the public and system ids in
225
+ // opposite orders.
226
+ htmlDocPtr doc = htmlNewDocNoDtD(/* URI */ NULL, /* ExternalID */NULL);
227
+ assert(doc);
228
+ if (dtd_name)
229
+ xmlCreateIntSubset(doc, BAD_CAST dtd_name, BAD_CAST public, BAD_CAST system);
230
+ return doc;
231
+ #else
232
+ // remove internal subset from newly created documents
233
+ VALUE doc;
234
+ // If system and public are both NULL, Document#new is going to set default
235
+ // values for them so we're going to have to remove the internal subset
236
+ // which seems to leak memory in Nokogiri, so leak as little as possible.
237
+ if (system == NULL && public == NULL) {
238
+ ID remove;
239
+ CONST_ID(remove, "remove");
240
+ doc = rb_funcall(Document, new, 2, /* URI */ Qnil, /* external_id */ rb_utf8_str_new_static("", 0));
241
+ rb_funcall(rb_funcall(doc, internal_subset, 0), remove, 0);
242
+ if (dtd_name) {
243
+ // We need to create an internal subset now.
244
+ ID create_internal_subset;
245
+ CONST_ID(create_internal_subset, "create_internal_subset");
246
+ rb_funcall(doc, create_internal_subset, 3, rb_utf8_str_new_cstr(dtd_name), Qnil, Qnil);
247
+ }
248
+ } else {
249
+ assert(dtd_name);
250
+ // Rather than removing and creating the internal subset as we did above,
251
+ // just create and then rename one.
252
+ VALUE r_system = system ? rb_utf8_str_new_cstr(system) : Qnil;
253
+ VALUE r_public = public ? rb_utf8_str_new_cstr(public) : Qnil;
254
+ doc = rb_funcall(Document, new, 2, r_system, r_public);
255
+ rb_funcall(rb_funcall(doc, internal_subset, 0), node_name_, 1, rb_utf8_str_new_cstr(dtd_name));
256
+ }
257
+ return doc;
258
+ #endif
259
+ }
260
+
261
+ static xmlNodePtr get_parent(xmlNodePtr node) {
262
+ #if NGLIB
263
+ return node->parent;
264
+ #else
265
+ if (!rb_respond_to(node, parent))
266
+ return Qnil;
267
+ return rb_funcall(node, parent, 0);
268
+ #endif
269
+ }
270
+
271
+ static GumboOutput *perform_parse(const GumboOptions *options, VALUE input) {
272
+ assert(RTEST(input));
273
+ Check_Type(input, T_STRING);
274
+ GumboOutput *output = gumbo_parse_with_options (
275
+ options,
276
+ RSTRING_PTR(input),
277
+ RSTRING_LEN(input)
278
+ );
279
+
280
+ const char *status_string = gumbo_status_to_string(output->status);
281
+ switch (output->status) {
282
+ case GUMBO_STATUS_OK:
283
+ break;
284
+ case GUMBO_STATUS_TOO_MANY_ATTRIBUTES:
285
+ case GUMBO_STATUS_TREE_TOO_DEEP:
286
+ gumbo_destroy_output(output);
287
+ rb_raise(rb_eArgError, "%s", status_string);
288
+ case GUMBO_STATUS_OUT_OF_MEMORY:
289
+ gumbo_destroy_output(output);
290
+ rb_raise(rb_eNoMemError, "%s", status_string);
291
+ }
292
+ return output;
293
+ }
294
+
295
+ static xmlNsPtr lookup_or_add_ns (
296
+ xmlDocPtr doc,
297
+ xmlNodePtr root,
298
+ const char *href,
299
+ const char *prefix
300
+ ) {
301
+ #if NGLIB
302
+ xmlNsPtr ns = xmlSearchNs(doc, root, BAD_CAST prefix);
303
+ if (ns)
304
+ return ns;
305
+ return xmlNewNs(root, BAD_CAST href, BAD_CAST prefix);
306
+ #else
307
+ ID add_namespace_definition;
308
+ CONST_ID(add_namespace_definition, "add_namespace_definition");
309
+ VALUE rprefix = rb_utf8_str_new_cstr(prefix);
310
+ VALUE rhref = rb_utf8_str_new_cstr(href);
311
+ return rb_funcall(root, add_namespace_definition, 2, rprefix, rhref);
312
+ #endif
313
+ }
314
+
315
+ static void set_line(xmlNodePtr node, size_t line) {
316
+ #if NGLIB
317
+ // libxml2 uses 65535 to mean look elsewhere for the line number on some
318
+ // nodes.
319
+ if (line < 65535)
320
+ node->line = (unsigned short)line;
321
+ #else
322
+ // XXX: If Nokogiri gets a `#line=` method, we'll use that.
323
+ #endif
324
+ }
325
+
326
+ // Construct an XML tree rooted at xml_output_node from the Gumbo tree rooted
327
+ // at gumbo_node.
328
+ static void build_tree (
329
+ xmlDocPtr doc,
330
+ xmlNodePtr xml_output_node,
331
+ const GumboNode *gumbo_node
332
+ ) {
333
+ xmlNodePtr xml_root = NIL;
334
+ xmlNodePtr xml_node = xml_output_node;
335
+ size_t child_index = 0;
336
+
337
+ while (true) {
338
+ assert(gumbo_node != NULL);
339
+ const GumboVector *children = gumbo_node->type == GUMBO_NODE_DOCUMENT?
340
+ &gumbo_node->v.document.children : &gumbo_node->v.element.children;
341
+ if (child_index >= children->length) {
342
+ // Move up the tree and to the next child.
343
+ if (xml_node == xml_output_node) {
344
+ // We've built as much of the tree as we can.
345
+ return;
346
+ }
347
+ child_index = gumbo_node->index_within_parent + 1;
348
+ gumbo_node = gumbo_node->parent;
349
+ xml_node = get_parent(xml_node);
350
+ // Children of fragments don't share the same root, so reset it and
351
+ // it'll be set below. In the non-fragment case, this will only happen
352
+ // after the html element has been finished at which point there are no
353
+ // further elements.
354
+ if (xml_node == xml_output_node)
355
+ xml_root = NIL;
356
+ continue;
357
+ }
358
+ const GumboNode *gumbo_child = children->data[child_index++];
359
+ xmlNodePtr xml_child;
360
+
361
+ switch (gumbo_child->type) {
362
+ case GUMBO_NODE_DOCUMENT:
363
+ abort(); // Bug in Gumbo.
364
+
365
+ case GUMBO_NODE_TEXT:
366
+ case GUMBO_NODE_WHITESPACE:
367
+ xml_child = xmlNewDocText(doc, BAD_CAST gumbo_child->v.text.text);
368
+ set_line(xml_child, gumbo_child->v.text.start_pos.line);
369
+ xmlAddChild(xml_node, xml_child);
370
+ break;
371
+
372
+ case GUMBO_NODE_CDATA:
373
+ xml_child = xmlNewCDataBlock(doc, BAD_CAST gumbo_child->v.text.text,
374
+ (int) strlen(gumbo_child->v.text.text));
375
+ set_line(xml_child, gumbo_child->v.text.start_pos.line);
376
+ xmlAddChild(xml_node, xml_child);
377
+ break;
378
+
379
+ case GUMBO_NODE_COMMENT:
380
+ xml_child = xmlNewDocComment(doc, BAD_CAST gumbo_child->v.text.text);
381
+ set_line(xml_child, gumbo_child->v.text.start_pos.line);
382
+ xmlAddChild(xml_node, xml_child);
383
+ break;
384
+
385
+ case GUMBO_NODE_TEMPLATE:
386
+ // XXX: Should create a template element and a new DocumentFragment
387
+ case GUMBO_NODE_ELEMENT:
388
+ {
389
+ xml_child = xmlNewDocNode(doc, NIL, BAD_CAST gumbo_child->v.element.name, NULL);
390
+ set_line(xml_child, gumbo_child->v.element.start_pos.line);
391
+ if (xml_root == NIL)
392
+ xml_root = xml_child;
393
+ xmlNsPtr ns = NIL;
394
+ switch (gumbo_child->v.element.tag_namespace) {
395
+ case GUMBO_NAMESPACE_HTML:
396
+ break;
397
+ case GUMBO_NAMESPACE_SVG:
398
+ ns = lookup_or_add_ns(doc, xml_root, "http://www.w3.org/2000/svg", "svg");
399
+ break;
400
+ case GUMBO_NAMESPACE_MATHML:
401
+ ns = lookup_or_add_ns(doc, xml_root, "http://www.w3.org/1998/Math/MathML", "math");
402
+ break;
403
+ }
404
+ if (ns != NIL)
405
+ xmlSetNs(xml_child, ns);
406
+ xmlAddChild(xml_node, xml_child);
407
+
408
+ // Add the attributes.
409
+ const GumboVector* attrs = &gumbo_child->v.element.attributes;
410
+ for (size_t i=0; i < attrs->length; i++) {
411
+ const GumboAttribute *attr = attrs->data[i];
412
+
413
+ switch (attr->attr_namespace) {
414
+ case GUMBO_ATTR_NAMESPACE_XLINK:
415
+ ns = lookup_or_add_ns(doc, xml_root, "http://www.w3.org/1999/xlink", "xlink");
416
+ break;
417
+
418
+ case GUMBO_ATTR_NAMESPACE_XML:
419
+ ns = lookup_or_add_ns(doc, xml_root, "http://www.w3.org/XML/1998/namespace", "xml");
420
+ break;
421
+
422
+ case GUMBO_ATTR_NAMESPACE_XMLNS:
423
+ ns = lookup_or_add_ns(doc, xml_root, "http://www.w3.org/2000/xmlns/", "xmlns");
424
+ break;
425
+
426
+ default:
427
+ ns = NIL;
428
+ }
429
+ xmlNewNsProp(xml_child, ns, BAD_CAST attr->name, BAD_CAST attr->value);
430
+ }
431
+
432
+ // Add children for this element.
433
+ child_index = 0;
434
+ gumbo_node = gumbo_child;
435
+ xml_node = xml_child;
436
+ }
437
+ }
438
+ }
439
+ }
440
+
441
+ static void add_errors(const GumboOutput *output, VALUE rdoc, VALUE input, VALUE url) {
442
+ const char *input_str = RSTRING_PTR(input);
443
+ size_t input_len = RSTRING_LEN(input);
444
+
445
+ // Add parse errors to rdoc.
446
+ if (output->errors.length) {
447
+ const GumboVector *errors = &output->errors;
448
+ VALUE rerrors = rb_ary_new2(errors->length);
449
+
450
+ for (size_t i=0; i < errors->length; i++) {
451
+ GumboError *err = errors->data[i];
452
+ GumboSourcePosition position = gumbo_error_position(err);
453
+ char *msg;
454
+ size_t size = gumbo_caret_diagnostic_to_string(err, input_str, input_len, &msg);
455
+ VALUE err_str = rb_utf8_str_new(msg, size);
456
+ free(msg);
457
+ VALUE syntax_error = rb_class_new_instance(1, &err_str, cNokogiriXmlSyntaxError);
458
+ const char *error_code = gumbo_error_code(err);
459
+ VALUE str1 = error_code? rb_utf8_str_new_static(error_code, strlen(error_code)) : Qnil;
460
+ rb_iv_set(syntax_error, "@domain", INT2NUM(1)); // XML_FROM_PARSER
461
+ rb_iv_set(syntax_error, "@code", INT2NUM(1)); // XML_ERR_INTERNAL_ERROR
462
+ rb_iv_set(syntax_error, "@level", INT2NUM(2)); // XML_ERR_ERROR
463
+ rb_iv_set(syntax_error, "@file", url);
464
+ rb_iv_set(syntax_error, "@line", INT2NUM(position.line));
465
+ rb_iv_set(syntax_error, "@str1", str1);
466
+ rb_iv_set(syntax_error, "@str2", Qnil);
467
+ rb_iv_set(syntax_error, "@str3", Qnil);
468
+ rb_iv_set(syntax_error, "@int1", INT2NUM(0));
469
+ rb_iv_set(syntax_error, "@column", INT2NUM(position.column));
470
+ rb_ary_push(rerrors, syntax_error);
471
+ }
472
+ rb_iv_set(rdoc, "@errors", rerrors);
473
+ }
474
+ }
475
+
476
+ typedef struct {
477
+ GumboOutput *output;
478
+ VALUE input;
479
+ VALUE url_or_frag;
480
+ xmlDocPtr doc;
481
+ } ParseArgs;
482
+
483
+ static void parse_args_mark(void *parse_args) {
484
+ ParseArgs *args = parse_args;
485
+ rb_gc_mark_maybe(args->input);
486
+ rb_gc_mark_maybe(args->url_or_frag);
487
+ }
488
+
489
+ // Wrap a ParseArgs pointer. The underlying ParseArgs must outlive the
490
+ // wrapper.
491
+ static VALUE wrap_parse_args(ParseArgs *args) {
492
+ return Data_Wrap_Struct(rb_cData, parse_args_mark, RUBY_NEVER_FREE, args);
493
+ }
494
+
495
+ // Returnsd the underlying ParseArgs wrapped by wrap_parse_args.
496
+ static ParseArgs *unwrap_parse_args(VALUE obj) {
497
+ ParseArgs *args;
498
+ Data_Get_Struct(obj, ParseArgs, args);
499
+ return args;
500
+ }
501
+
502
+ static VALUE parse_cleanup(VALUE parse_args) {
503
+ ParseArgs *args = unwrap_parse_args(parse_args);
504
+ gumbo_destroy_output(args->output);
505
+ // Make sure garbage collection doesn't mark the objects as being live based
506
+ // on references from the ParseArgs. This may be unnecessary.
507
+ args->input = Qnil;
508
+ args->url_or_frag = Qnil;
509
+ if (args->doc != NIL)
510
+ xmlFreeDoc(args->doc);
511
+ return Qnil;
512
+ }
513
+
514
+ static VALUE parse_continue(VALUE parse_args);
515
+
516
+ // Parse a string using gumbo_parse into a Nokogiri document
517
+ static VALUE parse(VALUE self, VALUE input, VALUE url, VALUE max_attributes, VALUE max_errors, VALUE max_depth) {
518
+ GumboOptions options = kGumboDefaultOptions;
519
+ options.max_attributes = NUM2INT(max_attributes);
520
+ options.max_errors = NUM2INT(max_errors);
521
+ options.max_tree_depth = NUM2INT(max_depth);
522
+
523
+ GumboOutput *output = perform_parse(&options, input);
524
+ ParseArgs args = {
525
+ .output = output,
526
+ .input = input,
527
+ .url_or_frag = url,
528
+ .doc = NIL,
529
+ };
530
+ VALUE parse_args = wrap_parse_args(&args);
531
+
532
+ return rb_ensure(parse_continue, parse_args, parse_cleanup, parse_args);
533
+ }
534
+
535
+ static VALUE parse_continue(VALUE parse_args) {
536
+ ParseArgs *args = unwrap_parse_args(parse_args);
537
+ GumboOutput *output = args->output;
538
+ xmlDocPtr doc;
539
+ if (output->document->v.document.has_doctype) {
540
+ const char *name = output->document->v.document.name;
541
+ const char *public = output->document->v.document.public_identifier;
542
+ const char *system = output->document->v.document.system_identifier;
543
+ public = public[0] ? public : NULL;
544
+ system = system[0] ? system : NULL;
545
+ doc = new_html_doc(name, system, public);
546
+ } else {
547
+ doc = new_html_doc(NULL, NULL, NULL);
548
+ }
549
+ args->doc = doc; // Make sure doc gets cleaned up if an error is thrown.
550
+ build_tree(doc, (xmlNodePtr)doc, output->document);
551
+ VALUE rdoc = Nokogiri_wrap_xml_document(Document, doc);
552
+ args->doc = NIL; // The Ruby runtime now owns doc so don't delete it.
553
+ add_errors(output, rdoc, args->input, args->url_or_frag);
554
+ return rdoc;
555
+ }
556
+
557
+ static int lookup_namespace(VALUE node, bool require_known_ns) {
558
+ ID namespace, href;
559
+ CONST_ID(namespace, "namespace");
560
+ CONST_ID(href, "href");
561
+ VALUE ns = rb_funcall(node, namespace, 0);
562
+
563
+ if (NIL_P(ns))
564
+ return GUMBO_NAMESPACE_HTML;
565
+ ns = rb_funcall(ns, href, 0);
566
+ assert(RTEST(ns));
567
+ Check_Type(ns, T_STRING);
568
+
569
+ const char *href_ptr = RSTRING_PTR(ns);
570
+ size_t href_len = RSTRING_LEN(ns);
571
+ #define NAMESPACE_P(uri) (href_len == sizeof uri - 1 && !memcmp(href_ptr, uri, href_len))
572
+ if (NAMESPACE_P("http://www.w3.org/1999/xhtml"))
573
+ return GUMBO_NAMESPACE_HTML;
574
+ if (NAMESPACE_P("http://www.w3.org/1998/Math/MathML"))
575
+ return GUMBO_NAMESPACE_MATHML;
576
+ if (NAMESPACE_P("http://www.w3.org/2000/svg"))
577
+ return GUMBO_NAMESPACE_SVG;
578
+ #undef NAMESPACE_P
579
+ if (require_known_ns)
580
+ rb_raise(rb_eArgError, "Unexpected namespace URI \"%*s\"", (int)href_len, href_ptr);
581
+ return -1;
582
+ }
583
+
584
+ static xmlNodePtr extract_xml_node(VALUE node) {
585
+ #if NGLIB
586
+ xmlNodePtr xml_node;
587
+ Data_Get_Struct(node, xmlNode, xml_node);
588
+ return xml_node;
589
+ #else
590
+ return node;
591
+ #endif
592
+ }
593
+
594
+ static VALUE fragment_continue(VALUE parse_args);
595
+
596
+ static VALUE fragment (
597
+ VALUE self,
598
+ VALUE doc_fragment,
599
+ VALUE tags,
600
+ VALUE ctx,
601
+ VALUE max_attributes,
602
+ VALUE max_errors,
603
+ VALUE max_depth
604
+ ) {
605
+ ID name = rb_intern_const("name");
606
+ const char *ctx_tag;
607
+ GumboNamespaceEnum ctx_ns;
608
+ GumboQuirksModeEnum quirks_mode;
609
+ bool form = false;
610
+ const char *encoding = NULL;
611
+
612
+ if (NIL_P(ctx)) {
613
+ ctx_tag = "body";
614
+ ctx_ns = GUMBO_NAMESPACE_HTML;
615
+ } else if (TYPE(ctx) == T_STRING) {
616
+ ctx_tag = StringValueCStr(ctx);
617
+ ctx_ns = GUMBO_NAMESPACE_HTML;
618
+ size_t len = RSTRING_LEN(ctx);
619
+ const char *colon = memchr(ctx_tag, ':', len);
620
+ if (colon) {
621
+ switch (colon - ctx_tag) {
622
+ case 3:
623
+ if (st_strncasecmp(ctx_tag, "svg", 3) != 0)
624
+ goto error;
625
+ ctx_ns = GUMBO_NAMESPACE_SVG;
626
+ break;
627
+ case 4:
628
+ if (st_strncasecmp(ctx_tag, "html", 4) == 0)
629
+ ctx_ns = GUMBO_NAMESPACE_HTML;
630
+ else if (st_strncasecmp(ctx_tag, "math", 4) == 0)
631
+ ctx_ns = GUMBO_NAMESPACE_MATHML;
632
+ else
633
+ goto error;
634
+ break;
635
+ default:
636
+ error:
637
+ rb_raise(rb_eArgError, "Invalid context namespace '%*s'", (int)(colon - ctx_tag), ctx_tag);
638
+ }
639
+ ctx_tag = colon+1;
640
+ } else {
641
+ // For convenience, put 'svg' and 'math' in their namespaces.
642
+ if (len == 3 && st_strncasecmp(ctx_tag, "svg", 3) == 0)
643
+ ctx_ns = GUMBO_NAMESPACE_SVG;
644
+ else if (len == 4 && st_strncasecmp(ctx_tag, "math", 4) == 0)
645
+ ctx_ns = GUMBO_NAMESPACE_MATHML;
646
+ }
647
+
648
+ // Check if it's a form.
649
+ form = ctx_ns == GUMBO_NAMESPACE_HTML && st_strcasecmp(ctx_tag, "form") == 0;
650
+ } else {
651
+ ID element_ = rb_intern_const("element?");
652
+
653
+ // Context fragment name.
654
+ VALUE tag_name = rb_funcall(ctx, name, 0);
655
+ assert(RTEST(tag_name));
656
+ Check_Type(tag_name, T_STRING);
657
+ ctx_tag = StringValueCStr(tag_name);
658
+
659
+ // Context fragment namespace.
660
+ ctx_ns = lookup_namespace(ctx, true);
661
+
662
+ // Check for a form ancestor, including self.
663
+ for (VALUE node = ctx;
664
+ !NIL_P(node);
665
+ node = rb_respond_to(node, parent) ? rb_funcall(node, parent, 0) : Qnil) {
666
+ if (!RTEST(rb_funcall(node, element_, 0)))
667
+ continue;
668
+ VALUE element_name = rb_funcall(node, name, 0);
669
+ if (RSTRING_LEN(element_name) == 4
670
+ && !st_strcasecmp(RSTRING_PTR(element_name), "form")
671
+ && lookup_namespace(node, false) == GUMBO_NAMESPACE_HTML) {
672
+ form = true;
673
+ break;
674
+ }
675
+ }
676
+
677
+ // Encoding.
678
+ if (RSTRING_LEN(tag_name) == 14
679
+ && !st_strcasecmp(ctx_tag, "annotation-xml")) {
680
+ VALUE enc = rb_funcall(ctx, rb_intern_const("[]"),
681
+ rb_utf8_str_new_static("encoding", 8));
682
+ if (RTEST(enc)) {
683
+ Check_Type(enc, T_STRING);
684
+ encoding = StringValueCStr(enc);
685
+ }
686
+ }
687
+ }
688
+
689
+ // Quirks mode.
690
+ VALUE doc = rb_funcall(doc_fragment, rb_intern_const("document"), 0);
691
+ VALUE dtd = rb_funcall(doc, internal_subset, 0);
692
+ if (NIL_P(dtd)) {
693
+ quirks_mode = GUMBO_DOCTYPE_NO_QUIRKS;
694
+ } else {
695
+ VALUE dtd_name = rb_funcall(dtd, name, 0);
696
+ VALUE pubid = rb_funcall(dtd, rb_intern_const("external_id"), 0);
697
+ VALUE sysid = rb_funcall(dtd, rb_intern_const("system_id"), 0);
698
+ quirks_mode = gumbo_compute_quirks_mode (
699
+ NIL_P(dtd_name)? NULL:StringValueCStr(dtd_name),
700
+ NIL_P(pubid)? NULL:StringValueCStr(pubid),
701
+ NIL_P(sysid)? NULL:StringValueCStr(sysid)
702
+ );
703
+ }
704
+
705
+ // Perform a fragment parse.
706
+ int depth = NUM2INT(max_depth);
707
+ GumboOptions options = kGumboDefaultOptions;
708
+ options.max_attributes = NUM2INT(max_attributes);
709
+ options.max_errors = NUM2INT(max_errors);
710
+ // Add one to account for the HTML element.
711
+ options.max_tree_depth = depth < 0 ? -1 : (depth + 1);
712
+ options.fragment_context = ctx_tag;
713
+ options.fragment_namespace = ctx_ns;
714
+ options.fragment_encoding = encoding;
715
+ options.quirks_mode = quirks_mode;
716
+ options.fragment_context_has_form_ancestor = form;
717
+
718
+ GumboOutput *output = perform_parse(&options, tags);
719
+ ParseArgs args = {
720
+ .output = output,
721
+ .input = tags,
722
+ .url_or_frag = doc_fragment,
723
+ .doc = (xmlDocPtr)extract_xml_node(doc),
724
+ };
725
+ VALUE parse_args = wrap_parse_args(&args);
726
+ rb_ensure(fragment_continue, parse_args, parse_cleanup, parse_args);
727
+ return Qnil;
728
+ }
729
+
730
+ static VALUE fragment_continue(VALUE parse_args) {
731
+ ParseArgs *args = unwrap_parse_args(parse_args);
732
+ GumboOutput *output = args->output;
733
+ VALUE doc_fragment = args->url_or_frag;
734
+ xmlDocPtr xml_doc = args->doc;
735
+
736
+ args->doc = NIL; // The Ruby runtime owns doc so make sure we don't delete it.
737
+ xmlNodePtr xml_frag = extract_xml_node(doc_fragment);
738
+ build_tree(xml_doc, xml_frag, output->root);
739
+ add_errors(output, doc_fragment, args->input, rb_utf8_str_new_static("#fragment", 9));
740
+ return Qnil;
741
+ }
742
+
743
+ // Initialize the Nokogumbo class and fetch constants we will use later.
744
+ void Init_nokogumbo() {
745
+ rb_funcall(rb_mKernel, rb_intern_const("gem"), 1, rb_utf8_str_new_static("nokogiri", 8));
746
+ rb_require("nokogiri");
747
+
748
+ VALUE line_supported = Qtrue;
749
+
750
+ #if !NGLIB
751
+ // Class constants.
752
+ VALUE mNokogiri = rb_const_get(rb_cObject, rb_intern_const("Nokogiri"));
753
+ VALUE mNokogiriXml = rb_const_get(mNokogiri, rb_intern_const("XML"));
754
+ cNokogiriXmlSyntaxError = rb_const_get(mNokogiriXml, rb_intern_const("SyntaxError"));
755
+ rb_gc_register_mark_object(cNokogiriXmlSyntaxError);
756
+ cNokogiriXmlElement = rb_const_get(mNokogiriXml, rb_intern_const("Element"));
757
+ rb_gc_register_mark_object(cNokogiriXmlElement);
758
+ cNokogiriXmlText = rb_const_get(mNokogiriXml, rb_intern_const("Text"));
759
+ rb_gc_register_mark_object(cNokogiriXmlText);
760
+ cNokogiriXmlCData = rb_const_get(mNokogiriXml, rb_intern_const("CDATA"));
761
+ rb_gc_register_mark_object(cNokogiriXmlCData);
762
+ cNokogiriXmlComment = rb_const_get(mNokogiriXml, rb_intern_const("Comment"));
763
+ rb_gc_register_mark_object(cNokogiriXmlComment);
764
+
765
+ // Interned symbols.
766
+ new = rb_intern_const("new");
767
+ node_name_ = rb_intern_const("node_name=");
768
+
769
+ // #line is not supported (returns 0)
770
+ line_supported = Qfalse;
771
+ #endif
772
+
773
+ // Class constants.
774
+ VALUE HTML5 = rb_const_get(mNokogiri, rb_intern_const("HTML5"));
775
+ Document = rb_const_get(HTML5, rb_intern_const("Document"));
776
+ rb_gc_register_mark_object(Document);
777
+
778
+ // Interned symbols.
779
+ internal_subset = rb_intern_const("internal_subset");
780
+ parent = rb_intern_const("parent");
781
+
782
+ // Define Nokogumbo module with parse and fragment methods.
783
+ VALUE Gumbo = rb_define_module("Nokogumbo");
784
+ rb_define_singleton_method(Gumbo, "parse", parse, 5);
785
+ rb_define_singleton_method(Gumbo, "fragment", fragment, 6);
786
+
787
+ // Add private constant for testing.
788
+ rb_define_const(Gumbo, "LINE_SUPPORTED", line_supported);
789
+ rb_funcall(Gumbo, rb_intern_const("private_constant"), 1,
790
+ rb_utf8_str_new_cstr("LINE_SUPPORTED"));
791
+ }
792
+
793
+ // vim: set shiftwidth=2 softtabstop=2 tabstop=8 expandtab: