nokogumbo 0.1

data/README.md

@@ -0,0 +1,46 @@
+Nokogumbo - a Nokogiri interface to the Gumbo HTML5 parser.
+===========
+
+At the moment, this is a proof of concept, allowing a Ruby program to invoke
+the Gumbo HTML5 parser and access the result as a Nokogiri parsed document.
+
+Usage:
+-----
+
+```ruby
+require 'nokogumbo'
+doc = Nokogiri::HTML5(string)
+```
+
+Notes:
+-----
+
+* The `Nokogumbo.parse` function takes a string and passes it to the
+<code>gumbo_parse_with_options</code> method, using the default options.
+The resulting Gumbo parse tree is the walked, producing a Nokogiri parse tree.
+The original Gumbo parse tree is then destroyed, and the Nokogiri parse tree
+is returned. 
+
+* Instead of uppercase element names, lowercase element names are produced.
+
+* Instead of returning 'unknown' as the element name for unknown tags, the
+original tag name is returned verbatim.
+
+* Nothing meaningful is done with the `GumboDocument` struct, i.e., no
+Nokogiri `EntityDecl` is produced.
+
+Installation:
+============
+
+* Build and install the 
+[gumbo-parser](https://github.com/google/gumbo-parser#readme) C library
+
+* Execute `rake gem`
+
+* [sudo] gem install pkg/nokogumbo*.gem
+
+Related efforts:
+============
+
+* [ruby-gumbo](https://github.com/galdor/ruby-gumbo#readme) - a ruby binding
+for the Gumbo HTML5 parser.

data/Rakefile

@@ -0,0 +1,50 @@
+require 'rubygems/package_task'
+require 'rake/clean'
+
+task 'default' => 'test'
+
+file 'Makefile' => 'ext/extconf.rb' do
+  Dir.chdir 'ext' do 
+    ruby 'extconf.rb'
+  end
+end
+
+task 'test' => 'Makefile' do
+  Dir.chdir 'ext' do 
+    sh 'make -s'
+  end
+  ruby 'test-nokogumbo.rb'
+end
+
+CLEAN.include('ext/*.o', 'ext/*.so', 'ext/*.log', 'ext/Makefile', 'pkg')
+
+MANIFEST = %w(
+  ext/extconf.rb  
+  ext/nokogumbo.c  
+  lib/nokogumbo.rb  
+  Rakefile  
+  README.md
+)
+
+SPEC = Gem::Specification.new do |gem|
+  gem.name = 'nokogumbo'
+  gem.version = '0.1'
+  gem.email = 'rubys@intertwingly.net'
+  gem.homepage = 'https://github.com/rubys/nokogumbo/tree/master/ruby#readme'
+  gem.summary = 'Nokogiri interface to the Gumbo HTML5 parser'
+  gem.files = MANIFEST
+  gem.extensions = 'ext/extconf.rb'
+  gem.author = 'Sam Ruby'
+  gem.add_dependency 'nokogiri'
+  gem.license = 'MIT'
+  gem.description = %q(
+    At the moment, this is a proof of concept, allowing a Ruby
+    program to invoke the Gumbo HTML5 parser and access the result as a Nokogiri
+    parsed document.).strip.gsub(/\s+/, ' ')
+end
+
+task 'gem' => 'test'
+Gem::PackageTask.new(SPEC) do |pkg|
+  pkg.need_tar = true
+  pkg.need_zip = true
+end

data/ext/extconf.rb

@@ -0,0 +1,3 @@
+require 'mkmf'
+have_library('gumbo', 'gumbo_parse')
+create_makefile('nokogumboc')

data/ext/nokogumbo.c

@@ -0,0 +1,131 @@
+#include "ruby.h"
+#include "gumbo.h"
+
+// class constants
+static VALUE Nokogiri;
+static VALUE HTML;
+static VALUE XML;
+static VALUE Document;
+static VALUE Element;
+static VALUE Text;
+static VALUE CDATA;
+static VALUE Comment;
+static VALUE TAGS=0;
+static int Unknown=0;
+
+// interned symbols
+static VALUE new;
+static VALUE set_attribute;
+static VALUE add_child;
+
+// determine tag name for a given node
+static VALUE _name(GumboElement *node) {
+  if (!TAGS) {
+    // Deferred initialization of "Unknown" as the GumboParser class is
+    // defined *after* the Nokogumbo class is.
+    VALUE HTML5 = rb_const_get(Nokogiri, rb_intern("HTML5"));
+    TAGS = rb_const_get(HTML5, rb_intern("TAGS"));
+    Unknown = NUM2INT(rb_const_get(HTML5, rb_intern("Unknown")));
+  }
+
+  if (node->tag != Unknown) {
+    return rb_ary_entry(TAGS, (long) node->tag);
+  } else {
+    // Gumbo doesn't provide unknown tags, so we need to parse it ourselves:
+    // http://www.w3.org/html/wg/drafts/html/CR/syntax.html#tag-name-state
+    GumboStringPiece *tag = &node->original_tag;
+    int length;
+    for (length = 1; length < tag->length-1; length++) {
+      if (strchr(" \t\r\n<", *((char*)tag->data+length))) break; 
+    }
+    return rb_str_new(1+(char *)tag->data, length-1);
+  }
+}
+
+// Build a Nokogiri Element for a given GumboElement (recursively)
+static VALUE _element(VALUE document, GumboElement *node) {
+  int i;
+  VALUE element = rb_funcall(Element, new, 2, _name(node), document);
+
+  // add in the attributes
+  GumboVector* attrs = &node->attributes;
+  for (i=0; i < attrs->length; i++) {
+    GumboAttribute *attr = attrs->data[i];
+    VALUE name = rb_str_new2(attr->name);
+    rb_funcall(element, set_attribute, 2, name, rb_str_new2(attr->value));
+  }
+
+  // add in the children
+  GumboVector* children = &node->children;
+  for (i=0; i < children->length; i++) {
+    GumboNode* child = children->data[i];
+
+    VALUE node = 0;
+    VALUE text;
+
+    switch (child->type) {
+      case GUMBO_NODE_ELEMENT:
+        node = _element(document, &child->v.element);
+        break;
+      case GUMBO_NODE_WHITESPACE:
+      case GUMBO_NODE_TEXT:
+        text = rb_str_new2(child->v.text.text);
+        node = rb_funcall(Text, new, 2, text, document);
+        break;
+      case GUMBO_NODE_CDATA:
+        text = rb_str_new2(child->v.text.text);
+        node = rb_funcall(CDATA, new, 2, text, document);
+        break;
+      case GUMBO_NODE_COMMENT:
+        text = rb_str_new2(child->v.text.text);
+        node = rb_funcall(Comment, new, 2, document, text);
+        break;
+      case GUMBO_NODE_DOCUMENT:
+        break; // should never happen -- ignore
+    }
+
+    if (node) rb_funcall(element, add_child, 1, node);
+  }
+
+  return element;
+}
+
+// Parse a string using gumbo_parse into a Nokogiri document
+static VALUE t_parse(VALUE self, VALUE string) {
+  VALUE document = rb_funcall(Document, new, 0);
+
+  GumboOutput *output = gumbo_parse_with_options(
+    &kGumboDefaultOptions, RSTRING_PTR(string), RSTRING_LEN(string)
+  );
+  VALUE root = _element(document, (GumboElement*)&output->root->v.element);
+  rb_funcall(document, add_child, 1, root);
+  gumbo_destroy_output(&kGumboDefaultOptions, output);
+
+  return document;
+}
+
+// Initialize the Nokogumbo class and fetch constants we will use later
+void Init_nokogumboc() {
+  rb_funcall(rb_mKernel, rb_intern("gem"), 1, rb_str_new2("nokogiri"));
+  rb_require("nokogiri");
+
+  // class constants
+  Nokogiri = rb_const_get(rb_cObject, rb_intern("Nokogiri"));
+  HTML = rb_const_get(Nokogiri, rb_intern("HTML"));
+  XML = rb_const_get(Nokogiri, rb_intern("XML"));
+  Document = rb_const_get(HTML, rb_intern("Document"));
+  Element = rb_const_get(XML, rb_intern("Element"));
+  Text = rb_const_get(XML, rb_intern("Text"));
+  CDATA = rb_const_get(XML, rb_intern("CDATA"));
+  Comment = rb_const_get(XML, rb_intern("Comment"));
+
+  // interned symbols
+  new = rb_intern("new");
+  set_attribute = rb_intern("set_attribute");
+  add_child = rb_intern("add_child");
+
+  // define Nokogumbo class with a singleton parse method
+  VALUE Gumbo = rb_define_class("Nokogumbo", rb_cObject);
+  rb_define_singleton_method(Gumbo, "parse", t_parse, 1);
+}
+

data/lib/nokogumbo.rb

@@ -0,0 +1,166 @@
+require 'nokogiri'
+require 'nokogumboc'
+
+module Nokogiri
+  def self.HTML5(string)
+    Nokogumbo.parse(string)
+  end
+
+  module HTML5
+    TAGS = [
+      'HTML',
+      'HEAD',
+      'TITLE',
+      'BASE',
+      'LINK',
+      'META',
+      'STYLE',
+      'SCRIPT',
+      'NOSCRIPT',
+      'BODY',
+      'SECTION',
+      'NAV',
+      'ARTICLE',
+      'ASIDE',
+      'H1',
+      'H2',
+      'H3',
+      'H4',
+      'H5',
+      'H6',
+      'HGROUP',
+      'HEADER',
+      'FOOTER',
+      'ADDRESS',
+      'P',
+      'HR',
+      'PRE',
+      'BLOCKQUOTE',
+      'OL',
+      'UL',
+      'LI',
+      'DL',
+      'DT',
+      'DD',
+      'FIGURE',
+      'FIGCAPTION',
+      'DIV',
+      'A',
+      'EM',
+      'STRONG',
+      'SMALL',
+      'S',
+      'CITE',
+      'Q',
+      'DFN',
+      'ABBR',
+      'TIME',
+      'CODE',
+      'VAR',
+      'SAMP',
+      'KBD',
+      'SUB',
+      'SUP',
+      'I',
+      'B',
+      'MARK',
+      'RUBY',
+      'RT',
+      'RP',
+      'BDI',
+      'BDO',
+      'SPAN',
+      'BR',
+      'WBR',
+      'INS',
+      'DEL',
+      'IMAGE',
+      'IMG',
+      'IFRAME',
+      'EMBED',
+      'OBJECT',
+      'PARAM',
+      'VIDEO',
+      'AUDIO',
+      'SOURCE',
+      'TRACK',
+      'CANVAS',
+      'MAP',
+      'AREA',
+      'MATH',
+      'MI',
+      'MO',
+      'MN',
+      'MS',
+      'MTEXT',
+      'MGLYPH',
+      'MALIGNMARK',
+      'ANNOTATION_XML',
+      'SVG',
+      'FOREIGNOBJECT',
+      'DESC',
+      'TABLE',
+      'CAPTION',
+      'COLGROUP',
+      'COL',
+      'TBODY',
+      'THEAD',
+      'TFOOT',
+      'TR',
+      'TD',
+      'TH',
+      'FORM',
+      'FIELDSET',
+      'LEGEND',
+      'LABEL',
+      'INPUT',
+      'BUTTON',
+      'SELECT',
+      'DATALIST',
+      'OPTGROUP',
+      'OPTION',
+      'TEXTAREA',
+      'KEYGEN',
+      'OUTPUT',
+      'PROGRESS',
+      'METER',
+      'DETAILS',
+      'SUMMARY',
+      'COMMAND',
+      'MENU',
+      'APPLET',
+      'ACRONYM',
+      'BGSOUND',
+      'DIR',
+      'FRAME',
+      'FRAMESET',
+      'NOFRAMES',
+      'ISINDEX',
+      'LISTING',
+      'XMP',
+      'NEXTID',
+      'NOEMBED',
+      'PLAINTEXT',
+      'RB',
+      'STRIKE',
+      'BASEFONT',
+      'BIG',
+      'BLINK',
+      'CENTER',
+      'FONT',
+      'MARQUEE',
+      'MULTICOL',
+      'NOBR',
+      'SPACER',
+      'TT',
+      'U',
+      'UNKNOWN',
+    ].map(&:downcase).map(&:freeze).freeze
+
+    Unknown = TAGS.length - 1
+
+    def parse(string)
+      Nokogumbo.parse(string)
+    end
+  end
+end

metadata

@@ -0,0 +1,68 @@
+--- !ruby/object:Gem::Specification
+name: nokogumbo
+version: !ruby/object:Gem::Version
+  version: '0.1'
+  prerelease: 
+platform: ruby
+authors:
+- Sam Ruby
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-08-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: At the moment, this is a proof of concept, allowing a Ruby program to
+  invoke the Gumbo HTML5 parser and access the result as a Nokogiri parsed document.
+email: rubys@intertwingly.net
+executables: []
+extensions:
+- ext/extconf.rb
+extra_rdoc_files: []
+files:
+- ext/extconf.rb
+- ext/nokogumbo.c
+- lib/nokogumbo.rb
+- Rakefile
+- README.md
+homepage: https://github.com/rubys/nokogumbo/tree/master/ruby#readme
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Nokogiri interface to the Gumbo HTML5 parser
+test_files: []