html_cruncher 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6b52f018b73d713860ade94ef48d23806ec5e053
+  data.tar.gz: 5d1ddd2d216635668dfed1c2adf19c20cd5c6770
+SHA512:
+  metadata.gz: dea287e7ab8d8e911e90adcabf58148863c72dd9469f206e8d17e9f1f2d4e62b1c5e90df8e0ccab17473f5ea9c53f31a5836ea3f6b2798a5a00999be68ca9651
+  data.tar.gz: 3bc59bce9a64600f1f7c32331d61441a6d6c046033101bcabdb1e2756bdeada4cd07717273d8116b5ea0cb4146388a382ea64c54440ac27d7d545289dd2ff974

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in html_parser.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Mathias Biilmann Christensen
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,31 @@
+# HtmlCruncher
+
+Crunches through HTML
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'html_cruncher'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install html_cruncher
+
+## Usage
+
+See the tests for usage.
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/html_parser/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,12 @@
+require "bundler/gem_tasks"
+require "rake/extensiontask"
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.pattern = "test/*_test.rb"
+end
+
+Rake::ExtensionTask.new "html_parser" do |ext|
+  ext.lib_dir = "lib/html_cruncher"
+end

data/ext/html_parser/extconf.rb

@@ -0,0 +1,9 @@
+require 'mkmf'
+
+extension_name = 'html_parser'
+
+have_library("re2")
+have_header("re2.h")
+
+dir_config(extension_name)       # The destination
+create_makefile(extension_name)  # Create Makefile

data/ext/html_parser/html_parser.cc

@@ -0,0 +1,199 @@
+#include "ruby.h"
+#include <re2/re2.h>
+#include <iostream>
+#include <string>
+#include <new>
+
+using std::nothrow;
+using re2::StringPiece;
+
+static VALUE HtmlParser = Qnil;
+static RE2 *WHITESPACE_RE;
+static RE2 *START_TAG_RE;
+static RE2 *ATTRIBUTES_RE;
+static RE2 *END_TAG_RE;
+static StringPiece COMMENT_TAG;
+static StringPiece COMMENT_TAG_END;
+static StringPiece SELF_CLOSER;
+
+static void parse_attributes(VALUE attrs, StringPiece data) {
+  long offset = 0;
+  long length = data.size();
+  int ret, n_groups = 0, pos = 0;
+  StringPiece *groups = NULL;
+  StringPiece value;
+
+  while (offset < length) {
+    // Match Whitespace
+    n_groups = WHITESPACE_RE->NumberOfCapturingGroups() + 1;
+    groups = new(nothrow) StringPiece[n_groups];
+    ret = WHITESPACE_RE->Match(data, offset, length, RE2::ANCHOR_START, groups, n_groups);
+    if (ret == 1) {
+      offset += groups[0].size();
+    }
+    delete[] groups;
+
+    n_groups = ATTRIBUTES_RE->NumberOfCapturingGroups() + 1;
+    groups = new(nothrow) StringPiece[n_groups];
+    ret = ATTRIBUTES_RE->Match(
+      data,
+      offset,
+      length,
+      RE2::ANCHOR_START,
+      groups,
+      n_groups
+    );
+    if (ret == 1) {
+      if (groups[2].size()) {
+        value = groups[2];
+      } else if (groups[3].size()) {
+        value = groups[3];
+      } else if (groups[4].size()) {
+        value = groups[4];
+      } else {
+        value = StringPiece("");
+      }
+      rb_hash_aset(
+        attrs,
+        rb_str_new(groups[1].data(), groups[1].size()),
+        rb_str_new(value.data(), value.size())
+      );
+      offset += groups[0].size();
+      delete[] groups;
+    } else {
+      delete[] groups;
+      return;
+    }
+  }
+}
+
+extern "C" VALUE
+parse_html(VALUE self, VALUE string, VALUE handler)
+{
+  long offset = 0;
+  long length = RSTRING_LEN(string);
+  int tag_match = 0;
+  int ret, n_groups = 0, pos = 0;
+  StringPiece html = StringPiece(RSTRING_PTR(string), length);
+	StringPiece *groups = NULL;
+
+  ID m_text       = rb_intern("text");
+  ID m_whitespace = rb_intern("whitespace");
+  ID m_start_tag  = rb_intern("start_tag");
+  ID m_end_tag    = rb_intern("end_tag");
+  ID m_done       = rb_intern("done");
+
+  while (offset < length) {
+    tag_match = 0;
+    ret = 0;
+    n_groups = WHITESPACE_RE->NumberOfCapturingGroups() + 1;
+    groups = new(nothrow) StringPiece[n_groups];
+
+    // Match Whitespace
+    ret = WHITESPACE_RE->Match(html, offset, length, RE2::ANCHOR_START, groups, n_groups);
+    if (ret == 1) {
+      tag_match = 1;
+      rb_funcall(handler, m_whitespace, 1, rb_str_new(groups[0].data(), groups[0].size()));
+      offset += groups[0].size();
+    }
+    delete[] groups;
+
+    // Match Comment
+    if (html.substr(offset, COMMENT_TAG.size()) == COMMENT_TAG) {
+      tag_match = 1;
+      pos = html.find(COMMENT_TAG_END, offset + COMMENT_TAG.size());
+      rb_funcall(
+        handler,
+        m_text,
+        1,
+        rb_str_new(html.data()+offset, pos == -1 ? length-offset : (pos - offset) + COMMENT_TAG_END.size())
+      );
+      offset = pos == -1 ? length : pos + COMMENT_TAG_END.size();
+    }
+
+    // Match Start tag
+    if (tag_match == 0) {
+      n_groups = START_TAG_RE->NumberOfCapturingGroups() + 1;
+      groups = new(nothrow) StringPiece[n_groups];
+      ret = START_TAG_RE->Match(html, offset, length, RE2::ANCHOR_START, groups, n_groups);
+      if (ret == 1) {
+        tag_match = 1;
+        VALUE attrs = rb_hash_new();
+
+        if (groups[2].size() > 0) {
+          parse_attributes(attrs, groups[2]);
+        }
+
+        rb_funcall(
+          handler,
+          m_start_tag,
+          3,
+          rb_str_new(groups[1].data(), groups[1].size()),
+          attrs,
+          rb_str_new(groups[0].data(), groups[0].size())
+        );
+        offset += groups[0].size();
+        if (groups[0].ends_with(SELF_CLOSER)) {
+          rb_funcall(
+            handler,
+            m_end_tag,
+            2,
+            rb_str_new(groups[1].data(), groups[1].size()),
+            rb_str_new("", 0)
+          );
+        }
+      }
+      delete[] groups;
+    }
+
+    // Match end tag
+    if (tag_match == 0) {
+      n_groups = END_TAG_RE->NumberOfCapturingGroups() + 1;
+      groups = new(nothrow) StringPiece[n_groups];
+      ret = END_TAG_RE->Match(html, offset, length, RE2::ANCHOR_START, groups, n_groups);
+      if (ret == 1) {
+        tag_match = 1;
+        rb_funcall(
+          handler,
+          m_end_tag,
+          2,
+          rb_str_new(groups[1].data(), groups[1].size()),
+          rb_str_new(groups[0].data(), groups[0].size())
+        );
+        offset += groups[0].size();
+      }
+      delete[] groups;
+    }
+
+    // Match text
+    if (tag_match == 0) {
+      pos = html.find('<', offset + 1);
+      rb_funcall(
+        handler,
+        m_text,
+        1,
+        rb_str_new(html.data()+offset, pos == -1 ? length-offset : pos-offset)
+      );
+      offset = pos == -1 ? length : pos;
+    }
+  }
+
+  rb_funcall(handler, m_done, 0);
+
+  return handler;
+}
+
+extern "C" void
+Init_html_parser()
+{
+  WHITESPACE_RE   = new(nothrow) RE2("\\s+");
+  START_TAG_RE    = new(nothrow) RE2("<(\\w+[a-zA-Z0-9_\\.-]*)((?:\\s+[\\w-]+(?:\\s*=\\s*(?:(?:\"[^\"]*\")|(?:'[^']*')|(?:[^'\">\\s]*)))?)*)\\s*\\/?>");
+  END_TAG_RE      = new(nothrow) RE2("<\\/(\\w+[a-zA-Z0-9_:\\.-]*)\\s*>");
+  ATTRIBUTES_RE   = new(nothrow) RE2("([\\w_-]+)(?:\\s*=\\s*(?:(?:\"((?:\\.|[^\"])*)\")|(?:'((?:\\.|[^'])*)')|([^'\">\\s]*)))?");
+  COMMENT_TAG     = StringPiece("<!--");
+  COMMENT_TAG_END = StringPiece("-->");
+  SELF_CLOSER     = StringPiece("/>");
+
+  HtmlParser =  rb_const_get(rb_cObject, rb_intern("HtmlCruncher"));
+  rb_define_method(HtmlParser, "parse_html", (VALUE (*)(...))parse_html, 2);
+}

data/html_parser.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'html_cruncher/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "html_cruncher"
+  spec.version       = HtmlCruncher::VERSION
+  spec.authors       = ["Mathias Biilmann Christensen"]
+  spec.email         = ["info@mathias-biilmann.net"]
+  spec.summary       = %q{Will crunch through HTML and trigger callbacks on tags and text}
+  spec.description   = %q{HTML Cruncher is a streaming parser that'll gladly crunch through anything mildly resembling HTML.}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib", "ext"]
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake-compiler", "~> 0.9"
+
+  spec.extensions = %w{ext/html_parser/extconf.rb}
+end

data/lib/html_cruncher/version.rb

@@ -0,0 +1,3 @@
+class HtmlCruncher
+  VERSION = "1.0.0"
+end

data/lib/html_cruncher.rb

@@ -0,0 +1,15 @@
+require "html_cruncher/version"
+require "html_cruncher/html_parser"
+
+class HtmlCruncher
+  def initialize(handler)
+    @handler = handler
+  end
+
+  def crunch(source)
+    parse_html(source.dup.freeze, @handler)
+  end
+  alias :parse :crunch
+
+  private :parse_html
+end

data/test/html_cruncher_test.rb

@@ -0,0 +1,98 @@
+require 'minitest/autorun'
+require 'html_cruncher'
+
+
+class TestHtmlCruncher < MiniTest::Unit::TestCase
+  class Handler
+    attr_reader :stack
+
+    def initialize
+        @stack = []
+    end
+
+    def whitespace(w)
+      stack << Whitespace.new(w)
+    end
+
+    def text(t)
+      stack << Text.new(t)
+    end
+
+    def start_tag(t, a, r)
+      stack << StartTag.new(r, t, a)
+    end
+
+    def end_tag(t, r)
+      stack << EndTag.new(r, t)
+    end
+
+    def done
+      self
+    end
+
+    class Whitespace < Struct.new(:content); end
+    class Text < Struct.new(:content); end
+    class StartTag < Struct.new(:content, :tag, :attributes); end
+    class EndTag < Struct.new(:content, :tag); end
+  end
+
+  def parser
+    HtmlCruncher.new(Handler.new)
+  end
+
+  def test_html_with_whitespace
+    html = %[    Hello]
+    result = parser.parse(html)
+
+    assert_equal 2, result.stack.length
+    assert_equal Handler::Whitespace, result.stack.first.class
+    assert_equal Handler::Text, result.stack.last.class
+    assert_equal "Hello", result.stack.last.content
+  end
+
+  def test_html_with_simple_tag
+    html = %[<title>Hello</title>]
+    result = parser.parse(html)
+    assert_equal [Handler::StartTag, Handler::Text, Handler::EndTag], result.stack.map(&:class)
+    assert_equal "<title>", result.stack.first.content
+  end
+
+  def test_html_with_selfclosing_tag
+    html = %[<img src="img.png"/>]
+    result = parser.parse(html)
+    assert_equal Handler::StartTag, result.stack.first.class
+    assert_equal Handler::EndTag, result.stack.last.class
+    assert_equal html, result.stack.map(&:content).join
+    assert_equal({"src" => "img.png"}, result.stack.first.attributes)
+  end
+
+  def test_a_comment_tag
+    html = %[<!-- Hello --> hello]
+    result = parser.parse(html)
+    assert_equal [Handler::Text, Handler::Whitespace, Handler::Text], result.stack.map(&:class)
+    assert_equal ["<!-- Hello -->", " ", "hello"], result.stack.map(&:content)
+  end
+
+  def test_html_with_doctype
+    html = %[<!doctype html>
+<html>
+  <head><title>Test</title></head>
+  <body><h1>This is the body</h1></body>
+</html>]
+    result = parser.parse(html)
+
+    assert_equal html, result.stack.map(&:content).join
+  end
+
+  def test_unclosed_comment_tag
+    html = %[<!doctype html>
+<html>
+  <!-- Comment!
+  <head><title>Test</title></head>
+  <body><h1>This is the body</h1></body>
+</html>]
+    result = parser.parse(html)
+
+    assert_equal html, result.stack.map(&:content).join
+  end
+end

metadata

@@ -0,0 +1,102 @@
+--- !ruby/object:Gem::Specification
+name: html_cruncher
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Mathias Biilmann Christensen
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-10-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+description: HTML Cruncher is a streaming parser that'll gladly crunch through anything
+  mildly resembling HTML.
+email:
+- info@mathias-biilmann.net
+executables: []
+extensions:
+- ext/html_parser/extconf.rb
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- ext/html_parser/extconf.rb
+- ext/html_parser/html_parser.cc
+- html_parser.gemspec
+- lib/html_cruncher.rb
+- lib/html_cruncher/version.rb
+- test/html_cruncher_test.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+- ext
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Will crunch through HTML and trigger callbacks on tags and text
+test_files:
+- test/html_cruncher_test.rb
+has_rdoc: