rusty 0.1

data/lib/rusty/helpers.rb

@@ -0,0 +1,40 @@
+# This file is part of the rusty ruby gem.
+#
+# Copyright (c) 2013 @radiospiel
+# Distributed under the terms of the modified BSD license, see LICENSE.BSD
+
+# Helper support for Rusty.
+module Rusty::Helpers
+  # return all helpers
+  def helpers
+    @helpers ||= []
+  end
+  
+  # set up a helper. Examples:
+  #
+  #  module MyParser
+  #    extend Rusty::RuleSet
+  #
+  #    helper Rusty::Helpers::Text
+  #
+  #    helper do 
+  #      def foo
+  #        "bar"
+  #      end
+  #    end
+  #  end
+  #
+  def helper(*mods, &block)
+    helpers.concat mods
+    helpers << Module.new.tap { |mod| mod.class_eval(&block) } if block
+  end
+end
+
+# Some Text helpers.
+module Rusty::Helpers::Text
+  #
+  # Returns a cleaned version of a node's text.
+  def text(node)
+    node.text.gsub(/\u200e/, "").strip
+  end
+end

data/lib/rusty/nokogiri_ext.rb

@@ -0,0 +1,101 @@
+# This file is part of the rusty ruby gem.
+#
+# Copyright (c) 2013 @radiospiel
+# Distributed under the terms of the modified BSD license, see LICENSE.BSD
+
+# Nokogiri extensions as used by rusty
+
+# ---------------------------------------------------------------------
+
+class Nokogiri::XML::Node
+  # returns a nodes attributes as a hash
+  def attributes_hash
+    attributes.inject({}) do |hash, (name, attr)|
+      hash.update name => attr.value
+    end 
+  end
+
+  # returns an array of classes as Strings
+  def classes
+    return [] unless classes = self["class"]
+    classes.strip.split(/\s+/)
+  end
+
+  # does this node has a class with a given name?
+  def has_class?(name)
+    @class_syms ||= classes.map(&:to_sym)
+    @class_syms.include?(name.to_sym)
+  end
+
+  def parents
+    return [] if parent == document
+    self_and_parents(parent)
+  end
+  
+  # return a list of all parent nodes, up until and excluding the document node.
+  def self_and_parents(node=self)
+    [ ].tap do |parents|
+      while node.parent != node.document
+        parents.unshift(node)
+        node = node.parent
+      end
+      parents.unshift node
+    end
+  end
+
+  # returns the debug node name for this node; which is a simplified CSS node name
+  # name{#id}{.class}{.class}
+  def simplified_name
+    simplified_name = name
+    if id = self["id"]
+      simplified_name += "##{id}"
+    end
+    classes.each do |klass|
+      simplified_name += ".#{klass}"
+    end
+    simplified_name.gsub(/^div([\.\#])/, "\\1")
+  end
+end
+
+class Nokogiri::HTML::Document
+  # returns the encoding as defined in the meta[http-equiv=content-type]
+  # node. Available only in HTML documents.
+  def meta_encoding
+    # HTML5
+    css("meta[charset]").each do |meta|
+      next unless charset = meta.attribute("charset")
+
+      return charset.value
+    end
+    
+    # HTML4
+    css("meta[http-equiv=content-type]").each do |meta|
+      next unless content = meta.attribute("content") 
+      next unless content.value.split("; ").last =~ /^charset=(.*)/
+
+      return $1
+    end
+
+    nil
+  end
+end
+
+module Nokogiri::HTML
+  # loads a document from \a data. If the encoding as determined by Nokogiri
+  # does not match the meta_encoding, tries to reload the data with that 
+  # encoding.
+  def self.with_meta_encoding(data)
+    doc = Nokogiri.HTML(data)
+    
+    meta_encoding = doc.meta_encoding
+    return doc unless meta_encoding && doc.encoding != meta_encoding
+    
+    # try to reread with meta_encoding
+    doc2 = Nokogiri.HTML(data, nil, meta_encoding)
+    return doc2 if doc2.encoding == meta_encoding
+
+    # rereading failed, return original document
+    doc
+  end
+end
+

data/lib/rusty/rule_set.rb

@@ -0,0 +1,124 @@
+# This file is part of the rusty ruby gem.
+#
+# Copyright (c) 2013 @radiospiel
+# Distributed under the terms of the modified BSD license, see LICENSE.BSD
+
+module Rusty::RuleSet
+  include Rusty::Helpers
+
+  # A rule combines a selector with a proc.
+  class Rule < Struct.new(:selector, :proc)
+  end
+
+  # record a rule for any of these selectors.
+  #
+  # This rule will be activated when a node gets processed.
+  def on(*selectors, &block)
+    register_rule(:on, *selectors, &block)
+  end
+
+  # record a rule for any of these selectors.
+  #
+  # This rule will be activated when a node's processing is done.
+  #
+  # Note: The after method is similar to
+  # 
+  #  on "selector" do
+  #    callback do
+  #      do_something
+  #    end
+  #  end
+  def after(*selectors, &block)
+    register_rule(:after, *selectors, &block)
+  end
+  
+  private
+  
+  # Return a hash of rules in a given mode.
+  def rules_for_mode(mode)
+    @rules ||= {}
+    @rules[mode] ||= {}
+  end
+
+  # Register a rule for a number of selectors in a given mode.
+  # Mode should be :on or :after
+  def register_rule(mode, *selectors, &block)
+    rules = rules_for_mode(mode)
+
+    selectors.
+      map  { |selector| selector.split(",").map(&:strip) }.
+      flatten.
+      each { |selector| 
+        STDERR.puts "#{name}, in mode :#{mode}: redefining rule for #{selector}" if rules[selector]
+        rules[selector] = Rule.new(Rusty::Selector.new(selector), block) 
+      }
+  end
+  
+  public
+  
+  # return the best matching rule for a given node
+  # Mode should be :on or :after
+  def best_rule(mode, node)
+    rules_for_mode(mode).values.
+      select  { |rule| rule.selector.match?(node) }.
+      sort_by { |rule| rule.selector.weight }.
+      last
+  end
+
+  private
+  
+  # returns the class for event scopes in this RuleSet. This is a subclass of
+  # Rusty::CallbackBinding, which is named after the current modules name (i.e. if
+  # RuleSet is extended into a Module Foo, the subclass will be named Rusty::
+  # CallbackBinding::Foo) and which has all helpers correctly loaded. 
+
+  def callback_binding_klass
+    @callback_binding_klass ||= Rusty::CallbackBinding.subclass_with_name_and_helpers name, *helpers
+  end
+
+  public
+  
+  # transform a node, and return transformed data.
+  def transform!(node, scope = nil)
+    if node.is_a?(Nokogiri::XML::Document)
+      node = node.root
+    end
+
+    scope ||= Rusty::Scope.new(node)
+
+    # The callback scope for this node.
+    callback_binding = callback_binding_klass.new(scope) 
+
+    has_rule = false
+    
+    [ :on, :after ].each do |mode|
+      # find explicit rule for this node. Warn if there is none.
+      if rule = best_rule(mode, node)
+        has_rule = true
+        callback_binding.instance_eval(&rule.proc)
+      end
+
+      # in :on mode: process children, unless explicitely skipped.
+      if mode == :on && !callback_binding.skip?
+        node.children.each do |child|
+          next if child.text? || child.cdata?
+          next if child.comment?
+
+          transform! child, Rusty::Scope.new(child, scope)
+        end
+      end
+
+      # run callback
+      if callback = callback_binding.callback
+        callback_binding.instance_eval(&callback)
+      end
+    end
+    
+    unless has_rule
+      path = node.self_and_parents.map(&:simplified_name).join(" > ")
+      STDERR.puts "no rule registered: #{path}" 
+    end
+
+    scope
+  end
+end

data/lib/rusty/scope.rb

@@ -0,0 +1,29 @@
+# This file is part of the rusty ruby gem.
+#
+# Copyright (c) 2013 @radiospiel
+# Distributed under the terms of the modified BSD license, see LICENSE.BSD
+
+#
+# A Rusty output scope, is related to a input node, 
+# and might or might not have a parent.
+class Rusty::Scope < Rusty::DX
+  attr :node, true
+  private :node=
+
+  def initialize(node, parent=nil)
+    @node, @parent = node, parent
+  end
+  
+  # Does this scope matches a given name?
+  def has_name?(name)
+    return @parent.nil? if name == "document"
+
+    node.name == name || node.has_class?(name)
+  end
+
+  # yields all nodes starting at self up to the top.
+  def up!(&block)
+    yield(self)
+    @parent.up!(&block) if @parent
+  end
+end

data/lib/rusty/selector.rb

@@ -0,0 +1,87 @@
+# This file is part of the rusty ruby gem.
+#
+# Copyright (c) 2013 @radiospiel
+# Distributed under the terms of the modified BSD license, see LICENSE.BSD
+
+# -- selector engines ---------------------------------------------------------
+
+# A selector is an object which is created based on a selector string, and
+# which implements the `weight` and `match?` methods.
+module Rusty::Selector
+  # A simple, nokogiri based CSS matcher.
+  class CSS
+    attr :matcher, :name
+
+    # Create selector
+    def initialize(selector)
+      @name = @selector = selector
+
+      # == special case: "*"
+      #
+      # The "*" selector matches all nodes, and Nokogiri::XML::Node#css returns 
+      # a huge array of nodes, which R::S::CSS#match? would have to walk through.
+      # Implementing that special case speeds up things by ~10% in the google 
+      # example, and reduces memory load.
+      #
+      # Note: by defining it directly on <self> this special case implementation
+      # also overrides match? methods defined in subclasses.
+      if @selector == "*"
+        def self.match?(node); !node.nil?; end
+      end
+    end
+
+    # The weight of the selector; is close to, but not exactly as
+    # CSS's weight definition.
+    def weight
+      @weight ||= @selector.split(/\s+/).inject(0) do |weight, part|
+          weight += case part
+            when /#/          then 1_000_000 # part with an ID, much much weight
+            when /\./         then     1_000 # selector with a class
+            when /^[a-zA-Z_]/ then     1_000 # node name
+            else                           1
+            end
+        end
+    end
+
+    # Does this selector matches a specific node?
+    def match?(node)
+      return false unless node
+      
+      node.document.css(@selector).include?(node)
+    end
+  end
+
+  # A cached CSS selector, caches matching nodes within a document.
+  class CachedCSS < CSS
+    # Does this selector matches a specific node?
+    def match?(node)
+      return false unless node
+      
+      cache_document(node.document)
+      @matching_nodes.include?(node)
+    end
+
+
+    private
+
+    def cache_document(document)
+      return if @cached_document && @cached_document == document
+
+      @cached_document = document
+      @matching_nodes = document.css(@selector)
+    end
+  end
+
+  # You probably want cached selectors, especially when working with 
+  # larger documents. If these eat to much memory, try to use
+  #
+  # DEFAULT_SELECTOR = Rusty::Selector::CSS
+  #
+  # but expect exploding runtimes: this increases O(m+n) -> O(m*n).
+  DEFAULT_SELECTOR = Rusty::Selector::CachedCSS
+
+  # Create a Selector object for a given `selector` string.
+  def self.new(selector)
+    DEFAULT_SELECTOR.new selector
+  end
+end

data/lib/rusty/version.rb

@@ -0,0 +1,8 @@
+# This file is part of the rusty ruby gem.
+#
+# Copyright (c) 2013 @radiospiel
+# Distributed under the terms of the modified BSD license, see LICENSE.BSD
+
+module Rusty
+  VERSION = "0.1"
+end

data/rusty.gemspec

@@ -0,0 +1,24 @@
+# This file is part of the rusty ruby gem.
+#
+# Copyright (c) 2013 @radiospiel
+# Distributed under the terms of the modified BSD license, see LICENSE.BSD
+require "#{File.dirname(__FILE__)}/lib/rusty/version.rb"
+
+Gem::Specification.new do |gem|
+  gem.name     = "rusty"
+  gem.version  = Rusty::VERSION
+  
+  gem.authors   = ["radiospiel"]
+  gem.email     = ["eno@radiospiel.org"]
+  gem.homepage  = "http://github.com/radiospiel/rusty"
+  gem.summary   = "XML parsing without the hassle."
+  
+  gem.description = gem.summary
+
+  gem.add_dependency "nokogiri"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+end

data/tasks/rdoc.rake

@@ -0,0 +1,18 @@
+# This file is part of the rusty ruby gem.
+#
+# Copyright (c) 2013 @radiospiel
+# Distributed under the terms of the modified BSD license, see LICENSE.BSD
+
+$:.unshift File.expand_path("../lib", __FILE__)
+
+require 'rdoc/task'
+
+RDoc::Task.new do |rdoc|
+  require_relative "../lib/rusty/version"
+  version = Rusty::VERSION
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "rusty #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/tasks/test.rake

@@ -0,0 +1,8 @@
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+