saxxy 0.1.0

data/lib/saxxy/node_action.rb

@@ -0,0 +1,59 @@
+module Saxxy
+
+  ##
+  # @author rubymaniac
+  #
+  # NodeAction describes something that should be run on a
+  # node. In order to check whether to run this action it
+  # accepts as the first argument an activation_rule.
+  #
+  #
+  # @!attribute [r] activation_rule
+  #   @return [Context] this action's activation rule
+  #
+  # @!attribute [r] action
+  #   @return [Proc] the block of code that will run on a node
+  ##
+  class NodeAction
+    attr_reader :activation_rule, :action
+
+    # Initializes a NodeAction with an `activation_rule` a context to run
+    # its action (block) and the block.
+    #
+    # @param activation_rule [NodeRule] an instance of NodeRule
+    #   used to check whether to run this action on a node
+    #
+    # @param context [Object] a context (object) on which the block
+    #   will be evaluated
+    #
+    # @param block [Proc] a block that will get evaluated on context
+    #
+    def initialize(activation_rule, context = self, &block)
+      @activation_rule = activation_rule
+      @ctx = context
+      @action = block_given? ? block : ->(e) { e }
+    end
+
+    # Delegates the call to its `activation_rule`
+    #
+    # @param element_name [String] the name of a node
+    #
+    # @param attributes [Hash<String, String>] the attributes of a node
+    #
+    # @return [Boolean] whether it matches the node
+    #
+    def matches(element_name, attributes)
+      activation_rule.matches(element_name, attributes)
+    end
+
+    # Evaluates the block that was given to the constructor on the context
+    # and passes the arguments to the block
+    #
+    # @param args [Array] variable arguments that pass to the block
+    #
+    def call(*args)
+      @ctx.instance_exec(args, &action)
+    end
+  end
+
+end

data/lib/saxxy/node_rule.rb

@@ -0,0 +1,90 @@
+module Saxxy
+
+  ##
+  # @author rubymaniac
+  #
+  # NodeRule describes a rule that will be tested upon an XML node
+  # and will check if the node satisfies this NodeRule.
+  #
+  # The NodeRule consists of two parts. The `element` part which
+  # refers to what should hold for the node's name. It can be
+  # either a String (where the strict equality is should hold) or
+  # a Regexp (where the Regexp must match the node name).
+  #
+  # The other part is the `attributes` part which refers to what
+  # should hold for the attributes of the node. It consists of key-value
+  # pairs where the key is the attribute to check and the value is what
+  # should hold for that attribute.
+  #
+  # @!attribute [r] element
+  #   @return [String|Regexp] node's name rule
+  #
+  # @!attribute [r] attributes
+  #   @return [Hash<String, String|Regexp>] node's attributes rule
+  ##
+  class NodeRule
+    attr_reader :element, :attributes
+
+    # Initializes a NodeRule with an `element` part and an `attributes` part.
+    #
+    # @param element [String|Regexp] what should hold for the node name
+    # @param attributes [Hash<String, String|Regexp>]
+    #   what should hold for node's attributes
+    #
+    def initialize(element, attributes = {})
+      @element = element
+      @attributes = Saxxy::Helpers.stringify_keys(attributes)
+    end
+
+    # Checks whether this NodeRule matches a node.
+    #
+    # @param element_name [String] node's name
+    # @param attrs [Hash<String, String>] node's attributes
+    #
+    # @return [Boolean] whether this NodeRule matches the node
+    #
+    def matches(element_name, attrs = {})
+      match_element_name(element_name) && match_attributes(attrs)
+    end
+
+    # Checks whether this NodeRule is equal to another.
+    #
+    # @param rule [NodeRule] the other NodeRule
+    #
+    # @return [Boolean] whether this NodeRule equals rule
+    #
+    def equals(rule)
+      element == rule.element && attributes == rule.attributes
+    end
+
+    # Checks whether this NodeRule matches only the name of a node.
+    #
+    # @param element_name [String] node's name
+    #
+    # @return [Boolean] whether this NodeRule matches node's name
+    #
+    def match_element_name(element_name)
+      match(element, element_name)
+    end
+
+    # Checks whether this NodeRule matches only the attributes of a node.
+    #
+    # @param attrs [Hash<String, String>] node's attributes
+    #
+    # @return [Boolean] whether this NodeRule matches node's attributes
+    #
+    def match_attributes(attrs)
+      attrs = Saxxy::Helpers.stringify_keys(attrs)
+      attributes.reduce(true) do |b, (k, v)|
+        value = attrs[k]
+        b && ((!value.nil? && match(v, value)) || (v.nil? && value.nil?))
+      end
+    end
+
+    private
+    def match(obj, value)
+      obj.is_a?(Regexp) ? !obj.match(value).nil? : obj == value
+    end
+  end
+
+end

data/lib/saxxy/parsers/base.rb

@@ -0,0 +1,28 @@
+module Saxxy
+  module Parsers
+
+    class NotImplemented < StandardError; end
+
+    class Base
+      attr_reader :context_tree, :options
+
+      def initialize(context_tree, options = {})
+        @context_tree = context_tree
+        @options = options
+      end
+
+      def parse_file(path_to_file)
+        raise NotImplemented
+      end
+
+      def parse_string(string)
+        raise NotImplemented
+      end
+
+      def parse_io(io)
+        raise NotImplemented
+      end
+    end
+
+  end
+end

data/lib/saxxy/parsers/libxml.rb

@@ -0,0 +1,52 @@
+require "saxxy/parsers/base"
+require "saxxy/callbacks/libxml"
+
+
+module Saxxy
+  module Parsers
+
+    class Libxml < Base
+      def initialize(context_tree, options = {})
+        super
+        @__internal_context_options =
+          case options[:mode]
+          when :html, nil
+            LibXML::XML::Parser::Options::RECOVER |
+            LibXML::XML::Parser::Options::NOERROR |
+            LibXML::XML::Parser::Options::NOWARNING |
+            LibXML::XML::Parser::Options::NONET
+          when :xml
+            LibXML::XML::Parser::Options::RECOVER |
+            LibXML::XML::Parser::Options::NONET
+          end
+      end
+
+      def parse_string(string, encoding = LibXML::XML::Encoding::UTF_8)
+        parse_with LibXML::XML::SaxParser.new(build_context(:string, string, encoding))
+      end
+
+      def parse_file(path_to_file, encoding = LibXML::XML::Encoding::UTF_8)
+        parse_with LibXML::XML::SaxParser.new(build_context(:file, path_to_file, encoding))
+      end
+
+      def parse_io(io, encoding = LibXML::XML::Encoding::UTF_8)
+        parse_with LibXML::XML::SaxParser.new(build_context(:io, io, encoding))
+      end
+
+      private
+      def build_context(method, obj, encoding)
+        LibXML::XML::Parser::Context.public_send(method, obj).tap do |ctx|
+          ctx.options = @__internal_context_options
+          ctx.encoding = encoding
+          ctx.recovery = true
+        end
+      end
+
+      def parse_with(parser)
+        parser.callbacks = Saxxy::Callbacks::Libxml.new(context_tree.root)
+        parser.parse
+      end
+    end
+
+  end
+end

data/lib/saxxy/parsers/nokogiri.rb

@@ -0,0 +1,28 @@
+require "saxxy/parsers/base"
+require "saxxy/callbacks/nokogiri"
+
+
+module Saxxy
+  module Parsers
+
+    class Nokogiri < Base
+      def parse_string(string, encoding = 'UTF-8', &block)
+        new_parser.parse_memory(string, encoding, &block)
+      end
+
+      def parse_file(path_to_file, encoding = 'UTF-8', &block)
+        new_parser.parse_file(path_to_file, encoding, &block)
+      end
+
+      def parse_io(io, encoding = 'UTF-8', &block)
+        new_parser.parse_io(io, encoding, &block)
+      end
+
+      private
+      def new_parser
+        ::Nokogiri::HTML::SAX::Parser.new(Saxxy::Callbacks::Nokogiri.new(context_tree.root))
+      end
+    end
+
+  end
+end

data/lib/saxxy/parsers/ox.rb

@@ -0,0 +1,30 @@
+require "saxxy/parsers/base"
+require "saxxy/callbacks/ox"
+
+
+module Saxxy
+  module Parsers
+
+    class Ox < Base
+      def parse_string(string, encoding = nil)
+        parse(StringIO.new(string), encoding)
+      end
+
+      def parse_file(path_to_file, encoding = nil)
+        parse(File.new(path_to_file), encoding)
+      end
+
+      def parse_io(io, encoding = nil)
+        parse(io, encoding)
+      end
+
+      private
+      def parse(io, encoding)
+        io.set_encoding(encoding) if encoding
+        callbacks = Saxxy::Callbacks::Ox.new(context_tree.root)
+        ::Ox.sax_parse(callbacks, io, {smart: true}.merge(options))
+      end
+    end
+
+  end
+end

data/lib/saxxy/service.rb

@@ -0,0 +1,47 @@
+require "saxxy/context_tree"
+
+
+module Saxxy
+
+  module Parsers
+    autoload :Nokogiri, "saxxy/parsers/nokogiri"
+    autoload :Ox, "saxxy/parsers/ox"
+    autoload :Libxml, "saxxy/parsers/libxml"
+  end
+
+  class Service
+    attr_reader :parser
+
+    def initialize(parser, options = {}, &block)
+      @parser = build_parser(parser, options, &block)
+    end
+
+    def parse_file(*args)
+      @parser.parse_file(*args)
+    end
+
+    def parse_string(*args)
+      @parser.parse_string(*args)
+    end
+
+    def parse_io(*args)
+      @parser.parse_io(*args)
+    end
+
+    private
+    def build_parser(parser, options, &block)
+      ctx = eval("self", block.binding)
+      parser_class_from(parser).new(Saxxy::ContextTree.new(ctx, &block), options)
+    end
+
+    def parser_class_from(obj)
+      case obj
+      when Symbol, String
+        Saxxy::Parsers.const_get(Saxxy::Helpers.camelize(obj))
+      else
+        obj
+      end
+    end
+  end
+
+end

data/lib/saxxy/utils/agent.rb

@@ -0,0 +1,66 @@
+require "net/http"
+require "uri"
+
+
+module Saxxy
+
+  # The Agent is a thin wrapper over Net::HTTP::Proxy in order to be used
+  # for crawling purposes. Supports GET and POST via its get and post methods.
+  class Agent
+    attr_reader :url, :uri, :proxy, :agent, :response
+
+    # Initializes an agent with optional proxy options.
+    # Url: A string that it is the url that the agent is going to use for issuing
+    #      requests. It can be reset to another via the self.uri = method.
+    # Options:
+    # - proxy:
+    #   - address: The address of the proxy.
+    #   - port: The port the proxy will use.
+    #   - username: The username if the proxy needs auth.
+    #   - password: The password if the proxy needs auth.
+    def initialize(url, opts = {})
+      @proxy = opts[:proxy] || {}
+      @agent = proxy.empty? ? Net::HTTP : Net::HTTP::Proxy(proxy[:address], proxy[:port], proxy[:username], proxy[:password])
+      self.uri = url
+    end
+
+    # Sets the url and uri by inspecting the argument. Can accept either a string
+    # which must be a valid URL or a URI object.
+    def uri=(url_or_uri)
+      @uri = url_or_uri.is_a?(URI) ? url_or_uri : URI(url_or_uri)
+      @url = uri.to_s
+    end
+
+    # Issues a get request either by using the url provided as an argument or
+    # the one the agent currently holds.
+    # Note: if the provided url is different from the agent's it updates the
+    #       agent's url also. See set_uri_for.
+    def get(url = nil)
+      issue_request(url, :get_response)
+    end
+
+    # Issues a post request either by using the url provided as an argument or
+    # the one the agent currently holds. Uses the post_form method of the
+    # Net::HTTP::Proxy and forwards any passed options to the underlying agent.
+    # Note: if the provided url is different from the agent's it updates the
+    #       agent's url also. See set_uri_for.
+    def post(url = nil, opts = {})
+      issue_request(url, :post_form, opts)
+    end
+
+    private
+    def set_uri_for(url = nil)
+      self.uri = url if url
+    end
+
+    def issue_request(*args)
+      new_url_or_uri = args.shift
+      if new_url_or_uri.to_s != url
+        set_uri_for(new_url_or_uri)
+        @response = agent.public_send(args.shift, uri, *args)
+      end
+      response.body
+    end
+  end
+
+end

data/lib/saxxy/utils/callback_array.rb

@@ -0,0 +1,27 @@
+module Saxxy
+
+  class CallbackArray < Array
+    def <<(obj)
+      super(obj)
+      @add_callback.call(obj) if @add_callback
+      self
+    end
+
+    def >>(obj)
+      delete(obj)
+      @remove_callback.call(obj) if @remove_callback
+      self
+    end
+
+    def on_remove(&block)
+      @remove_callback = block
+      self
+    end
+
+    def on_add(&block)
+      @add_callback = block
+      self
+    end
+  end
+
+end

data/lib/saxxy/utils/helpers.rb

@@ -0,0 +1,13 @@
+module Saxxy
+  class Helpers
+
+    def self.camelize(obj)
+      obj.to_s.split(/[^a-z0-9]/i).map(&:capitalize).join
+    end
+
+    def self.stringify_keys(hash)
+      Hash[hash.map { |k, v| [k.to_s, v] }]
+    end
+
+  end
+end