saper 0.5.0

data/lib/saper/arguments/timezone.rb

@@ -0,0 +1,11 @@
+module Saper
+  module Arguments
+    class Timezone < Argument
+  
+      def valid?(value)
+        value.is_a?(String)
+      end
+      
+    end
+  end
+end

data/lib/saper/arguments/variable.rb

@@ -0,0 +1,11 @@
+module Saper
+  module Arguments
+    class Variable < Argument
+    
+      def valid?(value)
+        value.is_a?(String) || value.is_a?(Symbol)
+      end
+      
+    end
+  end
+end

data/lib/saper/arguments/xpath.rb

@@ -0,0 +1,11 @@
+module Saper
+  module Arguments
+    class XPath < Argument
+  
+      def valid?(value)
+        value.is_a?(String)
+      end
+    
+    end
+  end
+end

data/lib/saper/core/action.rb

@@ -0,0 +1,209 @@
+module Saper
+  class Action
+    
+    # Tracks subclasses of Saper::Action.
+    # @return [Class]
+    def self.inherited(base)
+      subclasses[base.type] = base
+    end
+    
+    # Returns a hash of subclasses.
+    # @return [Hash]
+    def self.subclasses
+      @subclasses ||= {}
+    end
+    
+    # Returns class name as an underscored string.
+    # @return [String]
+    def self.type
+      name.split("::").last.gsub(/([a-z])([A-Z])/,'\1_\2').downcase
+    end
+    
+    # Returns a subclass with specified type.
+    # @param type [Symbol] action type
+    # @return [Saper::Action]
+    def self.[](type)
+      subclasses[type.to_s] || raise(ActionNotFound, type)
+    end
+    
+    # Defines a new argument.
+    # @param type [Symbol] argument type (e.g. :text, :xpath)
+    # @param options [Hash] argument options (e.g. :optional => true)
+    # @return [void]
+    def self.argument(type, options = {})
+      if Argument.exists?(type)
+        arguments.push options.merge(:type => type)
+      else
+        raise(InvalidType, "Invalid action argument: %s" % type)
+      end
+    end
+    
+    # Returns a list of arguments assigned to this action.
+    # @return [Array<Hash>]
+    def self.arguments
+      @arguments ||= []
+    end
+    
+    # Sets (one) or returns (all) acceptable input types.
+    # @param input [Symbol]
+    # @return [Array<Symbol>]
+    def self.accepts(input = nil, options = {})
+      if input.nil?
+        return @types.nil? ? [] : @types.keys
+      end
+      if input == :anything
+        return Item.subclasses.keys.map { |type| accepts(type.to_sym, options) }
+      end
+      output = options[:returns] || input
+      unless Item.exists?(input)
+        raise(InvalidInput, "Invalid input type: %s" % input)
+      end
+      if output.is_a?(Symbol) && !Item.exists?(output)
+        raise(InvalidInput, "Invalid output type: %s" % output)
+      end
+      @types ||= {}
+      @types[input] = output
+    end
+    
+    # Returns `true` if action accepts specified type as input.
+    # @param type [Symbol]
+    # @return [Boolean]
+    def self.accepts?(type)
+      @types.nil? ? false : @types.keys.include?(type)
+    end
+    
+    # Saves Proc that encapsulates action logic and will be used later for data processing.
+    # @return [void]
+    def self.run(&block)
+      @block ||= block
+    end
+    
+    # Returns `true` if action returns multiple items. Note that this method will report incorrect data for some actions. Use #multiple? instead.
+    def self.returns_multiple_items?
+      @multiple == true
+    end
+    
+    # Sets a flag, indicating that this action returns multiple items.
+    # @return [void]
+    def self.returns_multiple_items!
+      @multiple = true
+    end
+    
+    # Returns a new instance of Saper::Action.
+    # @param data [Hash, Array<Hash>]
+    # @return [Saper::Action]
+    def self.unserialize(data, namespace = nil, &block)
+      if data.is_a?(Array)
+        return data.map { |item| unserialize(item, namespace) }
+      end
+      unless data.is_a?(Hash)
+        raise InvalidAction.new(data)
+      end
+      new(data[:type], *data[:args], :namespace => namespace) do |action|
+        if block_given?
+          yield action
+        end
+      end
+    end
+    
+    # Returns a new instance of Saper::Action.
+    # @return [Saper::Action]
+    def self.new(*args, &block)
+      if self == Action
+        self[args.shift].new(*args, &block)
+      else
+        super(*args, &block)
+      end
+    end
+    
+    attr_reader :options
+    
+    # Returns a new instance of Saper::Action.
+    # @return [Saper::Action]
+    def initialize(*args)
+      @arguments = []
+      @options   = {}
+      if args.last.is_a?(Hash)
+        @options = args.pop
+      end
+      self.class.arguments.each_with_index do |opts, i|
+        opts.merge!(:value => args[i], :action => self)
+        @arguments << Argument.new(opts[:type], opts)
+      end
+      if block_given?
+        yield self
+      end
+    end
+    
+    # Runs action and returns results.
+    # @param input [object] input
+    # @return [void] depends on action type
+    def run(input = nil, runtime = nil)
+      unless input.is_a?(Item)
+        input = self.class.accepts.map { |type| Item.try(type, input) }.compact.first
+      end
+      if input.nil?
+        input = Items::Nothing.new
+      end
+      unless self.class.accepts?(input.type)
+        raise(InvalidInput, input)
+      end
+      if runtime.nil?
+        begin
+          block.call(input, *args)
+        rescue NameError
+          raise RuntimeMissing
+        end
+      else
+        runtime.instance_exec(input, *args, &self.block)
+      end
+    end
+    
+    # Returns human readable action name.
+    # @return [String]
+    def name
+      self.class.name.split("::").last.gsub(/([a-z])([A-Z])/,'\1 \2')
+    end
+    
+    # Returns Saper::Namespace instance.
+    # @return [Namespace]
+    def namespace
+      @options[:namespace].is_a?(Namespace) ? @options[:namespace] : nil
+    end
+    
+    # Returns a list of data types that are accepted as input.
+    # @return [Array<Symbol>]
+    def requires
+      self.class.accepts
+    end
+    
+    # Returns values of action arguments.
+    # @return [Array]
+    def args
+      @arguments.map(&:value)
+    end
+    
+    # Returns a serialized representation of this action.
+    # @return [Hash]
+    def serialize
+      { :type => self.class.type, :args => @arguments.map(&:serialize) }
+    end
+    
+    # Returns `true` if action returns multiple items.
+    # @return [Boolean]
+    def multiple?
+      self.class.returns_multiple_items?
+    end
+    
+    # Returns Proc that encapsulates action logic (i.e. processes data).
+    # @return [Proc]
+    def block
+      self.class.run || Proc.new { |input, *args| input }
+    end
+    
+    def to_string
+      "\t%s %s" % [self.class.type, @arguments.map(&:to_string).join(", ")]
+    end
+    
+  end
+end

data/lib/saper/core/argument.rb

@@ -0,0 +1,106 @@
+module Saper
+  class Argument
+    
+    # Tracks subclasses of Saper::Argument.
+    # @return [Class]
+    def self.inherited(base)
+      subclasses[base.type] = base
+    end
+    
+    # Returns a hash of subclasses.
+    # @return [Hash]
+    def self.subclasses
+      @subclasses ||= {}
+    end
+    
+    # Returns class name as an underscored string.
+    # @return [String]
+    def self.type
+      name.split("::").last.gsub(/([a-z])([A-Z])/,'\1_\2').downcase
+    end
+    
+    # Returns a subclass with specified type.
+    # @param type [Symbol] action type
+    # @return [Saper::Argument]
+    def self.[](type)
+      subclasses[type.to_s] || raise(InvalidType, "Invalid argument: %s" % type)
+    end
+    
+    # Returns `true` if there is a subclass with specified type.
+    # @param type [Symbol] action type
+    # @return [Boolean]
+    def self.exists?(type)
+      subclasses.keys.include?(type.to_s)
+    end
+    
+    # Returns a new instance of Saper::Argument.
+    # @return [Saper::Argument]
+    def self.new(*args, &block)
+      if self == Argument
+        self[args.shift].new(*args, &block)
+      else
+        super(*args, &block)
+      end
+    end
+    
+    # @todo
+    def initialize(opts = {})
+      @value = nil
+      @opts  = opts
+      if mandatory? || !@opts[:value].nil?
+        set @opts.delete(:value)
+      end
+    end
+    
+    # @todo
+    def set(value)
+      unless valid?(value)
+        raise InvalidArgument, value
+      else
+        @value = normalize(value)
+      end
+      self
+    end
+    
+    # @todo
+    def valid?(value)
+      true
+    end
+    
+    # @todo
+    def normalize(value)
+      value
+    end
+    
+    # @todo
+    def value
+      @value
+    end
+    
+    # @todo
+    def serialize
+      value
+    end
+    
+    # @todo
+    def mandatory?
+      not optional?
+    end
+    
+    # @todo
+    def action
+      @opts[:action]
+    end
+    
+    # @todo
+    def optional?
+      @opts[:optional] == true
+    end
+    
+    # @todo
+    def to_string
+      value.to_s.inspect
+    end
+    
+  end
+end

data/lib/saper/core/browser.rb

@@ -0,0 +1,87 @@
+module Saper
+  class Browser
+  
+    require 'mechanize'
+    
+    attr_reader :history, :received, :sent
+    
+    def initialize(options = {})
+      @agent    = options.delete(:agent)
+      @headers  = options.delete(:headers)
+      @logger   = options.delete(:logger) || Saper::Logger.new
+      @history  = []
+      @received = 0
+      @sent     = 0
+      @mech = Mechanize.new do |a|
+        a.robots                 = false
+        a.user_agent             = agent
+        a.request_headers        = headers
+        a.pluggable_parser.csv   = nil
+        a.pluggable_parser.html  = nil
+        a.pluggable_parser.xhtml = nil
+        a.pluggable_parser.xml   = nil
+      end
+      @mech.pre_connect_hook do |agent, req|
+        @sent += req.to_hash.to_s.size
+        @sent += (req.body.nil? ? 0 : req.body.size)
+      end
+      @mech.post_connect_hook do |agent, uri, resp, body|
+        @received += resp.to_hash.to_s.size
+        @received += body.size
+      end
+    end
+    
+    # Returns number of HTTP requests
+    def requests
+      @history.size
+    end
+    
+    #
+    def headers
+      @headers.respond_to?(:to_hash) ? @headers : {}
+    end
+    
+    #
+    def get(url, query = {})
+      @logger.download(url)
+      @history.push url
+      data = @mech.get(url, query)
+      Saper::Items::Document.new data
+    end
+    
+    #
+    def post(url, query = {})
+      @logger.download(url)
+      @history.push url
+      data = @mech.post(url, query)
+      Saper::Items::Document.new data
+    end
+    
+    #
+    def agent
+      case @agent
+      when :ie6
+        'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)'
+      when :ie7
+        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)'
+      when :ie8
+        'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727)'
+      when :ie9
+        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)'
+      when :mozilla
+        'Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.4b) Gecko/20030516 Mozilla Firebird/0.6'
+      when :safari
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/534.51.22 (KHTML, like Gecko) Version/5.1.1 Safari/534.51.22'
+      when :iphone
+        'Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1C28 Safari/419.3'
+      when :ipad
+        'Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B334b Safari/531.21.10'
+      when :android
+        'Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13'
+      else
+        'Mozilla/5.0 (compatible; Saper Ruby client %s)' % Saper::VERSION
+      end
+    end
+  
+  end
+end

data/lib/saper/core/dsl.rb

@@ -0,0 +1,68 @@
+module Saper
+  module DSL
+    
+    def self.new
+      Module.new.extend(Methods)
+    end
+    
+    def self.included(base)
+      base.extend(Methods)
+    end
+    
+    module Methods
+      
+      def namespace
+        @namespace ||= Saper::Namespace.new
+      end
+      
+      def recipe(id, name = nil, &block)
+        namespace[id] = Recipe.parse(id, name, :namespace => namespace, &block)
+      end
+      
+      def [](name)
+        namespace[name]
+      end
+      
+      def run_by_default(*args)
+        namespace.run_by_default(*args)
+      end
+      
+      def run(*args)
+        namespace.run(*args)
+      end
+      
+    end
+    
+    class Recipe
+      # Parses block and returns a Recipe instance or a proxy object.
+      # If namespace is specified within options, full initialization of
+      # Recipe is delayed and a proxy object is returned (which supports
+      # `#to_recipe`).
+      # @param id [Symbol] recipe ID
+      # @return [Saper::Recipe, Saper::DSL::Recipe]
+      def self.parse(id, name = nil, options = {}, &block)
+        instance = self.new(id, options.merge(:name => name), &block)
+        if options[:namespace].is_a?(Namespace)
+          instance
+        else
+          instance.to_recipe
+        end
+      end
+      
+      attr_reader :recipe
+      
+      def initialize(id = nil, options = {}, &block)
+        @recipe, @block = Saper::Recipe.new(id, options), block
+      end
+      
+      def to_recipe
+        self.instance_eval(&@block) if recipe.empty?; recipe
+      end
+      
+      def method_missing(name, *args, &block)
+        @recipe << Saper::Action.new(name, *args, :namespace => @recipe.namespace, &block)
+      end
+      
+    end
+  end
+end