saper 0.5.2 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7aba132c13d3bc80c0034121a49cb2c45841a5c2
-  data.tar.gz: 12e4ca23e2d339a1951e2495f7791444160f3daa
+  metadata.gz: fd213bbb0ccdf59a2743bea247a3466463d71e14
+  data.tar.gz: 682c8ef1d14f62253819905414c964837f8818d9
 SHA512:
-  metadata.gz: 0ff6858a582e44d4e61682a014e351d688d9b303870b516a897dd48c50ab66efddd42d3077d2cc87470da4cd4173faeefbc13f3375df214353ce1d17869d13f7
-  data.tar.gz: f8cc29550eaa47a9040fbebbf659dddd627b470dfbcd4d2a42998741d5d4aab187b2b49bcfe813279b2e3eff3989be4fdc0ccb36d47e5e3f9b0a63777cdd0414
+  metadata.gz: b31225725e71ed95b1fe7af32a912755ac5d0198acd8e5a0e968c31c4ef97dc517ed0ce786ec6e9992975809c7493632d42a1de03dcd37d413e6318a9598d6d0
+  data.tar.gz: 9c656ccb71b79b7b708db2b7f9116af6c3675a9df3cf2a7fdc1dda7b26fda64c5572481ac8474434e2b0f2c483786f3c38ea6ac195f98c63ee240786c33a3de8

data/README.md

@@ -11,6 +11,7 @@ Common use scenarios:
 
 [![Gem Version](https://badge.fury.io/rb/saper.svg)](http://badge.fury.io/rb/saper)
 [![Build Status](https://travis-ci.org/merimond/saper.svg?branch=master)](https://travis-ci.org/merimond/saper)
+[![Code Climate](https://codeclimate.com/github/merimond/saper/badges/gpa.svg)](https://codeclimate.com/github/merimond/saper)
 
 ## Installing
 
@@ -43,7 +44,7 @@ Given that file is saved as _myrecipe.txt_, you can now use the command line:
 Alternatively, you can use Ruby:
 
     #!/usr/bin/env ruby
-    Saper.run("myrecipe.txt", :bloomberg).results
+    Saper.run_from_file("myrecipe.txt", :bloomberg).serialize
 
 ## Data flow
 
@@ -109,7 +110,7 @@ Below is a list of all available actions:
 
 - **set_input** (string) - sets input for the following action.
 - **create_atom** - create an Atom from saved variables.
-- **run_recipe_and_save** (variable, recipe) - run another recipe and save its result as a variable.
+- **run_recipe_and_save** (recipe, variable) - run another recipe and save its result as a variable.
 - **run_recipe** (recipe) - run another recipe and use it's output as input for the next action.
 - **save** (variable) - save input as a variable.
 
@@ -126,4 +127,4 @@ Below is a list of all available actions:
 
 Copyright (c) 2013 Merimond Corporation. MIT license, see [LICENSE] for details.
 
-[LICENSE]: http://github.com/merimond/saper/blob/master/LICENSE
+[LICENSE]: http://github.com/merimond/saper/blob/master/LICENSE

data/Rakefile

@@ -1,14 +1,16 @@
 require 'rubygems'
 require 'bundler'
-require 'rake/testtask'
+#require 'rake/testtask'
 require 'yard'
 
 Bundler.require
 Bundler::GemHelper.install_tasks
 
-desc "Run tests"
-Rake::TestTask.new do |t|
-  t.test_files = FileList['spec/**/*_spec.rb']
+begin
+  require 'rspec/core/rake_task'
+  RSpec::Core::RakeTask.new(:spec)
+rescue LoadError
+  # no rspec available
 end
 
 desc "Generate docs"
@@ -16,4 +18,4 @@ YARD::Rake::YardocTask.new do |t|
   t.files = ['lib/saper/**/*.rb']
 end
 
-task :default => :test
+task :default => :spec

data/bin/saper

@@ -14,6 +14,7 @@ Options are ...
   opt :input,    "Recipe input",  :type => :string
   opt :agent,    "User-Agent",    :type => :string
   opt :keychain, "Keychain file", :type => :string
+  opt :verbose,  "Verbose output"
 end
 
 if opts[:recipe].nil?
@@ -24,8 +25,57 @@ unless ARGV.size == 1
   Trollop::die "Please specify filename"
 end
 
+def bold(string)
+  puts "\n\033[1m%s\033[0m" % string
+end
+
+def red(string)
+  puts "\033[31m%s\033[0m" % string
+end
+
+def green(string)
+  puts "\033[32m%s\033[0m" % string
+end
+
+def print_value(value)
+  puts "> %s" % value.class
+end
+
+def print_action_name(action, error = false)
+  error ? red(action.name) : green(action.name)
+end
+
+def print_each_action(stack, actions)
+  actions.each_with_index do |action, i|
+    print_value stack.to_a[i]
+    print_action_name action, stack.to_a[i+1].is_a?(Saper::Error)
+  end
+end
+
+def print_each_stack(result, actions)
+  result.stacks.each_with_index do |stack, i|
+    bold "STACK %s / %s" % [i + 1, result.size]
+    print_each_action(stack, actions)
+  end
+end
+
+if opts[:verbose]
+  bold "ACTION LOG"
+  opts[:logger] = Saper::Logger.new($stdout)
+end
+
 begin
-  puts Saper.run_from_file(ARGV.first, opts.delete(:recipe), opts.delete(:input), opts).serialize
+  recipe = Saper.load(ARGV.first)[opts[:recipe]]
+  result = recipe.run(opts[:input], opts)
+  if opts[:verbose]
+    bold "RESULTS: %s" % result.size
+  end
+  result.to_serialized_array.each do |result|
+    result.nil? ? red("null") : green(result)
+  end
+  if opts[:verbose] && result.failure?
+    print_each_stack(result, recipe.actions)
+  end
 rescue Errno::ENOENT => e
   Trollop::die("File not found: %s" % ARGV.first)
 rescue NoMethodError => e

data/lib/saper/actions/create_atom.rb

@@ -12,11 +12,6 @@ module Saper
         end
       end
       
-      # Overrides default logic
-      def requires
-        []
-      end
-      
     end
   end
 end

data/lib/saper/actions/nothing.rb

@@ -2,6 +2,8 @@ module Saper
   module Actions
     class Nothing < Action
       
+      accepts :anything
+      
       run do |runtime, input|
         input
       end

data/lib/saper/actions/remove_matching.rb

@@ -6,7 +6,7 @@ module Saper
       accepts :text, :returns => :text
       
       run do |runtime, input, pattern|
-        input.to_s =~ Regexp.new(pattern) ? Items::Nothing.new : input
+        input.to_s =~ Regexp.new(pattern) ? Items::Nothing.new(:filtered => true) : input
       end
       
     end

data/lib/saper/actions/run_recipe.rb

@@ -3,20 +3,20 @@ module Saper
     class RunRecipe < Action
       
       argument :recipe
-      accepts :anything, :returns => Proc.new { |instance| nil } # TODO
+      accepts :anything, :returns => Proc.new { |instance| instance.args.first.requires }
       
       run do |runtime, input, subrecipe|
         subrecipe.run(input).to_saper
       end
       
       # Overrides default logic
-      def multiple?
-        args.first.multiple?
+      def accepts?(type)
+        args.first.accepts?(type)
       end
       
       # Overrides default logic
-      def requires
-        args.first.input_required
+      def multiple?
+        args.first.multiple?
       end
       
     end

data/lib/saper/actions/run_recipe_and_save.rb

@@ -2,19 +2,19 @@ module Saper
   module Actions
     class RunRecipeAndSave < Action
       
-      argument :variable
       argument :recipe
-      accepts :anything
+      argument :variable
+      accepts :anything, :returns => Proc.new { |instance| instance.args.first.requires }
       
-      run do |runtime, input, variable, subrecipe|
+      run do |runtime, input, subrecipe, variable|
         result = subrecipe.run(input).to_saper
         runtime[variable] = result unless result.nil?
         input
       end
       
       # Overrides default logic
-      def requires
-        args.last.input_required
+      def accepts?(type)
+        args.first.accepts?(type)
       end
       
     end

data/lib/saper/actions/select_matching.rb

@@ -6,7 +6,7 @@ module Saper
       accepts :text, :returns => :text
       
       run do |runtime, input, pattern|
-        input.to_s =~ Regexp.new(pattern) ? input : Items::Nothing.new
+        input.to_s =~ Regexp.new(pattern) ? input : Items::Nothing.new(:filtered => true)
       end
       
     end

data/lib/saper/actions/set_input.rb

@@ -9,11 +9,6 @@ module Saper
         string
       end
       
-      # Overrides default logic
-      def requires
-        []
-      end
-      
     end
   end
 end

data/lib/saper/core/action.rb

@@ -1,6 +1,8 @@
 module Saper
   class Action
     
+    INPUT_TYPES = %i(text atom document html json markdown nothing time url xml)
+    
     # Tracks subclasses of Saper::Action.
     # @return [Class]
     def self.inherited(base)
@@ -52,47 +54,17 @@ module Saper
         return @types.nil? ? [] : @types.keys
       end
       if input == :anything
-        return @types = {}
+        INPUT_TYPES.each { |type| accepts(type, options) }
       end
       output = options[:returns] || input
-      unless Item.exists?(input)
-        raise(InvalidInput, "Invalid input type: %s" % input)
-      end
-      if output.is_a?(Symbol) && !Item.exists?(output)
-        raise(InvalidInput, "Invalid output type: %s" % output)
-      end
       @types ||= {}
       @types[input] = output
     end
     
-    # Returns `true` if action accepts no input.
-    # @return [Boolean]
-    def self.accepts_nothing?
-      @types.nil?
-    end
-
-    # Returns `true` if action accepts at least some kind of input.
-    # @return [Boolean]
-    def self.accepts_something?
-      !accepts_nothing? && !accepts_anything?
-    end
-    
-    # Returns `true` if action accepts any kind of input.
-    # @return [Boolean]
-    def self.accepts_anything?
-      @types.empty?
-    end
-    
     # Returns `true` if action accepts specified type as input.
     # @param type [Symbol]
     # @return [Boolean]
     def self.accepts?(type)
-      if accepts_nothing?
-        return false
-      end
-      if accepts_anything?
-        return true
-      end
       accepts.include?(type)
     end
     
@@ -102,6 +74,13 @@ module Saper
       @block ||= block
     end
     
+    # Returns the type of output for specfied input type.
+    # @param type [Symbol] argument type (e.g. :text, :xpath)
+    # @return [Array<Symbol>]
+    def self.output_for(type)
+      accepts?(type) ? @types[type] : :nothing
+    end
+    
     # Returns `true` if action returns multiple items. Note that this method will report incorrect data for some actions. Use #multiple? instead.
     def self.returns_multiple_items?
       @multiple == true
@@ -168,25 +147,9 @@ module Saper
     # @return [void] depends on action type
     def run(input = nil, runtime = nil)
       runtime ||= Runtime.new
-      input = validate_input(input)
-      block.call(runtime, input, *args)
-    end
-    
-    # @todo
-    def validate_input(input)
-      unless self.class.accepts_something?
-        return input
-      end
-      unless input.is_a?(Item)
-        input = self.class.accepts.map { |type| Item.try(type, input) }.compact.first
-      end
-      if input.nil?
-        raise(InvalidInput, input)
-      end
-      unless self.class.accepts?(input.type)
-        raise(InvalidInput, input)
-      end
-      input
+      input  = validate_input(input)
+      output = block.call(runtime, input, *args)
+      validate_output output, output_for(input.type)
     end
     
     # Returns human readable action name.
@@ -201,10 +164,18 @@ module Saper
       @options[:namespace].is_a?(Namespace) ? @options[:namespace] : nil
     end
     
-    # Returns a list of data types that are accepted as input.
+    # Returns `true` if action accepts specified type as input.
+    # @param type [Symbol]
+    # @return [Boolean]
+    def accepts?(type)
+      self.class.accepts.include?(type)
+    end
+    
+    # Returns the type of output for specfied input type.
+    # @param type [Symbol] argument type (e.g. :text, :xpath)
     # @return [Array<Symbol>]
-    def requires
-      self.class.accepts
+    def output_for(type)
+      out = self.class.output_for(type); out.is_a?(Proc) ? out.call(self) : out
     end
     
     # Returns values of action arguments.
@@ -219,6 +190,12 @@ module Saper
       { :type => self.class.type, :args => @arguments.map(&:serialize) }
     end
     
+    # Returns a JSON representation of this action.
+    # @return [String]
+    def to_json(*args)
+      serialize.to_json(*args)
+    end
+    
     # Returns `true` if action returns multiple items.
     # @return [Boolean]
     def multiple?
@@ -237,5 +214,44 @@ module Saper
       "\t%s %s" % [self.class.type, @arguments.map(&:to_string).join(", ")]
     end
     
+    private
+    
+    # @todo
+    def validate_output(output, type)
+      if output.is_a?(Array) && output.empty?
+        raise InvalidOutput
+      end
+      if output.is_a?(Array)
+        return output.map { |o| validate_output(o, type) }
+      end
+      if output.is_a?(Items::Nothing) && output.filtered?
+        return output
+      end
+      unless output.is_a?(Item)
+        output = Item.try(type, output)
+      end
+      if output.is_a?(Item) && output.type == type
+        return output
+      end
+      raise InvalidOutput
+    end
+    
+    # @todo
+    def validate_input(input)
+      unless input.is_a?(Item)
+        input = self.class.accepts.map { |type| Item.try(type, input) }.compact.first
+      end
+      if input.nil? && accepts?(:nothing)
+        return Items::Nothing.new
+      end
+      if input.nil?
+        raise(InvalidInput, input)
+      end
+      unless accepts?(input.type)
+        raise(InvalidInput, input)
+      end
+      input
+    end
+    
   end
 end

data/lib/saper/core/argument.rb

@@ -60,6 +60,9 @@ module Saper
     # @param value [Object]
     # @return [self]
     def set(value)
+      if value.is_a?(Saper::Argument)
+        value = value.value
+      end
       unless valid?(value)
         raise InvalidArgument, value
       else

data/lib/saper/core/browser.rb

@@ -52,6 +52,7 @@ module Saper
         @received += resp.to_hash.to_s.size
         @received += body.size
       end
+      @logger.new_browser(self)
     end
     
     # Returns the number of HTTP requests.
@@ -71,7 +72,7 @@ module Saper
     # @param query [Hash] query options
     # @return [Saper::Document]
     def get(url, query = {})
-      @logger.download(url)
+      @logger.new_get_request(url)
       @history.push url
       data = @mech.get(url, query)
       Saper::Items::Document.new data.body, data.uri, data.header
@@ -84,7 +85,7 @@ module Saper
     # @param query [Hash] payload
     # @return [Saper::Document]
     def post(url, query = {}, headers = {})
-      @logger.download(url)
+      @logger.new_post_request(url)
       @history.push url
       data = @mech.post(url, query, headers)
       Saper::Items::Document.new data.body, data.uri, data.header