web_crawler 0.2.0 → 0.3.1

data/Gemfile

@@ -1,5 +1,8 @@
 source :gemcutter
 
+gem 'thor', '>=0.14.6'
+gem 'mime-types', '>=1.16'
+
 # Specify your gem's dependencies in web_crawler.gemspec
 gemspec
 

data/lib/web_crawler/batch_request.rb

@@ -20,6 +20,7 @@ module WebCrawler
         block_given? ? yield(@handler.process) : @handler.process
       else
         @responses ||= requests.map do |req|
+          WebCrawler.logger.info "start request to #{req.url.to_s}"
           block_given? ? yield(req.process) : req.process
         end
       end

data/lib/web_crawler/cli.rb

@@ -14,13 +14,16 @@ module WebCrawler
 
     class_option :format, type: :string, desc: "output format [json, xml, csv]", default: 'plain'
     class_option :json, type: :boolean, desc: "json output format. shortcut for --format json"
+    class_option :yaml, type: :boolean, desc: "yaml output format. shortcut for --format yaml"
     class_option :xml, type: :boolean, desc: "xml output format. shortcut for --format xml"
     class_option :csv, type: :boolean, desc: "csv output format. shortcut for --format csv"
     class_option :table, type: :boolean, desc: "table output format. shortcut for --format table"
     class_option :cached, type: :boolean, desc: "use cached requests. if ./tmp/cache exists use it for cache files"
     class_option :follow, type: :boolean, desc: "follow to urls on the pages"
     class_option :run, type: :string, desc: "run custom script with api access"
+    class_option :console, type: :boolean, desc: "run irb console after execution"
     class_option :log, type: :string, desc: "log file path"
+    class_option :output, type: :string, desc: "output file path"
 
     before_action except: :help do
       @options = options.dup
@@ -29,6 +32,7 @@ module WebCrawler
       @options[:format] = 'csv' if options[:csv]
       @options[:format] = 'table' if options[:table]
       @options[:format] = 'plain' if options[:plain]
+      @options[:format] = 'yaml' if options[:yaml]
 
       @options[:original_format] = @options[:format] if options[:run]
       @options[:format] = 'runner' if options[:run]

data/lib/web_crawler/response.rb

@@ -1,8 +1,10 @@
+require 'mime/types'
+
 module WebCrawler
   class Response
     extend ::Forwardable
 
-    delegate [:body, :http_version, :code, :message, :msg, :code_type, :[], :redirect_path, :redirect?] => '@response'
+    delegate [:http_version, :code, :message, :msg, :code_type, :[], :redirect_path, :redirect?] => '@response'
 
     attr_reader :url, :expire, :date, :cached
 
@@ -13,6 +15,14 @@ module WebCrawler
       @expire ||= Time.parse(self['Expires']) rescue Time.now
     end
 
+    [:xml, :html, :json].each do |type|
+      class_eval <<-RUBY, __FILE__, __LINE__ + 1
+        def #{type}?
+          mime_type.sub_type == '#{type}'
+        end
+      RUBY
+    end
+
     def set_cached_flag
       @cached = ' CACHED'
     end
@@ -36,8 +46,34 @@ module WebCrawler
           "#{redirected}>"
     end
 
+    def mime_type
+      MIME::Types[header['content-type']].first
+    end
+
+    def header
+      @header ||= Hash[@response.to_hash.map(&:flatten)]
+    end
+
+    def body
+      type, encoding = self['Content-Type'].split("=")
+      @body ||= if encoding.upcase == 'UTF-8'
+                  @response.body
+                else
+                  encode_body(encoding.upcase)
+                end
+    end
+
     alias :to_s :body
 
+    def encode_body(from)
+      require "iconv" unless defined?(Iconv)
+      encoded = Iconv.iconv('UTF-8', from, @response.body).first
+      if xml?
+        encoded = encoded.gsub(/<\?xml version="(.*?)" encoding=".*?"\?>/, "<?xml version=\"1.0\" encoding=\"utf-8\"?>")
+      end
+      encoded
+    end
+
     def type
       @response.class
     end

data/lib/web_crawler/version.rb

@@ -1,8 +1,8 @@
 module WebCrawler
   module VERSION
     MAJOR  = 0
-    MINOR  = 2
-    TINY   = 0
+    MINOR  = 3
+    TINY   = 1
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/lib/web_crawler/view/runner.rb

@@ -3,9 +3,22 @@ require "fileutils"
 module WebCrawler::View
   class Runner < Base
 
-    module Space
-      extend self
+    class WorkSpace
+      # array of responses
       attr_accessor :responses
+      attr_accessor :results
+
+      def q
+        exit
+      end
+
+      def returning(value)
+        self.results = value  
+      end
+
+      def method_missing(meth, *args, &block)
+        puts "\e[31m\e[1mError: method \"\e[0m\e[31m#{meth}\e[0m\e[31m\e[1m\" is missing\e[0m"
+      end
     end
 
     def render
@@ -13,8 +26,48 @@ module WebCrawler::View
         @options['run'] = File.expand_path @options['run'], FileUtils.pwd
       end
 
-      Space.responses = input.freeze
-      Space.module_eval(File.open(@options['run'], 'r').read)
+      @work_space = WorkSpace.new
+      @work_space.responses = input.freeze
+      @work_space.results   = eval(File.open(@options['run'], 'r').read, @work_space.instance_eval("binding"), @options['run'])
+
+      load_console! if @options['console']
+
+      WebCrawler::View.factory(@options['original_format'], @work_space.results, @options).render
+    end
+
+    def load_console! 
+      require "irb"
+      IRB.init_config nil
+      IRB.instance_exec do
+        @CONF[:BACK_TRACE_LIMIT] = 1
+
+        @CONF[:PROMPT][:SIMPLE] = { :PROMPT_I => "[\e[1m\e[31mWebCrawler::API\e[0m](%n)>> ",
+                                    :PROMPT_N => "[\e[1m\e[31mWebCrawler::API\e[0m](%n)>> ",
+                                    :PROMPT_S => "[\e[1m\e[31mWebCrawler::API\e[0m](%n)*",
+                                    :PROMPT_C => "(%n)?> ",
+                                    :RETURN   => "\e[90m#=> %s\n\e[0m" }
+
+        @CONF[:PROMPT_MODE] = :SIMPLE
+      end
+
+      irb = IRB::Irb.new IRB::WorkSpace.new(@work_space)
+
+
+      IRB.instance_exec { @CONF[:IRB_RC].call(irb.context) if @CONF[:IRB_RC] }
+      IRB.instance_exec { @CONF[:MAIN_CONTEXT] = irb.context }
+
+
+      trap("SIGINT") do
+        irb.signal_handle
+      end
+
+      begin
+        catch(:IRB_EXIT) do
+          irb.eval_input
+        end
+      ensure
+        IRB.irb_at_exit
+      end
     end
   end
 end

data/lib/web_crawler/view/yaml.rb

@@ -0,0 +1,9 @@
+require 'yaml'
+
+module WebCrawler::View
+  class Yaml < Base
+    def render
+      YAML.dump(responses: input)
+    end
+  end
+end

data/lib/web_crawler/view.rb

@@ -6,6 +6,7 @@ module WebCrawler::View
   autoload :Plain, 'web_crawler/view/plain'
   autoload :Table, 'web_crawler/view/table'
   autoload :Runner, 'web_crawler/view/runner'
+  autoload :Yaml, 'web_crawler/view/yaml'
 
   extend self
 
@@ -18,8 +19,9 @@ module WebCrawler::View
 
     class << self
       attr_accessor :default_options
+
       def default_options
-        @default_options ||= { }
+        @default_options ||= { 'output' => $stdout }
       end
     end
 
@@ -32,13 +34,29 @@ module WebCrawler::View
       [*input].map { |i| format(i) }.join
     end
 
-    def draw(output=$stdout)
-      output.puts render
+    def draw(output=nil)
+      begin
+        present_output(output).puts render
+      ensure
+        output.close if output.respond_to? :close
+      end
     end
 
     def format(item)
       item
     end
+
+    protected
+
+    def present_output(override=nil)
+      @present_output = if override && override.respond_to?(:puts)
+                          override
+                        elsif @options['output'].is_a?(String)
+                          File.open(@options['output'], 'w+')
+                        elsif @options['output'].respond_to? :puts
+                          @options['output']
+                        end
+    end
   end
 
 end

data/web_crawler.gemspec

@@ -23,7 +23,8 @@ Gem::Specification.new do |s|
 
   s.bindir = "bin"
 
-  s.add_dependency 'thor'
+  s.add_dependency 'thor', '>=0.14.6'
+  s.add_dependency 'mime-types', '>=1.16'
 
   s.add_development_dependency(%q<rspec>, [">=2.6"])
   s.add_development_dependency(%q<fakeweb>)

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
-  - 2
-  - 0
-  version: 0.2.0
+  - 3
+  - 1
+  version: 0.3.1
 platform: ruby
 authors: 
 - Anton Sozontov
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-05-31 00:00:00 +04:00
+date: 2011-06-03 00:00:00 +04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -27,13 +27,29 @@ dependencies:
       - !ruby/object:Gem::Version 
         segments: 
         - 0
-        version: "0"
+        - 14
+        - 6
+        version: 0.14.6
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  name: rspec
+  name: mime-types
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 16
+        version: "1.16"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -43,11 +59,11 @@ dependencies:
         - 6
         version: "2.6"
   type: :development
-  version_requirements: *id002
+  version_requirements: *id003
 - !ruby/object:Gem::Dependency 
   name: fakeweb
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -56,7 +72,7 @@ dependencies:
         - 0
         version: "0"
   type: :development
-  version_requirements: *id003
+  version_requirements: *id004
 description: Web crawler help you with parse and collect data from the web
 email: 
 - a.sozontov@gmail.com
@@ -104,6 +120,7 @@ files:
 - lib/web_crawler/view/runner.rb
 - lib/web_crawler/view/table.rb
 - lib/web_crawler/view/xml.rb
+- lib/web_crawler/view/yaml.rb
 - spec/fake_web_generator.rb
 - spec/spec_helper.rb
 - spec/web_crawler/batch_request_spec.rb