maxwell 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0b4a07c10e6638f009e5d96fa34d98d2c92985aa
-  data.tar.gz: 641ac72166235db4b28e7ccebee77e8da1b7c3e0
+  metadata.gz: 3062e2f1c6b55eb5cc7318da36e44986f1d73fd2
+  data.tar.gz: d95d0c7e51a9d0dfe383bca8d09c22b3c26431e6
 SHA512:
-  metadata.gz: 04d92bce282bbbf263c83b1434f05cbd9043a169bf893a360a80701755d645aef03c60af4a69187ac763c8a1fa95e1596854d175fd0aaa6d66c787c3eab36081
-  data.tar.gz: 8cfb57804bc5575573a30cb10166cf46b8c6761969ca8bbdef1accd758c16d0823a9c6c203103bf45a244303f9c54f8c26ff1f709425d3c944e7ec7767e7819b
+  metadata.gz: 1bc5e93ee5211c5c9a4c1a5ed6390e6969bba4912677b7ad8a9e015b81b5f9106b7c5bf8302ef9d673a8995f8bbce4ca0d9379d73668d13cb22a6598bd6a91f6
+  data.tar.gz: 7f0e764c5e8a6b8ac3e43ec03dc2efc2b199a61cc7fc5cbc77534496e2a3d52e49e66f7dbc1725963982991ed230323b48def1b4e6769eb017d091a50cb567c5

data/README.md

@@ -4,16 +4,10 @@ Maxwell makes web scraping more simpler and faster with Ruby.
 
 ## Installation
 
-Add this line to your application's Gemfile:
-
 ```ruby
-gem 'maxwell'
+echo "gem 'maxwell'" >> Gemfile && bundle
 ```
 
-And then execute:
-
-    $ bundle
-
 Or install it yourself as:
 
     $ gem install maxwell
@@ -22,21 +16,25 @@ Or install it yourself as:
 
 ```ruby
 class YahooScraper < Maxwell::Base
-  attr_scrape :title, :url, :address
+  attr_accessor :title, :url, :address
 
-  regist_strategy("h3.slcHead.cFix a") do
-    @title   = @html.title
-    @url     = @html.css("td.sdhk jdj").text
-    @address = @html.css("table tr.ddad").text
+  javascript true
+
+  concurrency 4
+
+  def parser html
+    @title   = html.title
+    @url     = html.css("td.sdhk jdj").text
+    @address = html.css("table tr.ddad").text
   end
 
-  regist_handler do |result|
+  def handler result
     p result
+    #=> { title: "...", url: "...", address: "..." }
   end
-
 end
 
-YahooScraper.new.execute("https://www.yahoo.com/")
+YahooScraper.execute ["https://www.yahoo.com/"]
 ```
 
 ## Development

data/lib/maxwell/converter.rb

@@ -38,7 +38,9 @@ module Maxwell
 
         session.driver.headers = { 'User-Agent' => @user_agent }
         session.visit url
-        Nokogiri::HTML(session.html)
+        Nokogiri::HTML(session.html).tap do
+          session.driver.quit
+        end
       end
     end
   end

data/lib/maxwell/helper.rb

@@ -0,0 +1,8 @@
+module Maxwell
+  class Helper
+    def self.open_links(url, link_selectore, use_poltergeist=false)
+      html = ::Maxwell::Converter.call(url, use_poltergeist)
+      html.css(link_selectore).map { |a| a[:href] }
+    end
+  end
+end

data/lib/maxwell/version.rb

@@ -1,3 +1,3 @@
 module Maxwell
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/lib/maxwell.rb

@@ -1,86 +1,62 @@
+require 'parallel'
+
 require "maxwell/converter"
+require "maxwell/helper"
 
 module Maxwell
+  class NoParserDefinedErr; end
+
   class Base
     class << self
-      def attr_scrape(*attr_scrapes)
-        @acquirer_class = Class.new do
-          attr_accessor *attr_scrapes
-          @attr_scrapes = attr_scrapes
+      def execute(urls)
+        Parallel.
+          map_with_index(urls, in_threads: @concurrency || 1) do |url, id|
+            p "scraping: #{ id + 1 }"
 
-          def self.attr_scrapes
-            @attr_scrapes
-          end
+            scraper = self.new
+            html = Maxwell::Converter.call(url, @use_poltergeist)
 
-          def initialize(nokogiri_obj)
-            @html = nokogiri_obj
-          end
+            scraper.parser html
 
-          def result
-            self.class.attr_scrapes.map { |k| [k, send(k)]  }.to_h
+            scraper.handler ({ id: id + 1 }).merge(scraper.result)
           end
-        end
       end
 
-      def regist_strategy(link_selectore=nil, &strategy_blk)
-        @link_selectore = link_selectore
-        @strategy_blk = strategy_blk
+      def attr_accessor(*attrs)
+        @attrs ||= []
+        @attrs.concat attrs
+        super
       end
 
-      def regist_handler(&handler_blk)
-        @handler_blk = handler_blk
+      def attrs
+        @attrs
       end
 
-      def use_poltergeist(value)
+      def javascript(value)
         @use_poltergeist = value
       end
-    end
 
-    def execute(root_url)
-      if self.link_selectore
-        html = Maxwell::Converter.call(root_url, use_poltergeist)
-        html.css(self.link_selectore).each do |a|
-          execute_for_result a[:href]
-        end
-      else
-        execute_for_result root_url
+      def concurrency(value)
+        @concurrency = value
       end
     end
 
-    def use_poltergeist
-      self.class.instance_eval("@use_poltergeist")
-    end
-
-    def link_selectore
-      self.class.instance_eval("@link_selectore")
-    end
-
-    def strategy_blk
-      self.class.instance_eval("@strategy_blk")
+    def parser html
+      raise NoParserDefinedErr "You need to define #{self}#parser"
     end
 
-    def handler_blk
-      self.class.instance_eval("@handler_blk")
+    def handler result
+      p result
     end
 
-    def acquirer_class
-      self.class.instance_eval("@acquirer_class")
+    def result
+      self.class.attrs.map { |k| [k, self.send(k)]  }.to_h
     end
-
-    private
-      def execute_for_result(tip_url)
-        acquirer = acquirer_class.new(Maxwell::Converter.call(tip_url, use_poltergeist))
-        acquirer.instance_eval &self.strategy_blk
-
-        acquirer.result.tap do |result|
-          self.handler_blk.call(result) if self.handler_blk
-        end
-      end
   end
 end
 
 class ::String
   def trim
-    delete("\r\n\t")
+    delete "\r\n\t"
   end
 end

data/maxwell.gemspec

@@ -23,6 +23,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "httpclient"
   spec.add_dependency 'poltergeist'
   spec.add_dependency 'capybara'
+  spec.add_dependency 'parallel'
 
   spec.add_development_dependency "bundler", "~> 1.10"
   spec.add_development_dependency "rake", "~> 10.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: maxwell
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - gogotanaka
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-02-18 00:00:00.000000000 Z
+date: 2016-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -128,6 +142,7 @@ files:
 - exe/maxwell
 - lib/maxwell.rb
 - lib/maxwell/converter.rb
+- lib/maxwell/helper.rb
 - lib/maxwell/version.rb
 - maxwell.gemspec
 homepage: http://gogotanaka.me/
@@ -150,7 +165,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: Maxwell makes web scraping more simpler and faster with Ruby.