maxwell 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4dee2d7fc3af9b8e4dc64b1b6588be1735aa9f35
-  data.tar.gz: cc1b31de45500795d3630bad15a2b4c0dc0b6117
+  metadata.gz: 0b4a07c10e6638f009e5d96fa34d98d2c92985aa
+  data.tar.gz: 641ac72166235db4b28e7ccebee77e8da1b7c3e0
 SHA512:
-  metadata.gz: ca1d52b57421703c92f3e2697f56dca22765baca6180b4e28da0669474b74d4ad580ff709c8934b3c173f44694d991e760c579f8c2469f1ad0770b839b7caf47
-  data.tar.gz: 7ad8eee5272c711110a5111a23b7ce450f4f5ceb5a1fe7b5943be57980d5ed1504d09f02707c9474b0c39df269db901e0e86124343ee6bf043339d6f639591b6
+  metadata.gz: 04d92bce282bbbf263c83b1434f05cbd9043a169bf893a360a80701755d645aef03c60af4a69187ac763c8a1fa95e1596854d175fd0aaa6d66c787c3eab36081
+  data.tar.gz: 8cfb57804bc5575573a30cb10166cf46b8c6761969ca8bbdef1accd758c16d0823a9c6c203103bf45a244303f9c54f8c26ff1f709425d3c944e7ec7767e7819b

data/README.md

@@ -21,19 +21,22 @@ Or install it yourself as:
 ## Usage
 
 ```ruby
-Maxwell::DO({
-  "root url" => {
-    "css selector to links" => {
-      "css selector to links" => ->(html) {
-        html.title
-        # write parser here
-      }
-    }
-  }
-}) do |result|
-  p result
-  # write code for handling results
+class YahooScraper < Maxwell::Base
+  attr_scrape :title, :url, :address
+
+  regist_strategy("h3.slcHead.cFix a") do
+    @title   = @html.title
+    @url     = @html.css("td.sdhk jdj").text
+    @address = @html.css("table tr.ddad").text
+  end
+
+  regist_handler do |result|
+    p result
+  end
+
 end
+
+YahooScraper.new.execute("https://www.yahoo.com/")
 ```
 
 ## Development

data/lib/maxwell/converter.rb

@@ -1,22 +1,45 @@
 require 'nokogiri'
 require 'httpclient'
 
-class Maxwell
-  module Converter
-    def self.execute(url)
-      client = HTTPClient.new(
-        default_header: {
-          "User-Agent" => "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"
-        }
-      )
+require 'nokogiri'
+require 'capybara'
+require 'capybara/poltergeist'
 
-      html = begin
-        client.get_content(url)
-      rescue
-        ""
+module Maxwell
+  class Converter
+    @user_agent = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"
+    class << self
+      def call(url, use_poltergeist=false)
+        use_poltergeist ? call_with_js(url) : call_without_js(url)
       end
 
-      Nokogiri::HTML(html)
+      def call_without_js(url)
+        client = HTTPClient.new(
+          default_header: {
+            "User-Agent" => @user_agent
+          }
+        )
+
+        html = begin
+          client.get_content(url)
+        rescue
+          ""
+        end
+
+        Nokogiri::HTML(html)
+      end
+
+      def call_with_js(url)
+        Capybara.register_driver :poltergeist do |app|
+          Capybara::Poltergeist::Driver.new(app, { js_errors: false, timeout: 1000 })
+        end
+        Capybara.default_selector = :xpath
+        session = Capybara::Session.new(:poltergeist)
+
+        session.driver.headers = { 'User-Agent' => @user_agent }
+        session.visit url
+        Nokogiri::HTML(session.html)
+      end
     end
   end
 end

data/lib/maxwell/version.rb

@@ -1,3 +1,3 @@
-class Maxwell
-  VERSION = "0.2.0"
+module Maxwell
+  VERSION = "0.3.0"
 end

data/lib/maxwell.rb

@@ -1,19 +1,23 @@
 require "maxwell/converter"
 
-class Maxwell
+module Maxwell
   class Base
     class << self
       def attr_scrape(*attr_scrapes)
         @acquirer_class = Class.new do
           attr_accessor *attr_scrapes
-          @@attributes = attr_scrapes
+          @attr_scrapes = attr_scrapes
+
+          def self.attr_scrapes
+            @attr_scrapes
+          end
 
           def initialize(nokogiri_obj)
             @html = nokogiri_obj
           end
 
           def result
-            @@attributes.map { |k| [k, send(k)]  }.to_h
+            self.class.attr_scrapes.map { |k| [k, send(k)]  }.to_h
           end
         end
       end
@@ -26,11 +30,15 @@ class Maxwell
       def regist_handler(&handler_blk)
         @handler_blk = handler_blk
       end
+
+      def use_poltergeist(value)
+        @use_poltergeist = value
+      end
     end
 
     def execute(root_url)
       if self.link_selectore
-        html = Maxwell::Converter.execute(root_url)
+        html = Maxwell::Converter.call(root_url, use_poltergeist)
         html.css(self.link_selectore).each do |a|
           execute_for_result a[:href]
         end
@@ -39,6 +47,10 @@ class Maxwell
       end
     end
 
+    def use_poltergeist
+      self.class.instance_eval("@use_poltergeist")
+    end
+
     def link_selectore
       self.class.instance_eval("@link_selectore")
     end
@@ -57,7 +69,7 @@ class Maxwell
 
     private
       def execute_for_result(tip_url)
-        acquirer = acquirer_class.new(Maxwell::Converter.execute(tip_url))
+        acquirer = acquirer_class.new(Maxwell::Converter.call(tip_url, use_poltergeist))
         acquirer.instance_eval &self.strategy_blk
 
         acquirer.result.tap do |result|

data/maxwell.gemspec

@@ -21,6 +21,8 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "nokogiri"
   spec.add_dependency "httpclient"
+  spec.add_dependency 'poltergeist'
+  spec.add_dependency 'capybara'
 
   spec.add_development_dependency "bundler", "~> 1.10"
   spec.add_development_dependency "rake", "~> 10.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: maxwell
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - gogotanaka
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-02-08 00:00:00.000000000 Z
+date: 2016-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -38,6 +38,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: poltergeist
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: capybara
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement