juknife 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b457a3c9dece497c1266292d703d066695faf656
-  data.tar.gz: c144c4631890a7c2702263090465ce4c900e5f84
+  metadata.gz: ad09b3333ab9843a6d35109af520c0ae6d15bd45
+  data.tar.gz: 54a7bdd6f233f1832d9749cccad7067bfa1189a8
 SHA512:
-  metadata.gz: 35111faac73c8ec38c8fdb11cb657f120d19ce3f3f05846e27f281f106343897000c563acf6f3f55acad0de109a31ddf7e7e319e831ecf6140647593cbdc60fa
-  data.tar.gz: 5a967a81ae83cff36c5f45dcb2a064b49b4fe73d6b8a1a401f2bb79638c07fe8c98c67f904fe49ee41c8df54732f38fb1d1f28f9c69bb00a3d695bf8bb8adc15
+  metadata.gz: 857e0b2836c4ae7299ddd8c7b1f86d430bf4c3df34dae6d92a32927cdc315e35d877e86e6349979beb36a2d1b116d0b447639fe29d08360e28ca687bcf9fc3c6
+  data.tar.gz: f6a53f1e907d93b53cd074a6d46855b4aede515385fa96ec5588bab82153373cf6969fce8850cbb0d6147d420c71d58e0eb972684d73c93f234cd50e12ee77eb

data/.codeclimate.yml

@@ -0,0 +1,20 @@
+engines:
+  rubocop:
+    enabled: true
+    channel: rubocop-0-48
+  duplication:
+    enabled: true
+    config:
+      languages:
+        - ruby
+  fixme:
+    enabled: true
+
+ratings:
+  paths:
+    - Gemfile.lock
+    - "**.rb"
+
+exclude_paths:
+  - spec/
+  - vendor/

data/.gitignore

@@ -10,3 +10,6 @@
 
 # rspec failure tracking
 .rspec_status
+
+# simplecov
+build

data/README.md

@@ -7,7 +7,7 @@ A Ruby library for Web scraping. It allows to describe scraping nodes and text s
 For example, scraping on Google search can be written as follows:
 
 ```ruby
-class GoogleSearchScraper < Juknife::Scraper
+class GoogleSearchAgent < Juknife::Agent
   request do
     get 'https://www.google.co.jp/search'
     user_agent 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)'
@@ -27,8 +27,8 @@ class GoogleSearchScraper < Juknife::Scraper
   end
 end
 
-scraper = GoogleSearchScraper.new
-scraper.scrape(query: 'test')
+agent = GoogleSearchAgent.new
+agent.scrape(query: 'test')
 # =>
 # {:results=>
 #   [

data/circle.yml

@@ -0,0 +1,3 @@
+machine:
+  ruby:
+    version: 2.4.1

data/juknife.gemspec

@@ -29,10 +29,13 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'activesupport', '~> 5.0.2'
   spec.add_runtime_dependency 'nokogiri', '~> 1.7'
   spec.add_runtime_dependency 'faraday', '~> 0.1'
+  spec.add_runtime_dependency 'faraday_middleware', '~> 0.11'
   spec.add_development_dependency 'bundler', '~> 1.14'
+  spec.add_development_dependency 'codeclimate-test-reporter'
+  spec.add_development_dependency 'pry'
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'simplecov'
   spec.add_development_dependency 'yard'
-  spec.add_development_dependency 'pry'
 end

data/lib/juknife.rb

@@ -1,10 +1,15 @@
 # frozen_string_literal: true
 
 require 'juknife/version'
+require 'active_support/dependencies/autoload'
 
 # The module that contains everything Juknife-related.
 module Juknife
-  autoload :Request, 'juknife/request'
-  autoload :Scraper, 'juknife/scraper'
-  autoload :Scraping, 'juknife/scraping'
+  extend ActiveSupport::Autoload
+
+  autoload :Agent
+  autoload :Error
+  autoload :Middleware
+  autoload :Request
+  autoload :Scraping
 end

data/lib/juknife/agent.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require 'faraday_middleware'
+
+module Juknife
+  # The agent to execute request and scraping to web page.
+  class Agent
+    def scrape(params = {})
+      response(params).body
+    end
+
+    def response(params)
+      connection(params).get
+    end
+
+    def connection(params)
+      @connection ||= Faraday.new do |b|
+        # request middlewares
+        b.use Middleware::RequestDSL, params, &self.class.request_dsl
+        b.use Faraday::Request::UrlEncoded
+
+        # response middlewares
+        b.use Middleware::ScrapingDSL, params, &self.class.scraping_dsl
+        b.use Middleware::HTMLParser
+        b.use Middleware::HTTPErrorHandler
+        b.use FaradayMiddleware::FollowRedirects
+
+        b.adapter :net_http
+      end
+    end
+
+    class << self
+      attr_reader :request_dsl, :scraping_dsl
+
+      def request(&block)
+        @request_dsl = block
+      end
+
+      def scraping(&block)
+        @scraping_dsl = block
+      end
+    end
+  end
+end

data/lib/juknife/error.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Juknife
+  class Error < StandardError
+  end
+
+  class HTTPClientError < Error
+  end
+
+  class HTTPServerError < Error
+  end
+end

data/lib/juknife/middleware.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Juknife
+  # :nodoc:
+  module Middleware
+    autoload :HTMLParser, 'juknife/middleware/html_parser'
+    autoload :HTTPErrorHandler, 'juknife/middleware/http_error_handler'
+    autoload :RequestDSL, 'juknife/middleware/request_dsl'
+    autoload :ScrapingDSL, 'juknife/middleware/scraping_dsl'
+  end
+end

data/lib/juknife/middleware/html_parser.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require 'faraday'
+require 'nokogiri'
+
+module Juknife
+  # :nodoc:
+  module Middleware
+    # A Fraday middleware to parse body string to Nokogiri document.
+    class HTMLParser < Faraday::Response::Middleware
+      def parse(body)
+        Nokogiri.parse(body)
+      end
+    end
+  end
+end

data/lib/juknife/middleware/http_error_handler.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'faraday'
+
+module Juknife
+  # :nodoc:
+  module Middleware
+    # A middleware to handle HTTP errors
+    class HTTPErrorHandler < Faraday::Response::Middleware
+      def on_complete(env)
+        case env[:status]
+        when 400..499
+          raise HTTPClientError, env[:status]
+        when 500..599
+          raise HTTPServerError, env[:status]
+        end
+      end
+    end
+  end
+end

data/lib/juknife/middleware/request_dsl.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'faraday'
+
+module Juknife
+  # :nodoc:
+  module Middleware
+    # A Faraday middleware to interpret Juknife request DSL
+    class RequestDSL < Faraday::Middleware
+      include Juknife::Request::DSL
+
+      attr_reader :params
+
+      def initialize(app, params = {}, &block)
+        super(app)
+        @params = params
+        instance_eval(&block)
+      end
+
+      def call(env) # rubocop: disable Metrics/AbcSize
+        uri = URI.parse(url_builder.call)
+        uri.query = query_builder&.call&.to_query
+
+        env[:url] = uri
+        env[:method] = http_method
+        env[:body] = body_builder&.call
+        env[:request_headers] ||= {}
+
+        if user_agent_builder
+          env[:request_headers]['User-Agent'] = user_agent_builder&.call
+        end
+
+        @app.call(env)
+      end
+    end
+  end
+end

data/lib/juknife/middleware/scraping_dsl.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'faraday'
+
+module Juknife
+  # :nodoc:
+  module Middleware
+    # A Faraday middleware to interpret Juknife scraping DSL
+    class ScrapingDSL < Faraday::Response::Middleware
+      include Juknife::Scraping::DSL
+
+      def initialize(app, params = {}, &block)
+        super(app)
+        @params = params
+        instance_eval(&block)
+      end
+
+      def parse(doc)
+        context = Juknife::Scraping::Context.new(doc)
+
+        children.each do |child|
+          child.visit(context)
+        end
+
+        context.result
+      end
+    end
+  end
+end

data/lib/juknife/request.rb

@@ -3,7 +3,9 @@
 module Juknife
   # :nodoc:
   module Request
-    autoload :DSL, 'juknife/request/dsl'
-    autoload :Runner, 'juknife/request/runner'
+    extend ActiveSupport::Autoload
+
+    autoload :DSL
+    autoload :Runner
   end
 end

data/lib/juknife/scraping.rb

@@ -3,8 +3,10 @@
 module Juknife
   # :nodoc:
   module Scraping
-    autoload :DSL, 'juknife/scraping/dsl'
-    autoload :Context, 'juknife/scraping/context'
-    autoload :Runner, 'juknife/scraping/runner'
+    extend ActiveSupport::Autoload
+
+    autoload :DSL
+    autoload :Context
+    autoload :Runner
   end
 end

data/lib/juknife/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Juknife
-  VERSION = '0.1.3'
+  VERSION = '0.2.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: juknife
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - nyamadori
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-05-11 00:00:00.000000000 Z
+date: 2017-05-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: faraday_middleware
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.11'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.11'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -66,6 +80,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: codeclimate-test-reporter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -109,7 +151,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: yard
+  name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -123,7 +165,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: yard
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -145,10 +187,10 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".codeclimate.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
-- ".travis.yml"
 - ".yardopts"
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -157,12 +199,19 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- circle.yml
 - juknife.gemspec
 - lib/juknife.rb
+- lib/juknife/agent.rb
+- lib/juknife/error.rb
+- lib/juknife/middleware.rb
+- lib/juknife/middleware/html_parser.rb
+- lib/juknife/middleware/http_error_handler.rb
+- lib/juknife/middleware/request_dsl.rb
+- lib/juknife/middleware/scraping_dsl.rb
 - lib/juknife/request.rb
 - lib/juknife/request/dsl.rb
 - lib/juknife/request/runner.rb
-- lib/juknife/scraper.rb
 - lib/juknife/scraping.rb
 - lib/juknife/scraping/context.rb
 - lib/juknife/scraping/dsl.rb

data/.travis.yml

@@ -1,8 +0,0 @@
-sudo: false
-language: ruby
-rvm:
-  - 2.4.1
-before_install: gem install bundler -v 1.14.6
-script:
-  - bundle exec rspec
-  - bundle exec rubocop

data/lib/juknife/scraper.rb

@@ -1,28 +0,0 @@
-# frozen_string_literal: true
-
-module Juknife
-  # Juknife::Scraper is a scraper to request a web page and to extract data.
-  class Scraper
-    def initialize
-      @scraping = Scraping::Runner.new(&self.class.scraping_block)
-      @request = Request::Runner.new(&self.class.request_block)
-    end
-
-    def scrape(params = {})
-      source = @request.run(params)
-      @scraping.run(source, params)
-    end
-
-    class << self
-      attr_reader :scraping_block, :request_block
-
-      def scraping(&block)
-        @scraping_block = block
-      end
-
-      def request(&block)
-        @request_block = block
-      end
-    end
-  end
-end