sinew 1.0.4 → 2.0.0

data/test/test_requests.rb

@@ -0,0 +1,135 @@
+require_relative 'test_helper'
+
+class TestRequests < MiniTest::Test
+  def test_user_agent
+    sinew.dsl.get('http://httpbin.org/get', a: 1, b: 2)
+    assert_match(/sinew/, sinew.dsl.json[:headers][:'User-Agent'])
+  end
+
+  def test_basic_methods
+    sinew.dsl.get('http://httpbin.org/get', a: 1, b: 2)
+    assert_equal({ a: '1', b: '2' }, sinew.dsl.json[:args])
+
+    sinew.dsl.post('http://httpbin.org/post', a: 1, b: 2)
+    assert_equal({ a: '1', b: '2' }, sinew.dsl.json[:form])
+
+    sinew.dsl.post_json('http://httpbin.org/post', a: 1, b: 2)
+    assert_equal({ a: 1, b: 2 }, sinew.dsl.json[:json])
+  end
+
+  def test_custom_headers
+    sinew.dsl.http('get', 'http://httpbin.org/get', headers: { "User-Agent": '007' })
+    assert_match(/007/, sinew.dsl.json[:headers][:'User-Agent'])
+  end
+
+  def test_redirects
+    # absolute redirect
+    sinew.dsl.get('http://httpbin.org/redirect/2')
+    assert_equal 'http://httpbin.org/get', sinew.dsl.url
+
+    # and relative redirect
+    sinew.dsl.get('http://httpbin.org/relative-redirect/2')
+    assert_equal 'http://httpbin.org/get', sinew.dsl.url
+  end
+
+  def test_errors
+    skip if test_network?
+
+    # 500
+    assert_output(/failed with 500/) do
+      sinew.dsl.get('http://httpbin.org/status/500')
+      assert_equal '500', sinew.dsl.raw
+    end
+
+    # timeout
+    assert_output(/failed with 999/) do
+      sinew.dsl.get('http://httpbin.org/delay/1')
+      assert_equal 'timeout', sinew.dsl.raw
+    end
+  end
+
+  def test_retry_timeout
+    skip if test_network?
+
+    errors = 2
+    stub_request(:get, %r{http://[^/]+/error}).to_return do
+      if errors > 0
+        errors -= 1
+        raise Timeout::Error
+      end
+      { body: 'done', status: 200 }
+    end
+    sinew.dsl.get('http://httpbin.org/error')
+    assert_equal 0, errors
+    assert_equal 'done', sinew.dsl.raw
+  end
+
+  def test_retry_500
+    skip if test_network?
+
+    errors = 2
+    stub_request(:get, %r{http://[^/]+/error}).to_return do
+      if errors > 0
+        errors -= 1
+        return { status: 500 }
+      end
+      { body: 'done', status: 200 }
+    end
+    sinew.dsl.get('http://httpbin.org/error')
+    assert_equal 0, errors
+    assert_equal 'done', sinew.dsl.raw
+  end
+
+  def test_cache_key
+    # empty
+    req = Sinew::Request.new(sinew, 'get', 'http://host')
+    assert_equal req.cache_key, 'host/_root_'
+
+    # path
+    req = Sinew::Request.new(sinew, 'get', 'http://host/path')
+    assert_equal req.cache_key, 'host/path'
+
+    # query
+    req = Sinew::Request.new(sinew, 'get', 'http://host/path', query: { a: 'b' })
+    assert_equal req.cache_key, 'host/path,a=b'
+
+    # post with body
+    req = Sinew::Request.new(sinew, 'post', 'http://host/path', body: { c: 'd' })
+    assert_equal req.cache_key, 'host/post,path,c=d'
+
+    # too long should turn into digest
+    path = 'xyz' * 123
+    req = Sinew::Request.new(sinew, 'get', "http://host/#{path}")
+    assert_equal "host/#{Digest::MD5.hexdigest(path)}", req.cache_key
+  end
+
+  def test_before_generate_cache_key
+    sinew.runtime_options.before_generate_cache_key = method(:redact_cache_key)
+    req = Sinew::Request.new(sinew, 'get', 'http://host', query: { secret: 'xyz' })
+    assert_equal 'host/secret=redacted', req.cache_key
+  end
+
+  def test_urls
+    # simple
+    req = Sinew::Request.new(sinew, 'get', 'https://host')
+    assert_equal 'https://host', req.uri.to_s
+
+    # with query
+    req = Sinew::Request.new(sinew, 'get', 'https://host', query: { a: 1 })
+    assert_equal 'https://host?a=1', req.uri.to_s
+
+    # entity decoding
+    req = Sinew::Request.new(sinew, 'get', 'https://host?a=&lt;5')
+    assert_equal 'https://host?a=%3C5', req.uri.to_s
+
+    # sloppy urls
+    req = Sinew::Request.new(sinew, 'get', 'https://host?a=b c&d=f\'g')
+    assert_equal 'https://host?a=b%20c&d=f%27g', req.uri.to_s
+  end
+
+  def redact_cache_key(key)
+    key[:query].gsub!(/secret=[^&]+/, 'secret=redacted')
+    key
+  end
+  protected :redact_cache_key
+end

data/test/test_utf8.rb

@@ -0,0 +1,39 @@
+require_relative 'test_helper'
+
+class TestRequests < MiniTest::Test
+  def test_get
+    # network (or stub)
+    sinew.dsl.get('http://httpbin.org/get')
+    assert_equal 'UTF-8', sinew.dsl.raw.encoding.name
+
+    # disk
+    sinew.dsl.get('http://httpbin.org/get')
+    assert_equal 'UTF-8', sinew.dsl.raw.encoding.name
+  end
+
+  def test_utf8
+    skip if !test_network?
+
+    # network
+    sinew.dsl.get('http://httpbin.org/encoding/utf8')
+    assert_equal 'UTF-8', sinew.dsl.raw.encoding.name
+    assert_match(/∑/, sinew.dsl.raw)
+
+    # disk
+    sinew.dsl.get('http://httpbin.org/encoding/utf8')
+    assert_equal 'UTF-8', sinew.dsl.raw.encoding.name
+    assert_match(/∑/, sinew.dsl.raw)
+  end
+
+  def test_encode
+    skip if !test_network?
+
+    # network
+    sinew.dsl.get('https://www.google.co.jp')
+    assert_equal 'UTF-8', sinew.dsl.raw.encoding.name
+
+    # disk
+    sinew.dsl.get('https://www.google.co.jp')
+    assert_equal 'UTF-8', sinew.dsl.raw.encoding.name
+  end
+end

metadata

@@ -1,113 +1,155 @@
 --- !ruby/object:Gem::Specification
 name: sinew
 version: !ruby/object:Gem::Version
-  version: 1.0.4
+  version: 2.0.0
 platform: ruby
 authors:
 - Adam Doppelt
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-10 00:00:00.000000000 Z
+date: 2018-05-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: activesupport
+  name: awesome_print
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '1.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '1.8'
 - !ruby/object:Gem::Dependency
-  name: awesome_print
+  name: htmlentities
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.3'
 - !ruby/object:Gem::Dependency
-  name: htmlentities
+  name: httparty
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.16'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.16'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: scripto
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: stringex
+  name: slop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '4.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '4.6'
 - !ruby/object:Gem::Dependency
-  name: trollop
+  name: stringex
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.8'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.11'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '12.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '12.3'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
 description: Crawl web sites easily using ruby recipes, with caching and nokogiri.
 email:
 - amd@gurge.com
@@ -116,29 +158,40 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
+- ".rubocop.yml"
+- ".travis.yml"
+- ".vscode/extensions.json"
+- ".vscode/settings.json"
 - Gemfile
 - LICENSE
 - README.md
 - Rakefile
 - bin/sinew
 - lib/sinew.rb
-- lib/sinew/curler.rb
+- lib/sinew/cache.rb
+- lib/sinew/core_ext.rb
+- lib/sinew/dsl.rb
 - lib/sinew/main.rb
 - lib/sinew/nokogiri_ext.rb
-- lib/sinew/text_util.rb
-- lib/sinew/util.rb
+- lib/sinew/output.rb
+- lib/sinew/request.rb
+- lib/sinew/response.rb
+- lib/sinew/runtime_options.rb
 - lib/sinew/version.rb
 - sample.sinew
 - sinew.gemspec
-- test/helper.rb
 - test/test.html
-- test/test_curler.rb
+- test/test_cache.rb
+- test/test_helper.rb
 - test/test_main.rb
 - test/test_nokogiri_ext.rb
-- test/test_text_util.rb
+- test/test_output.rb
+- test/test_requests.rb
+- test/test_utf8.rb
 homepage: http://github.com/gurgeous/sinew
-licenses: []
+licenses:
+- MIT
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -146,24 +199,26 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - "~>"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: sinew
-rubygems_version: 2.0.6
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Sinew - structured web crawling using recipes.
 test_files:
-- test/helper.rb
 - test/test.html
-- test/test_curler.rb
+- test/test_cache.rb
+- test/test_helper.rb
 - test/test_main.rb
 - test/test_nokogiri_ext.rb
-- test/test_text_util.rb
+- test/test_output.rb
+- test/test_requests.rb
+- test/test_utf8.rb

data/lib/sinew/curler.rb

@@ -1,173 +0,0 @@
-require "uri"
-
-module Sinew
-  class Curler
-    class Error < StandardError ; end
-    
-    DEFAULT_OPTIONS = {
-      :cache_errors => true,
-      :max_time => 30,
-      :retry => 3,
-      :verbose => true,
-    }
-    
-    attr_reader :url, :uri, :root
-
-    def initialize(options = {})
-      @options = DEFAULT_OPTIONS.merge(options)
-      @curl_args = ["--silent", "--fail", "--user-agent", @options[:user_agent], "--max-time", @options[:max_time], "--retry", @options[:retry], "--location", "--max-redirs", "3"]
-      @last_request = Time.at(0)      
-
-      @root = @options[:dir]
-      if !@root
-        if File.exists?(ENV["HOME"]) && File.stat(ENV["HOME"]).writable?
-          @root = "#{ENV["HOME"]}/.sinew"
-        else
-          @root = "/tmp/sinew"
-        end
-      end
-    end
-
-    def get(url)
-      curl(url, nil)
-    end
-
-    def post(url, body)
-      curl(url, body)
-    end
-    
-    def curl(url, body)
-      #
-      # prepare url/uri and calculate paths
-      #
-
-      @uri = url.is_a?(URI) ? url : Curler.url_to_uri(url.to_s)
-      @url = @uri.to_s
-
-      path = fullpath(@uri)
-      path = "#{path},#{Util.pathify(body)}" if body
-
-      # shorten long paths
-      if path.length > 250
-        dir, base = File.dirname(path), File.basename(path)
-        path = "#{dir}/#{Util.md5(base)}"
-      end
-      
-      head = "#{File.dirname(path)}/head/#{File.basename(path)}"
-
-      if !File.exists?(path)
-        verbose(body ? "curl #{@url} (POST)" : "curl #{@url}")
-        tmp = "/tmp/curler_#{Util.random_text(6)}"
-        tmph = "#{tmp}.head"
-        begin
-          rate_limit
-          Util.mkdir_if_necessary(File.dirname(path))
-          Util.mkdir_if_necessary(File.dirname(head))        
-          begin
-            command = []
-            command += @curl_args
-            if body
-              command += ["--data-binary", body]
-              command += ["--header", "Content-Type: application/x-www-form-urlencoded"]
-            end
-            command += ["--output", tmp]
-            command += ["--dump-header", tmph]
-            command << @url
-            
-            Util.run("curl", command)
-
-            # empty response?
-            if !File.exists?(tmp)
-              Util.touch(tmp)
-              Util.touch(tmph)            
-            end
-          rescue Util::RunError => e
-            message = "curl error"
-            if e.message =~ /(\d+)$/
-              message = "#{message} (#{$1})"
-            end
-            
-            # cache the error?
-            if @options[:cache_errors]
-              File.open(path, "w") { |f| f.puts "" }
-              File.open(head, "w") { |f| f.puts "CURLER_ERROR\t#{message}" }
-            end
-            
-            raise Error, message
-          end
-          Util.mv(tmp, path)
-          Util.mv(tmph, head)
-        ensure
-          Util.rm_if_necessary(tmp)
-          Util.rm_if_necessary(tmph)
-        end
-      end
-      
-      #
-      # handle redirects (recalculate @uri/@url)
-      #
-
-      if File.exists?(head)
-        head_contents = File.read(head)
-        # handle cached errors
-        if head_contents =~ /^CURLER_ERROR\t(.*)/
-          raise Error, $1
-        end
-        original = @uri
-        head_contents.scan(/\A(HTTP\/\d\.\d (\d+).*?\r\n\r\n)/m) do |i|
-          headers, code = $1, $2
-          if code =~ /^3/
-            if redir = headers[/^Location: ([^\r\n]+)/, 1]
-              @uri += redir
-              @url = @uri.to_s
-            end
-          end
-        end
-        # kill unnecessary head files
-        if original == @uri
-          Util.rm(head)
-        end
-      end
-      
-      path
-    end
-    
-    def verbose(s)
-      $stderr.puts s if @options[:verbose]
-    end
-
-    #
-    # helpers
-    #
-
-    def fullpath(uri)
-      "#{@root}/#{Curler.uri_to_path(uri)}"  
-    end
-
-    def uncache!(url)
-      Util.rm_if_necessary("#{@root}/#{Curler.url_to_path(url)}")
-    end
-
-    def self.url_to_uri(url)
-      url = url.gsub(" ", "%20")
-      url = url.gsub("'", "%27")
-      URI.parse(url)
-    end
-    
-    def self.url_to_path(url)
-      uri_to_path(url_to_uri(url))
-    end
-
-    def self.uri_to_path(uri)
-      s = uri.path
-      s = "#{s}?#{uri.query}" if uri.query
-      "#{Util.pathify(uri.host)}/#{Util.pathify(s)}"
-    end
-    
-    def rate_limit
-      sleep = (@last_request + 1) - Time.now
-      sleep(sleep) if sleep > 0
-      @last_request = Time.now
-    end
-  end
-end