sinew 2.0.1 → 3.0.0

data/lib/sinew/response.rb

@@ -1,5 +1,8 @@
+require 'stringio'
+require 'zlib'
+
 #
-# An HTTP response. Mostly a wrapper around HTTParty.
+# An HTTP response.
 #
 
 module Sinew
@@ -10,69 +13,36 @@ module Sinew
     # factory methods
     #
 
-    def self.from_network(request, party_response)
-      Response.new.tap do |response|
-        response.request = request
-        response.uri = party_response.request.last_uri
-        response.code = party_response.code
-        response.headers = party_response.headers.to_h
-
-        # force to utf-8 as best we can
-        body = party_response.body
-        if body.encoding != Encoding::UTF_8
-          body = body.encode('UTF-8', invalid: :replace, undef: :replace, replace: '?')
-        end
-        response.body = body
+    def self.from_network(request, fday_response)
+      Response.new.tap do
+        _1.request = request
+        _1.uri = fday_response.env.url
+        _1.code = fday_response.status
+        _1.headers = fday_response.headers.to_h
+        _1.body = process_body(fday_response)
       end
     end
 
-    def self.from_cache(request, body, head)
-      Response.new.tap do |response|
-        response.request = request
-        response.body = body
+    # helper for decoding bodies before parsing
+    def self.process_body(response)
+      body = response.body
 
-        # defaults
-        response.uri = request.uri
-        response.code = 200
-        response.headers = {}
+      # inflate if necessary
+      bits = body[0, 10].force_encoding('BINARY')
+      if bits =~ /\A\x1f\x8b/n
+        body = Zlib::GzipReader.new(StringIO.new(body)).read
+      end
 
-        # overwrite with cached response headers
-        if head
-          if head !~ /^{/
-            return from_legacy_head(response, head)
+      # force to utf-8 if we think this could be text
+      if body.encoding != Encoding::UTF_8
+        if content_type = response.headers['content-type']
+          if content_type =~ /\b(html|javascript|json|text|xml)\b/
+            body = body.encode('UTF-8', invalid: :replace, undef: :replace, replace: '?')
           end
-          head = JSON.parse(head, symbolize_names: true)
-          response.uri = URI.parse(head[:uri])
-          response.code = head[:code]
-          response.headers = head[:headers]
         end
       end
-    end
-
-    def self.from_timeout(request)
-      Response.new.tap do |response|
-        response.request = request
-        response.uri = request.uri
-        response.body = 'timeout'
-        response.code = 999
-        response.headers = {}
-      end
-    end
 
-    def self.from_legacy_head(response, head)
-      response.tap do |response|
-        case head
-        when /\ACURLER_ERROR/
-          # error
-          response.code = 999
-        when /\AHTTP/
-          # redirect
-          location = head.scan(/Location: ([^\r\n]+)/).flatten.last
-          response.uri += location
-        else
-          $stderr.puts "unknown cached /head for #{response.uri}"
-        end
-      end
+      body
     end
 
     #

data/lib/sinew/runtime_options.rb

@@ -7,7 +7,8 @@ module Sinew
     attr_accessor :retries
     attr_accessor :rate_limit
     attr_accessor :headers
-    attr_accessor :before_generate_cache_key
+    attr_accessor :httpdisk_options
+    attr_accessor :insecure
 
     def initialize
       self.retries = 3
@@ -15,7 +16,8 @@ module Sinew
       self.headers = {
         'User-Agent' => "sinew/#{VERSION}",
       }
-      self.before_generate_cache_key = ->(i) { i }
+      self.httpdisk_options = {}
+      self.insecure = false
 
       # for testing
       if ENV['SINEW_TEST']

data/lib/sinew/version.rb

@@ -1,4 +1,4 @@
 module Sinew
   # Gem version
-  VERSION = '2.0.1'.freeze
+  VERSION = '3.0.0'.freeze
 end

data/sample.sinew

@@ -1,4 +1,4 @@
-get 'http://httpbin.org'
+get 'http://httpbingo.org'
 noko.css('ul li a').each do |a|
   row = {}
   row[:url] = a[:href]
@@ -6,4 +6,4 @@ noko.css('ul li a').each do |a|
   csv_emit(row)
 end
 
-get 'http://httpbin.org/redirect/2'
+get 'http://httpbingo.org/redirect/2'

data/sinew.gemspec

@@ -5,30 +5,29 @@ require 'sinew/version'
 Gem::Specification.new do |s|
   s.name        = 'sinew'
   s.version     = Sinew::VERSION
-  s.platform    = Gem::Platform::RUBY
   s.license     = 'MIT'
-  s.authors     = [ 'Adam Doppelt' ]
+  s.authors     = [ 'Adam Doppelt', 'Nathan Kriege' ]
   s.email       = [ 'amd@gurge.com' ]
   s.homepage    = 'http://github.com/gurgeous/sinew'
   s.summary     = 'Sinew - structured web crawling using recipes.'
   s.description = 'Crawl web sites easily using ruby recipes, with caching and nokogiri.'
-  s.required_ruby_version = '~> 2.3'
+  s.required_ruby_version = '>= 2.7'
 
-  s.rubyforge_project = 'sinew'
+  # what's in the gem?
+  s.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { _1.match(%r{^test/}) }
+  end
+  s.bindir = 'bin'
+  s.executables = s.files.grep(%r{^#{s.bindir}/}) { File.basename(_1) }
+  s.require_paths = [ 'lib' ]
 
-  s.add_runtime_dependency 'awesome_print', '~> 1.8'
+  s.add_runtime_dependency 'amazing_print', '~> 1.3'
+  s.add_runtime_dependency 'faraday', '~> 1.4'
+  s.add_runtime_dependency 'faraday-encoding', '~> 0'
   s.add_runtime_dependency 'htmlentities', '~> 4.3'
-  s.add_runtime_dependency 'httparty', '~> 0.16'
-  s.add_runtime_dependency 'nokogiri', '~> 1.8'
+  s.add_runtime_dependency 'httpdisk', '~> 0'
+  s.add_runtime_dependency 'nokogiri', '~> 1.11'
   s.add_runtime_dependency 'scripto', '~> 0'
-  s.add_runtime_dependency 'slop', '~> 4.6'
-  s.add_runtime_dependency 'stringex', '~> 2.8'
-  s.add_development_dependency 'minitest', '~> 5.11'
-  s.add_development_dependency 'rake', '~> 12.3'
-  s.add_development_dependency 'webmock', '~> 3.4'
-
-  s.files         = `git ls-files`.split("\n")
-  s.test_files    = `git ls-files -- test/*`.split("\n")
-  s.executables   = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
-  s.require_paths = [ 'lib' ]
+  s.add_runtime_dependency 'slop', '~> 4.8'
+  s.add_runtime_dependency 'sterile', '~> 1.0'
 end

metadata

@@ -1,73 +1,74 @@
 --- !ruby/object:Gem::Specification
 name: sinew
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 3.0.0
 platform: ruby
 authors:
 - Adam Doppelt
-autorequire: 
+- Nathan Kriege
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-05-02 00:00:00.000000000 Z
+date: 2021-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: awesome_print
+  name: amazing_print
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '1.3'
 - !ruby/object:Gem::Dependency
-  name: htmlentities
+  name: faraday
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.3'
+        version: '1.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.3'
+        version: '1.4'
 - !ruby/object:Gem::Dependency
-  name: httparty
+  name: faraday-encoding
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.16'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.16'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: htmlentities
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '4.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '4.3'
 - !ruby/object:Gem::Dependency
-  name: scripto
+  name: httpdisk
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -81,75 +82,61 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: slop
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.6'
+        version: '1.11'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.6'
+        version: '1.11'
 - !ruby/object:Gem::Dependency
-  name: stringex
+  name: scripto
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.8'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.8'
-- !ruby/object:Gem::Dependency
-  name: minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.11'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.11'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: slop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
-  type: :development
+        version: '4.8'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '4.8'
 - !ruby/object:Gem::Dependency
-  name: webmock
+  name: sterile
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.4'
-  type: :development
+        version: '1.0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.4'
+        version: '1.0'
 description: Crawl web sites easily using ruby recipes, with caching and nokogiri.
 email:
 - amd@gurge.com
@@ -158,9 +145,9 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/test.yml"
 - ".gitignore"
 - ".rubocop.yml"
-- ".travis.yml"
 - ".vscode/extensions.json"
 - ".vscode/settings.json"
 - Gemfile
@@ -169,7 +156,9 @@ files:
 - Rakefile
 - bin/sinew
 - lib/sinew.rb
-- lib/sinew/cache.rb
+- lib/sinew/connection.rb
+- lib/sinew/connection/log_formatter.rb
+- lib/sinew/connection/rate_limit.rb
 - lib/sinew/core_ext.rb
 - lib/sinew/dsl.rb
 - lib/sinew/main.rb
@@ -181,56 +170,27 @@ files:
 - lib/sinew/version.rb
 - sample.sinew
 - sinew.gemspec
-- test/legacy/eu.httpbin.org/head/redirect,3
-- test/legacy/eu.httpbin.org/head/status,500
-- test/legacy/eu.httpbin.org/redirect,3
-- test/legacy/eu.httpbin.org/status,500
-- test/legacy/legacy.sinew
-- test/test.html
-- test/test_cache.rb
-- test/test_helper.rb
-- test/test_legacy.rb
-- test/test_main.rb
-- test/test_nokogiri_ext.rb
-- test/test_output.rb
-- test/test_requests.rb
-- test/test_utf8.rb
 homepage: http://github.com/gurgeous/sinew
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - "~>"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: '2.3'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: sinew
-rubygems_version: 2.7.6
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: Sinew - structured web crawling using recipes.
-test_files:
-- test/legacy/eu.httpbin.org/head/redirect,3
-- test/legacy/eu.httpbin.org/head/status,500
-- test/legacy/eu.httpbin.org/redirect,3
-- test/legacy/eu.httpbin.org/status,500
-- test/legacy/legacy.sinew
-- test/test.html
-- test/test_cache.rb
-- test/test_helper.rb
-- test/test_legacy.rb
-- test/test_main.rb
-- test/test_nokogiri_ext.rb
-- test/test_output.rb
-- test/test_requests.rb
-- test/test_utf8.rb
+test_files: []