sinew 2.0.2 → 3.0.1

data/lib/sinew/runtime_options.rb

@@ -7,8 +7,8 @@ module Sinew
     attr_accessor :retries
     attr_accessor :rate_limit
     attr_accessor :headers
-    attr_accessor :httparty_options
-    attr_accessor :before_generate_cache_key
+    attr_accessor :httpdisk_options
+    attr_accessor :insecure
 
     def initialize
       self.retries = 3
@@ -16,8 +16,8 @@ module Sinew
       self.headers = {
         'User-Agent' => "sinew/#{VERSION}",
       }
-      self.httparty_options = {}
-      self.before_generate_cache_key = ->(i) { i }
+      self.httpdisk_options = {}
+      self.insecure = false
 
       # for testing
       if ENV['SINEW_TEST']

data/lib/sinew/version.rb

@@ -1,4 +1,4 @@
 module Sinew
   # Gem version
-  VERSION = '2.0.2'.freeze
+  VERSION = '3.0.1'.freeze
 end

data/sample.sinew

@@ -1,4 +1,4 @@
-get 'http://httpbin.org'
+get 'http://httpbingo.org'
 noko.css('ul li a').each do |a|
   row = {}
   row[:url] = a[:href]
@@ -6,4 +6,4 @@ noko.css('ul li a').each do |a|
   csv_emit(row)
 end
 
-get 'http://httpbin.org/redirect/2'
+get 'http://httpbingo.org/redirect/2'

data/sinew.gemspec

@@ -5,30 +5,28 @@ require 'sinew/version'
 Gem::Specification.new do |s|
   s.name        = 'sinew'
   s.version     = Sinew::VERSION
-  s.platform    = Gem::Platform::RUBY
   s.license     = 'MIT'
-  s.authors     = [ 'Adam Doppelt' ]
+  s.authors     = [ 'Adam Doppelt', 'Nathan Kriege' ]
   s.email       = [ 'amd@gurge.com' ]
   s.homepage    = 'http://github.com/gurgeous/sinew'
   s.summary     = 'Sinew - structured web crawling using recipes.'
   s.description = 'Crawl web sites easily using ruby recipes, with caching and nokogiri.'
-  s.required_ruby_version = '~> 2.3'
+  s.required_ruby_version = '>= 2.7'
 
-  s.rubyforge_project = 'sinew'
+  # what's in the gem?
+  s.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { _1.match(%r{^test/}) }
+  end
+  s.bindir = 'bin'
+  s.executables = s.files.grep(%r{^#{s.bindir}/}) { File.basename(_1) }
+  s.require_paths = [ 'lib' ]
 
-  s.add_runtime_dependency 'awesome_print', '~> 1.8'
-  s.add_runtime_dependency 'htmlentities', '~> 4.3'
-  s.add_runtime_dependency 'httparty', '~> 0.16'
-  s.add_runtime_dependency 'nokogiri', '~> 1.8'
+  s.add_runtime_dependency 'amazing_print', '~> 1.3'
+  s.add_runtime_dependency 'faraday', '~> 1.4'
+  s.add_runtime_dependency 'faraday-encoding', '~> 0'
+  s.add_runtime_dependency 'httpdisk', '~> 0'
+  s.add_runtime_dependency 'nokogiri', '~> 1.11'
   s.add_runtime_dependency 'scripto', '~> 0'
-  s.add_runtime_dependency 'slop', '~> 4.6'
-  s.add_runtime_dependency 'stringex', '~> 2.8'
-  s.add_development_dependency 'minitest', '~> 5.11'
-  s.add_development_dependency 'rake', '~> 12.3'
-  s.add_development_dependency 'webmock', '~> 3.4'
-
-  s.files         = `git ls-files`.split("\n")
-  s.test_files    = `git ls-files -- test/*`.split("\n")
-  s.executables   = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
-  s.require_paths = [ 'lib' ]
+  s.add_runtime_dependency 'slop', '~> 4.8'
+  s.add_runtime_dependency 'sterile', '~> 1.0'
 end

metadata

@@ -1,73 +1,60 @@
 --- !ruby/object:Gem::Specification
 name: sinew
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 3.0.1
 platform: ruby
 authors:
 - Adam Doppelt
-autorequire: 
+- Nathan Kriege
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-05-03 00:00:00.000000000 Z
+date: 2021-06-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: awesome_print
+  name: amazing_print
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '1.3'
 - !ruby/object:Gem::Dependency
-  name: htmlentities
+  name: faraday
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.3'
+        version: '1.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.3'
+        version: '1.4'
 - !ruby/object:Gem::Dependency
-  name: httparty
+  name: faraday-encoding
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.16'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.16'
-- !ruby/object:Gem::Dependency
-  name: nokogiri
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: scripto
+  name: httpdisk
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -81,75 +68,61 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: slop
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.6'
+        version: '1.11'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.6'
+        version: '1.11'
 - !ruby/object:Gem::Dependency
-  name: stringex
+  name: scripto
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.8'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.8'
-- !ruby/object:Gem::Dependency
-  name: minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.11'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.11'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: slop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
-  type: :development
+        version: '4.8'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '4.8'
 - !ruby/object:Gem::Dependency
-  name: webmock
+  name: sterile
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.4'
-  type: :development
+        version: '1.0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.4'
+        version: '1.0'
 description: Crawl web sites easily using ruby recipes, with caching and nokogiri.
 email:
 - amd@gurge.com
@@ -158,9 +131,9 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/test.yml"
 - ".gitignore"
 - ".rubocop.yml"
-- ".travis.yml"
 - ".vscode/extensions.json"
 - ".vscode/settings.json"
 - Gemfile
@@ -169,7 +142,9 @@ files:
 - Rakefile
 - bin/sinew
 - lib/sinew.rb
-- lib/sinew/cache.rb
+- lib/sinew/connection.rb
+- lib/sinew/connection/log_formatter.rb
+- lib/sinew/connection/rate_limit.rb
 - lib/sinew/core_ext.rb
 - lib/sinew/dsl.rb
 - lib/sinew/main.rb
@@ -181,74 +156,27 @@ files:
 - lib/sinew/version.rb
 - sample.sinew
 - sinew.gemspec
-- test/legacy/eu.httpbin.org/head/redirect,3
-- test/legacy/eu.httpbin.org/head/status,500
-- test/legacy/eu.httpbin.org/redirect,3
-- test/legacy/eu.httpbin.org/status,500
-- test/legacy/legacy.sinew
-- test/recipes/array_header.sinew
-- test/recipes/basic.sinew
-- test/recipes/dups.sinew
-- test/recipes/implicit_header.sinew
-- test/recipes/limit.sinew
-- test/recipes/noko.sinew
-- test/recipes/uri.sinew
-- test/recipes/xml.sinew
-- test/test.html
-- test/test_cache.rb
-- test/test_helper.rb
-- test/test_legacy.rb
-- test/test_main.rb
-- test/test_nokogiri_ext.rb
-- test/test_output.rb
-- test/test_recipes.rb
-- test/test_requests.rb
-- test/test_utf8.rb
 homepage: http://github.com/gurgeous/sinew
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - "~>"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: '2.3'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: sinew
-rubygems_version: 2.7.6
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: Sinew - structured web crawling using recipes.
-test_files:
-- test/legacy/eu.httpbin.org/head/redirect,3
-- test/legacy/eu.httpbin.org/head/status,500
-- test/legacy/eu.httpbin.org/redirect,3
-- test/legacy/eu.httpbin.org/status,500
-- test/legacy/legacy.sinew
-- test/recipes/array_header.sinew
-- test/recipes/basic.sinew
-- test/recipes/dups.sinew
-- test/recipes/implicit_header.sinew
-- test/recipes/limit.sinew
-- test/recipes/noko.sinew
-- test/recipes/uri.sinew
-- test/recipes/xml.sinew
-- test/test.html
-- test/test_cache.rb
-- test/test_helper.rb
-- test/test_legacy.rb
-- test/test_main.rb
-- test/test_nokogiri_ext.rb
-- test/test_output.rb
-- test/test_recipes.rb
-- test/test_requests.rb
-- test/test_utf8.rb
+test_files: []

data/.travis.yml

@@ -1,4 +0,0 @@
-language: ruby
-rvm:
-  - 2.3.7
-  - 2.5.1

data/lib/sinew/cache.rb

@@ -1,79 +0,0 @@
-require 'fileutils'
-require 'tempfile'
-
-#
-# This class handles the caching of http responses on disk.
-#
-
-module Sinew
-  class Cache
-    attr_reader :sinew
-
-    def initialize(sinew)
-      @sinew = sinew
-    end
-
-    def get(request)
-      body = read_if_exist(body_path(request))
-      return nil if !body
-
-      head = read_if_exist(head_path(request))
-      Response.from_cache(request, body, head)
-    end
-
-    def set(response)
-      body_path = body_path(response.request)
-      head_path = head_path(response.request)
-
-      FileUtils.mkdir_p(File.dirname(body_path))
-      FileUtils.mkdir_p(File.dirname(head_path))
-
-      # write body, and head if necessary
-      atomic_write(body_path, response.body)
-      if head_necessary?(response)
-        head = JSON.pretty_generate(response.head_as_json)
-        atomic_write(head_path, head)
-      end
-    end
-
-    def root_dir
-      sinew.options[:cache]
-    end
-    protected :root_dir
-
-    def head_necessary?(response)
-      response.error? || response.redirected?
-    end
-    protected :head_necessary?
-
-    def body_path(request)
-      "#{root_dir}/#{request.cache_key}"
-    end
-    protected :body_path
-
-    def head_path(request)
-      body_path = body_path(request)
-      dir, base = File.dirname(body_path), File.basename(body_path)
-      "#{dir}/head/#{base}"
-    end
-    protected :head_path
-
-    def read_if_exist(path)
-      if File.exist?(path)
-        IO.read(path, mode: 'r:UTF-8')
-      end
-    end
-    protected :read_if_exist
-
-    def atomic_write(path, data)
-      tmp = Tempfile.new('sinew', encoding: 'UTF-8')
-      tmp.write(data)
-      tmp.close
-      FileUtils.chmod(0o644, tmp.path)
-      FileUtils.mv(tmp.path, path)
-    ensure
-      FileUtils.rm(tmp.path, force: true)
-    end
-    protected :atomic_write
-  end
-end