sinew 2.0.5 → 3.0.0

data/lib/sinew/output.rb

@@ -1,13 +1,6 @@
 require 'csv'
 require 'set'
-require 'stringex'
-
-#
-# Stringex customizations
-#
-
-# turn '&' into '&', not 'and'
-Stringex::Localization::DefaultConversions::HTML_ENTITIES[:amp] = '&'
+require 'sterile'
 
 #
 # CSV output.
@@ -52,6 +45,7 @@ module Sinew
 
       # don't allow duplicate urls
       return if dup_url?(row)
+
       rows << row.dup
 
       # map columns to row, and normalize along the way
@@ -108,24 +102,14 @@ module Sinew
       # strip html tags. Note that we replace tags with spaces
       s = s.gsub(/<[^>]+>/, ' ')
 
-      #
-      # Below uses stringex
-      #
-      # github.com/rsl/stringex/blob/master/lib/stringex/string_extensions.rb
-      # github.com/rsl/stringex/blob/master/lib/stringex/localization/conversion_expressions.rb
-      #
-
       # Converts MS Word 'smart punctuation' to ASCII
-      s = s.convert_smart_punctuation
-
-      # "&aacute;".convert_accented_html_entities # => "a"
-      s = s.convert_accented_html_entities
+      s = Sterile.plain_format(s)
 
-      # &amp, &frac, etc.
-      s = s.convert_miscellaneous_html_entities
+      # &aacute; &amp; etc.
+      s = Sterile.decode_entities(s)
 
-      # convert unicode => regular characters
-      s = s.to_ascii
+      # "šţɽĩɳģ" => "string"
+      s = Sterile.transliterate(s)
 
       # squish
       s = s.squish

data/lib/sinew/request.rb

@@ -1,9 +1,8 @@
 require 'digest/md5'
-require 'httparty'
 require 'htmlentities'
 
 #
-# Process a single HTTP request. Mostly a wrapper around HTTParty.
+# Process a single HTTP request.
 #
 
 module Sinew
@@ -12,16 +11,19 @@ module Sinew
   class Request
     HTML_ENTITIES = HTMLEntities.new
     VALID_METHODS = %w[get post patch put delete head options].freeze
+    METHODS_WITH_BODY = %w[patch post put].freeze
 
-    attr_reader :sinew, :method, :uri, :options, :cache_key
+    attr_reader :sinew, :method, :uri, :options
 
-    # Options are largely compatible with HTTParty, except for :method.
+    # Supported options:
+    #  body: Body of http post
+    #  headers: Hash of HTTP headers (combined with runtime_options.headers)
+    #  query: Hash of query parameters to add to url
     def initialize(sinew, method, url, options = {})
       @sinew = sinew
       @method = method
       @options = options.dup
       @uri = parse_url(url)
-      @cache_key = calculate_cache_key
     end
 
     def proxy
@@ -33,28 +35,19 @@ module Sinew
     end
 
     # run the request, return the result
-    def perform
+    def perform(connection)
       validate!
 
-      party_options = options.dup
-
-      # merge proxy
-      if proxy = self.proxy
-        addr, port = proxy.split(':')
-        party_options[:http_proxyaddr] = addr
-        party_options[:http_proxyport] = port || 80
-      end
+      headers = sinew.runtime_options.headers
+      headers = headers.merge(options[:headers]) if options[:headers]
 
-      # now merge runtime_options
-      party_options = party_options.merge(sinew.runtime_options.httparty_options)
+      body = options.delete(:body)
 
-      # merge headers
-      headers = sinew.runtime_options.headers
-      headers = headers.merge(party_options[:headers]) if party_options[:headers]
-      party_options[:headers] = headers
+      fday_response = connection.send(method, uri, body, headers) do
+        _1.options[:proxy] = proxy
+      end
 
-      party_response = HTTParty.send(method, uri, party_options)
-      Response.from_network(self, party_response)
+      Response.from_network(self, fday_response)
     end
 
     # We accept sloppy urls and attempt to clean them up
@@ -68,11 +61,11 @@ module Sinew
       s = s.gsub(' ', '%20')
       s = s.gsub("'", '%27')
 
-      # append query manually (instead of letting HTTParty handle it) so we can
-      # include it in cache_key
+      # append query manually (instead of letting Faraday handle it) for consistent
+      # Request#uri and Response#uri
       query = options.delete(:query)
       if query.present?
-        q = HTTParty::HashConversions.to_params(query)
+        q = Faraday::Utils.default_params_encoder.encode(query)
         separator = s.include?('?') ? '&' : '?'
         s = "#{s}#{separator}#{q}"
       end
@@ -81,54 +74,10 @@ module Sinew
     end
     protected :parse_url
 
-    def calculate_cache_key
-      dir = pathify(uri.host)
-
-      body_key = if body.is_a?(Hash)
-        HTTParty::HashConversions.to_params(body)
-      else
-        body&.dup
-      end
-
-      # Build key, as a hash for before_generate_cache_key. Note that :scheme is
-      # just a placeholder in case someone wants to add it for real, so that
-      # it'll appear in the correct order. We remove the placerholder after we
-      # call the proc.
-      key = {
-        method: method.dup,
-        scheme: 'placeholder',
-        path: uri.path,
-        query: uri.query,
-        body: body_key,
-      }
-
-      args = [ key ]
-      if sinew.runtime_options.before_generate_cache_key.arity == 2
-        args << uri
-      end
-      key = sinew.runtime_options.before_generate_cache_key.call(*args)
-
-      # strip defaults
-      key.delete(:scheme) if key[:scheme] == 'placeholder'
-      key.delete(:method) if key[:method] == 'get'
-
-      # pull out the values, join and pathify
-      path = key.values.select(&:present?).join(',')
-      path = pathify(path)
-
-      # shorten long paths
-      if path.length > 250
-        path = Digest::MD5.hexdigest(path)
-      end
-
-      "#{dir}/#{path}"
-    end
-    protected :calculate_cache_key
-
     def validate!
       raise "invalid method #{method}" if !VALID_METHODS.include?(method)
       raise "invalid url #{uri}" if uri.scheme !~ /^http/
-      raise "can't get with a body" if method == 'get' && body
+      raise "can't #{method} with a body" if body && !METHODS_WITH_BODY.include?(method)
       raise "Content-Type doesn't make sense without a body" if content_type && !body
     end
     protected :validate!
@@ -164,7 +113,7 @@ module Sinew
       s = s.gsub(',,', ',')
       # encode invalid path chars
       s = s.gsub(/[^A-Za-z0-9_.,=-]/) do |i|
-        hex = i.unpack('H2').first
+        hex = i.unpack1('H2')
         "%#{hex}"
       end
       # handle empty case

data/lib/sinew/response.rb

@@ -2,7 +2,7 @@ require 'stringio'
 require 'zlib'
 
 #
-# An HTTP response. Mostly a wrapper around HTTParty.
+# An HTTP response.
 #
 
 module Sinew
@@ -13,62 +13,13 @@ module Sinew
     # factory methods
     #
 
-    def self.from_network(request, party_response)
-      Response.new.tap do |response|
-        response.request = request
-        response.uri = party_response.request.last_uri
-        response.code = party_response.code
-        response.headers = party_response.headers.to_h
-        response.body = process_body(party_response)
-      end
-    end
-
-    def self.from_cache(request, body, head)
-      Response.new.tap do |response|
-        response.request = request
-        response.body = body
-
-        # defaults
-        response.uri = request.uri
-        response.code = 200
-        response.headers = {}
-
-        # overwrite with cached response headers
-        if head
-          if head !~ /^{/
-            return from_legacy_head(response, head)
-          end
-          head = JSON.parse(head, symbolize_names: true)
-          response.uri = URI.parse(head[:uri])
-          response.code = head[:code]
-          response.headers = head[:headers]
-        end
-      end
-    end
-
-    def self.from_error(request, error)
-      Response.new.tap do |response|
-        response.request = request
-        response.uri = request.uri
-        response.body = error.to_s
-        response.code = 999
-        response.headers = {}
-      end
-    end
-
-    def self.from_legacy_head(response, head)
-      response.tap do |r|
-        case head
-        when /\ACURLER_ERROR/
-          # error
-          r.code = 999
-        when /\AHTTP/
-          # redirect
-          location = head.scan(/Location: ([^\r\n]+)/).flatten.last
-          r.uri += location
-        else
-          $stderr.puts "unknown cached /head for #{r.uri}"
-        end
+    def self.from_network(request, fday_response)
+      Response.new.tap do
+        _1.request = request
+        _1.uri = fday_response.env.url
+        _1.code = fday_response.status
+        _1.headers = fday_response.headers.to_h
+        _1.body = process_body(fday_response)
       end
     end
 

data/lib/sinew/runtime_options.rb

@@ -7,8 +7,8 @@ module Sinew
     attr_accessor :retries
     attr_accessor :rate_limit
     attr_accessor :headers
-    attr_accessor :httparty_options
-    attr_accessor :before_generate_cache_key
+    attr_accessor :httpdisk_options
+    attr_accessor :insecure
 
     def initialize
       self.retries = 3
@@ -16,8 +16,8 @@ module Sinew
       self.headers = {
         'User-Agent' => "sinew/#{VERSION}",
       }
-      self.httparty_options = {}
-      self.before_generate_cache_key = ->(i) { i }
+      self.httpdisk_options = {}
+      self.insecure = false
 
       # for testing
       if ENV['SINEW_TEST']

data/lib/sinew/version.rb

@@ -1,4 +1,4 @@
 module Sinew
   # Gem version
-  VERSION = '2.0.5'.freeze
+  VERSION = '3.0.0'.freeze
 end

data/sample.sinew

@@ -1,4 +1,4 @@
-get 'http://httpbin.org'
+get 'http://httpbingo.org'
 noko.css('ul li a').each do |a|
   row = {}
   row[:url] = a[:href]
@@ -6,4 +6,4 @@ noko.css('ul li a').each do |a|
   csv_emit(row)
 end
 
-get 'http://httpbin.org/redirect/2'
+get 'http://httpbingo.org/redirect/2'

data/sinew.gemspec

@@ -5,30 +5,29 @@ require 'sinew/version'
 Gem::Specification.new do |s|
   s.name        = 'sinew'
   s.version     = Sinew::VERSION
-  s.platform    = Gem::Platform::RUBY
   s.license     = 'MIT'
-  s.authors     = [ 'Adam Doppelt' ]
+  s.authors     = [ 'Adam Doppelt', 'Nathan Kriege' ]
   s.email       = [ 'amd@gurge.com' ]
   s.homepage    = 'http://github.com/gurgeous/sinew'
   s.summary     = 'Sinew - structured web crawling using recipes.'
   s.description = 'Crawl web sites easily using ruby recipes, with caching and nokogiri.'
-  s.required_ruby_version = '~> 2.3'
+  s.required_ruby_version = '>= 2.7'
 
-  s.rubyforge_project = 'sinew'
+  # what's in the gem?
+  s.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { _1.match(%r{^test/}) }
+  end
+  s.bindir = 'bin'
+  s.executables = s.files.grep(%r{^#{s.bindir}/}) { File.basename(_1) }
+  s.require_paths = [ 'lib' ]
 
-  s.add_runtime_dependency 'awesome_print', '~> 1.8'
+  s.add_runtime_dependency 'amazing_print', '~> 1.3'
+  s.add_runtime_dependency 'faraday', '~> 1.4'
+  s.add_runtime_dependency 'faraday-encoding', '~> 0'
   s.add_runtime_dependency 'htmlentities', '~> 4.3'
-  s.add_runtime_dependency 'httparty', '~> 0.16'
-  s.add_runtime_dependency 'nokogiri', '~> 1.8'
+  s.add_runtime_dependency 'httpdisk', '~> 0'
+  s.add_runtime_dependency 'nokogiri', '~> 1.11'
   s.add_runtime_dependency 'scripto', '~> 0'
-  s.add_runtime_dependency 'slop', '~> 4.6'
-  s.add_runtime_dependency 'stringex', '~> 2.8'
-  s.add_development_dependency 'minitest', '~> 5.11'
-  s.add_development_dependency 'rake', '~> 12.3'
-  s.add_development_dependency 'webmock', '~> 3.4'
-
-  s.files         = `git ls-files`.split("\n")
-  s.test_files    = `git ls-files -- test/*`.split("\n")
-  s.executables   = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
-  s.require_paths = [ 'lib' ]
+  s.add_runtime_dependency 'slop', '~> 4.8'
+  s.add_runtime_dependency 'sterile', '~> 1.0'
 end

metadata

@@ -1,73 +1,74 @@
 --- !ruby/object:Gem::Specification
 name: sinew
 version: !ruby/object:Gem::Version
-  version: 2.0.5
+  version: 3.0.0
 platform: ruby
 authors:
 - Adam Doppelt
-autorequire: 
+- Nathan Kriege
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-03-22 00:00:00.000000000 Z
+date: 2021-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: awesome_print
+  name: amazing_print
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '1.3'
 - !ruby/object:Gem::Dependency
-  name: htmlentities
+  name: faraday
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.3'
+        version: '1.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.3'
+        version: '1.4'
 - !ruby/object:Gem::Dependency
-  name: httparty
+  name: faraday-encoding
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.16'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.16'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: htmlentities
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '4.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '4.3'
 - !ruby/object:Gem::Dependency
-  name: scripto
+  name: httpdisk
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -81,75 +82,61 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: slop
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.6'
+        version: '1.11'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.6'
+        version: '1.11'
 - !ruby/object:Gem::Dependency
-  name: stringex
+  name: scripto
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.8'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.8'
-- !ruby/object:Gem::Dependency
-  name: minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.11'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.11'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: slop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
-  type: :development
+        version: '4.8'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '4.8'
 - !ruby/object:Gem::Dependency
-  name: webmock
+  name: sterile
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.4'
-  type: :development
+        version: '1.0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.4'
+        version: '1.0'
 description: Crawl web sites easily using ruby recipes, with caching and nokogiri.
 email:
 - amd@gurge.com
@@ -158,9 +145,9 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/test.yml"
 - ".gitignore"
 - ".rubocop.yml"
-- ".travis.yml"
 - ".vscode/extensions.json"
 - ".vscode/settings.json"
 - Gemfile
@@ -169,7 +156,9 @@ files:
 - Rakefile
 - bin/sinew
 - lib/sinew.rb
-- lib/sinew/cache.rb
+- lib/sinew/connection.rb
+- lib/sinew/connection/log_formatter.rb
+- lib/sinew/connection/rate_limit.rb
 - lib/sinew/core_ext.rb
 - lib/sinew/dsl.rb
 - lib/sinew/main.rb
@@ -181,74 +170,27 @@ files:
 - lib/sinew/version.rb
 - sample.sinew
 - sinew.gemspec
-- test/legacy/eu.httpbin.org/head/redirect,3
-- test/legacy/eu.httpbin.org/head/status,500
-- test/legacy/eu.httpbin.org/redirect,3
-- test/legacy/eu.httpbin.org/status,500
-- test/legacy/legacy.sinew
-- test/recipes/array_header.sinew
-- test/recipes/basic.sinew
-- test/recipes/dups.sinew
-- test/recipes/implicit_header.sinew
-- test/recipes/limit.sinew
-- test/recipes/noko.sinew
-- test/recipes/uri.sinew
-- test/recipes/xml.sinew
-- test/test.html
-- test/test_cache.rb
-- test/test_helper.rb
-- test/test_legacy.rb
-- test/test_main.rb
-- test/test_nokogiri_ext.rb
-- test/test_output.rb
-- test/test_recipes.rb
-- test/test_requests.rb
-- test/test_utf8.rb
 homepage: http://github.com/gurgeous/sinew
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - "~>"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: '2.3'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: sinew
-rubygems_version: 2.7.6
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: Sinew - structured web crawling using recipes.
-test_files:
-- test/legacy/eu.httpbin.org/head/redirect,3
-- test/legacy/eu.httpbin.org/head/status,500
-- test/legacy/eu.httpbin.org/redirect,3
-- test/legacy/eu.httpbin.org/status,500
-- test/legacy/legacy.sinew
-- test/recipes/array_header.sinew
-- test/recipes/basic.sinew
-- test/recipes/dups.sinew
-- test/recipes/implicit_header.sinew
-- test/recipes/limit.sinew
-- test/recipes/noko.sinew
-- test/recipes/uri.sinew
-- test/recipes/xml.sinew
-- test/test.html
-- test/test_cache.rb
-- test/test_helper.rb
-- test/test_legacy.rb
-- test/test_main.rb
-- test/test_nokogiri_ext.rb
-- test/test_output.rb
-- test/test_recipes.rb
-- test/test_requests.rb
-- test/test_utf8.rb
+test_files: []