sinew 2.0.2 → 3.0.1

data/lib/sinew/connection/rate_limit.rb

@@ -0,0 +1,29 @@
+module Sinew
+  module Connection
+    class RateLimit < Faraday::Middleware
+      attr_reader :rate_limit
+
+      def initialize(app, options = {})
+        super(app)
+
+        @last_request_tm = @current_request_tm = nil
+        @rate_limit = options.fetch(:rate_limit, 1)
+      end
+
+      def on_request(_env)
+        if @last_request_tm
+          sleep = (@last_request_tm + rate_limit) - Time.now
+          sleep(sleep) if sleep > 0
+        end
+
+        @current_request_tm = Time.now
+      end
+
+      def on_complete(env)
+        # Only rate limit on uncached requests
+        @last_request_tm = @current_request_tm unless env[:httpdisk]
+        @current_request_tm = nil
+      end
+    end
+  end
+end

data/lib/sinew/core_ext.rb

@@ -30,7 +30,7 @@ class String
     elsif limit >= size
       dup
     else
-      self[-limit..-1]
+      self[-limit..]
     end
   end
 

data/lib/sinew/dsl.rb

@@ -1,5 +1,6 @@
-require 'awesome_print'
+require 'amazing_print'
 require 'cgi'
+require 'json'
 
 #
 # The DSL available to .sinew files.
@@ -10,7 +11,7 @@ module Sinew
     # this is used to break out of --limit
     class LimitError < StandardError; end
 
-    attr_reader :sinew, :raw, :uri, :elapsed
+    attr_reader :sinew, :uri, :raw, :code, :elapsed
 
     def initialize(sinew)
       @sinew = sinew
@@ -52,14 +53,17 @@ module Sinew
     end
 
     def http(method, url, options = {})
-      # reset
-      instance_variables.each do |i|
-        instance_variable_set(i, nil) if i != :@sinew
+      # these need to be cleared before each request
+      %i[@html @noko @xml @json].each do |i|
+        instance_variable_set(i, nil)
       end
 
       # fetch and make response available to callers
       response = sinew.http(method, url, options)
-      @uri, @raw = response.uri, response.body
+      @uri, @raw, @code = response.uri, response.body, response.code
+
+      # don't confuse the user
+      nil
     end
 
     #

data/lib/sinew/main.rb

@@ -1,4 +1,5 @@
 require 'scripto'
+require 'sinew/connection'
 
 #
 # Main sinew entry point.
@@ -6,21 +7,13 @@ require 'scripto'
 
 module Sinew
   class Main < Scripto::Main
-    attr_reader :runtime_options, :request_tm, :request_count
+    attr_reader :runtime_options
 
     def initialize(options)
       super(options)
 
       # init
       @runtime_options = RuntimeOptions.new
-      @request_tm = Time.at(0)
-      @request_count = 0
-
-      if options[:proxy]
-        addr, port = options[:proxy].split(':')
-        runtime_options.httparty_options[:http_proxyaddr] = addr
-        runtime_options.httparty_options[:http_proxyport] = port || 80
-      end
     end
 
     def run
@@ -37,24 +30,12 @@ module Sinew
     end
 
     #
-    # http requests and caching
+    # http requests
     #
 
-    def cache
-      @cache ||= Cache.new(self)
-    end
-
     def http(method, url, options = {})
-      request = Request.new(self, method, url, options)
-
-      # try to get from cache
-      response = cache.get(request)
-
-      # perform if necessary
-      if !response
-        response = perform(request)
-        cache.set(response)
-      end
+      request = Request.new(method, url, request_options(options))
+      response = request.perform(connection)
 
       # always log error messages
       if response.error?
@@ -64,26 +45,10 @@ module Sinew
       response
     end
 
-    def perform(request)
-      before_perform_request(request)
-
-      response = nil
-
-      tries = runtime_options.retries + 1
-      while tries > 0
-        tries -= 1
-        begin
-          @request_count += 1
-          response = request.perform
-        rescue Timeout::Error
-          response = Response.from_timeout(request)
-        end
-        break if !response.error_500?
-      end
-
-      response
+    def connection
+      @connection ||= Connection.create(options: options, runtime_options: runtime_options)
     end
-    protected :perform
+    protected :connection
 
     #
     # output
@@ -97,23 +62,31 @@ module Sinew
     # helpers
     #
 
-    def before_perform_request(request)
-      # log
-      if !quiet?
-        msg = if request.method != 'get'
-          "req #{request.uri} (#{request.method})"
-        else
-          "req #{request.uri}"
+    def request_options(options)
+      options.dup.tap do |req|
+        req[:headers] = {}.tap do |h|
+          [ runtime_options.headers, options[:headers]].each do
+            h.merge!(_1) if _1
+          end
         end
-        $stderr.puts msg
+        req[:proxy] = random_proxy
+      end
+    end
+    protected :request_options
+
+    PROXY_RE = /\A#{URI::PATTERN::HOST}(:\d+)?\Z/.freeze
+
+    def random_proxy
+      return if !options[:proxy]
+
+      proxy = options[:proxy].split(',').sample
+      if proxy !~ PROXY_RE
+        raise ArgumentError, "invalid proxy #{proxy.inspect}, should be host[:port]"
       end
 
-      # rate limit
-      sleep = (request_tm + runtime_options.rate_limit) - Time.now
-      sleep(sleep) if sleep > 0
-      @request_tm = Time.now
+      "http://#{proxy}"
     end
-    protected :before_perform_request
+    protected :random_proxy
 
     def footer
       output.report

data/lib/sinew/output.rb

@@ -1,6 +1,6 @@
 require 'csv'
 require 'set'
-require 'stringex'
+require 'sterile'
 
 #
 # CSV output.
@@ -45,6 +45,7 @@ module Sinew
 
       # don't allow duplicate urls
       return if dup_url?(row)
+
       rows << row.dup
 
       # map columns to row, and normalize along the way
@@ -101,24 +102,14 @@ module Sinew
       # strip html tags. Note that we replace tags with spaces
       s = s.gsub(/<[^>]+>/, ' ')
 
-      #
-      # Below uses stringex
-      #
-      # github.com/rsl/stringex/blob/master/lib/stringex/string_extensions.rb
-      # github.com/rsl/stringex/blob/master/lib/stringex/localization/conversion_expressions.rb
-      #
-
       # Converts MS Word 'smart punctuation' to ASCII
-      s = s.convert_smart_punctuation
-
-      # "&aacute;".convert_accented_html_entities # => "a"
-      s = s.convert_accented_html_entities
+      s = Sterile.plain_format(s)
 
-      # &amp, &frac, etc.
-      s = s.convert_miscellaneous_html_entities
+      # &aacute; &amp; etc.
+      s = Sterile.decode_entities(s)
 
-      # convert unicode => regular characters
-      s = s.to_ascii
+      # "šţɽĩɳģ" => "string"
+      s = Sterile.transliterate(s)
 
       # squish
       s = s.squish

data/lib/sinew/request.rb

@@ -1,61 +1,57 @@
-require 'digest/md5'
-require 'httparty'
-require 'htmlentities'
+require 'sterile'
 
 #
-# Process a single HTTP request. Mostly a wrapper around HTTParty.
+# Process a single HTTP request.
 #
 
 module Sinew
   class Error < StandardError; end
 
   class Request
-    HTML_ENTITIES = HTMLEntities.new
     VALID_METHODS = %w[get post patch put delete head options].freeze
+    METHODS_WITH_BODY = %w[patch post put].freeze
 
-    attr_reader :sinew, :method, :uri, :options, :cache_key
+    attr_reader :method, :options, :uri
 
-    # Options are largely compatible with HTTParty, except for :method.
-    def initialize(sinew, method, url, options = {})
-      @sinew = sinew
+    # Supported options:
+    #  body: Body of http post
+    #  headers: Hash of HTTP headers (combined with runtime_options.headers)
+    #  query: Hash of query parameters to add to url
+    def initialize(method, url, options = {})
       @method = method
       @options = options.dup
       @uri = parse_url(url)
-      @cache_key = calculate_cache_key
     end
 
     # run the request, return the result
-    def perform
+    def perform(connection)
       validate!
 
-      # merge optons
-      options = self.options.merge(sinew.runtime_options.httparty_options)
-
-      # merge headers
-      headers = sinew.runtime_options.headers
-      headers = headers.merge(options[:headers]) if options[:headers]
-      options[:headers] = headers
+      body = options.delete(:body)
+      fday_response = connection.send(method, uri, body) do
+        _1.headers.update(options[:headers]) if options[:headers]
+        _1.options[:proxy] = options[:proxy]
+      end
 
-      party_response = HTTParty.send(method, uri, options)
-      Response.from_network(self, party_response)
+      Response.from_network(self, fday_response)
     end
 
     # We accept sloppy urls and attempt to clean them up
     def parse_url(url)
-      s = url
+      s = url.to_s
 
       # remove entities
-      s = HTML_ENTITIES.decode(s)
+      s = Sterile.decode_entities(s)
 
       # fix a couple of common encoding bugs
       s = s.gsub(' ', '%20')
       s = s.gsub("'", '%27')
 
-      # append query manually (instead of letting HTTParty handle it) so we can
-      # include it in cache_key
+      # append query manually (instead of letting Faraday handle it) for consistent
+      # Request#uri and Response#uri
       query = options.delete(:query)
       if query.present?
-        q = HTTParty::HashConversions.to_params(query)
+        q = Faraday::Utils.default_params_encoder.encode(query)
         separator = s.include?('?') ? '&' : '?'
         s = "#{s}#{separator}#{q}"
       end
@@ -64,44 +60,10 @@ module Sinew
     end
     protected :parse_url
 
-    def calculate_cache_key
-      dir = pathify(uri.host)
-
-      body_key = if body.is_a?(Hash)
-        HTTParty::HashConversions.to_params(body)
-      else
-        body&.dup
-      end
-
-      # build key, as a hash for before_generate_cache_key
-      key = {
-        method: method.dup,
-        path: uri.path,
-        query: uri.query,
-        body: body_key,
-      }
-      key = sinew.runtime_options.before_generate_cache_key.call(key)
-
-      # strip method for gets
-      key.delete(:method) if key[:method] == 'get'
-
-      # pull out the values, join and pathify
-      path = key.values.select(&:present?).join(',')
-      path = pathify(path)
-
-      # shorten long paths
-      if path.length > 250
-        path = Digest::MD5.hexdigest(path)
-      end
-
-      "#{dir}/#{path}"
-    end
-    protected :calculate_cache_key
-
     def validate!
       raise "invalid method #{method}" if !VALID_METHODS.include?(method)
       raise "invalid url #{uri}" if uri.scheme !~ /^http/
-      raise "can't get with a body" if method == 'get' && body
+      raise "can't #{method} with a body" if body && !METHODS_WITH_BODY.include?(method)
       raise "Content-Type doesn't make sense without a body" if content_type && !body
     end
     protected :validate!
@@ -120,32 +82,5 @@ module Sinew
       headers && headers['Content-Type']
     end
     protected :content_type
-
-    def form?
-      content_type == 'application/x-www-form-urlencoded'
-    end
-    protected :form?
-
-    def pathify(s)
-      # remove leading slash
-      s = s.gsub(/^\//, '')
-      # .. => comma
-      s = s.gsub('..', ',')
-      # query separators => comma
-      s = s.gsub(/[?\/&]/, ',')
-      # ,, => comma
-      s = s.gsub(',,', ',')
-      # encode invalid path chars
-      s = s.gsub(/[^A-Za-z0-9_.,=-]/) do |i|
-        hex = i.unpack('H2').first
-        "%#{hex}"
-      end
-      # handle empty case
-      s = '_root_' if s.blank?
-      # always downcase
-      s = s.downcase
-      s
-    end
-    protected :pathify
   end
 end

data/lib/sinew/response.rb

@@ -2,7 +2,7 @@ require 'stringio'
 require 'zlib'
 
 #
-# An HTTP response. Mostly a wrapper around HTTParty.
+# An HTTP response.
 #
 
 module Sinew
@@ -13,62 +13,13 @@ module Sinew
     # factory methods
     #
 
-    def self.from_network(request, party_response)
-      Response.new.tap do |response|
-        response.request = request
-        response.uri = party_response.request.last_uri
-        response.code = party_response.code
-        response.headers = party_response.headers.to_h
-        response.body = process_body(party_response)
-      end
-    end
-
-    def self.from_cache(request, body, head)
-      Response.new.tap do |response|
-        response.request = request
-        response.body = body
-
-        # defaults
-        response.uri = request.uri
-        response.code = 200
-        response.headers = {}
-
-        # overwrite with cached response headers
-        if head
-          if head !~ /^{/
-            return from_legacy_head(response, head)
-          end
-          head = JSON.parse(head, symbolize_names: true)
-          response.uri = URI.parse(head[:uri])
-          response.code = head[:code]
-          response.headers = head[:headers]
-        end
-      end
-    end
-
-    def self.from_timeout(request)
-      Response.new.tap do |response|
-        response.request = request
-        response.uri = request.uri
-        response.body = 'timeout'
-        response.code = 999
-        response.headers = {}
-      end
-    end
-
-    def self.from_legacy_head(response, head)
-      response.tap do |r|
-        case head
-        when /\ACURLER_ERROR/
-          # error
-          r.code = 999
-        when /\AHTTP/
-          # redirect
-          location = head.scan(/Location: ([^\r\n]+)/).flatten.last
-          r.uri += location
-        else
-          $stderr.puts "unknown cached /head for #{r.uri}"
-        end
+    def self.from_network(request, fday_response)
+      Response.new.tap do
+        _1.request = request
+        _1.uri = fday_response.env.url
+        _1.code = fday_response.status
+        _1.headers = fday_response.headers.to_h
+        _1.body = process_body(fday_response)
       end
     end