sinew 2.0.1 → 3.0.0

data/lib/sinew/connection/log_formatter.rb

@@ -0,0 +1,22 @@
+module Sinew
+  module Connection
+    class LogFormatter < Faraday::Logging::Formatter
+      def request(env)
+        info('req') do
+          # Only log the initial request, not the redirects
+          return if env[:redirect]
+
+          msg = apply_filters(env.url.to_s)
+          msg = "#{msg} (#{env.method})" if env.method != :get
+          msg = "#{msg} => #{env.request.proxy.uri}" if env.request.proxy
+
+          msg
+        end
+      end
+
+      def response(env)
+        # silent
+      end
+    end
+  end
+end

data/lib/sinew/connection/rate_limit.rb

@@ -0,0 +1,29 @@
+module Sinew
+  module Connection
+    class RateLimit < Faraday::Middleware
+      attr_reader :rate_limit
+
+      def initialize(app, options = {})
+        super(app)
+
+        @last_request_tm = @current_request_tm = nil
+        @rate_limit = options.fetch(:rate_limit, 1)
+      end
+
+      def on_request(_env)
+        if @last_request_tm
+          sleep = (@last_request_tm + rate_limit) - Time.now
+          sleep(sleep) if sleep > 0
+        end
+
+        @current_request_tm = Time.now
+      end
+
+      def on_complete(env)
+        # Only rate limit on uncached requests
+        @last_request_tm = @current_request_tm unless env[:httpdisk]
+        @current_request_tm = nil
+      end
+    end
+  end
+end

data/lib/sinew/core_ext.rb

@@ -30,7 +30,7 @@ class String
     elsif limit >= size
       dup
     else
-      self[-limit..-1]
+      self[-limit..]
     end
   end
 

data/lib/sinew/dsl.rb

@@ -1,5 +1,6 @@
-require 'awesome_print'
+require 'amazing_print'
 require 'cgi'
+require 'json'
 
 #
 # The DSL available to .sinew files.
@@ -7,7 +8,10 @@ require 'cgi'
 
 module Sinew
   class DSL
-    attr_reader :sinew, :raw, :uri, :elapsed
+    # this is used to break out of --limit
+    class LimitError < StandardError; end
+
+    attr_reader :sinew, :uri, :raw, :code, :elapsed
 
     def initialize(sinew)
       @sinew = sinew
@@ -15,8 +19,12 @@ module Sinew
 
     def run
       tm = Time.now
-      recipe = sinew.options[:recipe]
-      instance_eval(File.read(recipe, mode: 'rb'), recipe)
+      begin
+        recipe = sinew.options[:recipe]
+        instance_eval(File.read(recipe, mode: 'rb'), recipe)
+      rescue LimitError
+        # ignore - this is flow control for --limit
+      end
       @elapsed = Time.now - tm
     end
 
@@ -45,15 +53,17 @@ module Sinew
     end
 
     def http(method, url, options = {})
-      # reset
-      @html = @noko = @json = @url = nil
+      # these need to be cleared before each request
+      %i[@html @noko @xml @json].each do |i|
+        instance_variable_set(i, nil)
+      end
 
-      # fetch
+      # fetch and make response available to callers
       response = sinew.http(method, url, options)
+      @uri, @raw, @code = response.uri, response.body, response.code
 
-      # respond
-      @uri = response.uri
-      @raw = response.body
+      # don't confuse the user
+      nil
     end
 
     #
@@ -75,6 +85,10 @@ module Sinew
       @noko ||= Nokogiri::HTML(html)
     end
 
+    def xml
+      @xml ||= Nokogiri::XML(html)
+    end
+
     def json
       @json ||= JSON.parse(raw, symbolize_names: true)
     end
@@ -93,6 +107,9 @@ module Sinew
 
     def csv_emit(row)
       sinew.output.emit(row)
+      if sinew.output.count == sinew.options[:limit]
+        raise LimitError.new
+      end
     end
   end
 end

data/lib/sinew/main.rb

@@ -1,4 +1,5 @@
 require 'scripto'
+require 'sinew/connection'
 
 #
 # Main sinew entry point.
@@ -6,15 +7,13 @@ require 'scripto'
 
 module Sinew
   class Main < Scripto::Main
-    attr_reader :runtime_options, :request_tm, :request_count
+    attr_reader :runtime_options
 
     def initialize(options)
       super(options)
 
       # init
       @runtime_options = RuntimeOptions.new
-      @request_tm = Time.at(0)
-      @request_count = 0
     end
 
     def run
@@ -31,24 +30,12 @@ module Sinew
     end
 
     #
-    # http requests and caching
+    # http requests
     #
 
-    def cache
-      @cache ||= Cache.new(self)
-    end
-
     def http(method, url, options = {})
       request = Request.new(self, method, url, options)
-
-      # try to get from cache
-      response = cache.get(request)
-
-      # perform if necessary
-      if !response
-        response = perform(request)
-        cache.set(response)
-      end
+      response = request.perform(connection)
 
       # always log error messages
       if response.error?
@@ -58,26 +45,10 @@ module Sinew
       response
     end
 
-    def perform(request)
-      before_perform_request(request)
-
-      response = nil
-
-      tries = runtime_options.retries + 1
-      while tries > 0
-        tries -= 1
-        begin
-          @request_count += 1
-          response = request.perform
-        rescue Timeout::Error
-          response = Response.from_timeout(request)
-        end
-        break if !response.error_500?
-      end
-
-      response
+    def connection
+      @connection ||= Connection.create(options: options, runtime_options: runtime_options)
     end
-    protected :perform
+    protected :connection
 
     #
     # output
@@ -91,24 +62,6 @@ module Sinew
     # helpers
     #
 
-    def before_perform_request(request)
-      # log
-      if !quiet?
-        msg = if request.method != 'get'
-          "req #{request.uri} (#{request.method})"
-        else
-          "req #{request.uri}"
-        end
-        $stderr.puts msg
-      end
-
-      # rate limit
-      sleep = (request_tm + runtime_options.rate_limit) - Time.now
-      sleep(sleep) if sleep > 0
-      @request_tm = Time.now
-    end
-    protected :before_perform_request
-
     def footer
       output.report
       finished = output.count > 0 ? "Finished #{output.filename}" : 'Finished'

data/lib/sinew/output.rb

@@ -1,5 +1,6 @@
 require 'csv'
-require 'stringex'
+require 'set'
+require 'sterile'
 
 #
 # CSV output.
@@ -7,11 +8,12 @@ require 'stringex'
 
 module Sinew
   class Output
-    attr_reader :sinew, :columns, :rows, :csv
+    attr_reader :sinew, :columns, :rows, :urls, :csv
 
     def initialize(sinew)
       @sinew = sinew
       @rows = []
+      @urls = Set.new
     end
 
     def filename
@@ -41,6 +43,9 @@ module Sinew
       # implicit header if necessary
       header(row.keys) if !csv
 
+      # don't allow duplicate urls
+      return if dup_url?(row)
+
       rows << row.dup
 
       # map columns to row, and normalize along the way
@@ -94,27 +99,17 @@ module Sinew
         s.to_s
       end
 
-      #
-      # Below uses stringex
-      #
-      # github.com/rsl/stringex/blob/master/lib/stringex/string_extensions.rb
-      # github.com/rsl/stringex/blob/master/lib/stringex/localization/conversion_expressions.rb
-      #
-
-      # <a>b</a> => b
-      s = s.strip_html_tags
+      # strip html tags. Note that we replace tags with spaces
+      s = s.gsub(/<[^>]+>/, ' ')
 
       # Converts MS Word 'smart punctuation' to ASCII
-      s = s.convert_smart_punctuation
+      s = Sterile.plain_format(s)
 
-      # "&aacute;".convert_accented_html_entities # => "a"
-      s = s.convert_accented_html_entities
+      # &aacute; &amp; etc.
+      s = Sterile.decode_entities(s)
 
-      # &amp, &frac, etc.
-      s = s.convert_miscellaneous_html_entities
-
-      # convert unicode => regular characters
-      s = s.to_ascii
+      # "šţɽĩɳģ" => "string"
+      s = Sterile.transliterate(s)
 
       # squish
       s = s.squish
@@ -122,5 +117,17 @@ module Sinew
       s
     end
     protected :normalize
+
+    def dup_url?(row)
+      if url = row[:url]
+        if urls.include?(url)
+          sinew.warning("duplicate url: #{url}") if !sinew.quiet?
+          return true
+        end
+        urls << url
+      end
+      false
+    end
+    protected :dup_url?
   end
 end

data/lib/sinew/request.rb

@@ -1,9 +1,8 @@
 require 'digest/md5'
-require 'httparty'
 require 'htmlentities'
 
 #
-# Process a single HTTP request. Mostly a wrapper around HTTParty.
+# Process a single HTTP request.
 #
 
 module Sinew
@@ -12,29 +11,43 @@ module Sinew
   class Request
     HTML_ENTITIES = HTMLEntities.new
     VALID_METHODS = %w[get post patch put delete head options].freeze
+    METHODS_WITH_BODY = %w[patch post put].freeze
 
-    attr_reader :sinew, :method, :uri, :options, :cache_key
+    attr_reader :sinew, :method, :uri, :options
 
-    # Options are largely compatible with HTTParty, except for :method.
+    # Supported options:
+    #  body: Body of http post
+    #  headers: Hash of HTTP headers (combined with runtime_options.headers)
+    #  query: Hash of query parameters to add to url
     def initialize(sinew, method, url, options = {})
       @sinew = sinew
       @method = method
       @options = options.dup
       @uri = parse_url(url)
-      @cache_key = calculate_cache_key
+    end
+
+    def proxy
+      @proxy ||= begin
+        if proxies = sinew.options[:proxy]
+          proxies.split(',').sample
+        end
+      end
     end
 
     # run the request, return the result
-    def perform
+    def perform(connection)
       validate!
 
-      # merge global/options headers
       headers = sinew.runtime_options.headers
       headers = headers.merge(options[:headers]) if options[:headers]
-      options[:headers] = headers
 
-      party_response = HTTParty.send(method, uri, options)
-      Response.from_network(self, party_response)
+      body = options.delete(:body)
+
+      fday_response = connection.send(method, uri, body, headers) do
+        _1.options[:proxy] = proxy
+      end
+
+      Response.from_network(self, fday_response)
     end
 
     # We accept sloppy urls and attempt to clean them up
@@ -48,11 +61,11 @@ module Sinew
       s = s.gsub(' ', '%20')
       s = s.gsub("'", '%27')
 
-      # append query manually (instead of letting HTTParty handle it) so we can
-      # include it in cache_key
+      # append query manually (instead of letting Faraday handle it) for consistent
+      # Request#uri and Response#uri
       query = options.delete(:query)
       if query.present?
-        q = HTTParty::HashConversions.to_params(query)
+        q = Faraday::Utils.default_params_encoder.encode(query)
         separator = s.include?('?') ? '&' : '?'
         s = "#{s}#{separator}#{q}"
       end
@@ -61,44 +74,10 @@ module Sinew
     end
     protected :parse_url
 
-    def calculate_cache_key
-      dir = pathify(uri.host)
-
-      body_key = if body.is_a?(Hash)
-        HTTParty::HashConversions.to_params(body)
-      else
-        body&.dup
-      end
-
-      # build key, as a hash for before_generate_cache_key
-      key = {
-        method: method.dup,
-        path: uri.path,
-        query: uri.query,
-        body: body_key,
-      }
-      key = sinew.runtime_options.before_generate_cache_key.call(key)
-
-      # strip method for gets
-      key.delete(:method) if key[:method] == 'get'
-
-      # pull out the values, join and pathify
-      path = key.values.select(&:present?).join(',')
-      path = pathify(path)
-
-      # shorten long paths
-      if path.length > 250
-        path = Digest::MD5.hexdigest(path)
-      end
-
-      "#{dir}/#{path}"
-    end
-    protected :calculate_cache_key
-
     def validate!
       raise "invalid method #{method}" if !VALID_METHODS.include?(method)
       raise "invalid url #{uri}" if uri.scheme !~ /^http/
-      raise "can't get with a body" if method == 'get' && body
+      raise "can't #{method} with a body" if body && !METHODS_WITH_BODY.include?(method)
       raise "Content-Type doesn't make sense without a body" if content_type && !body
     end
     protected :validate!
@@ -134,7 +113,7 @@ module Sinew
       s = s.gsub(',,', ',')
       # encode invalid path chars
       s = s.gsub(/[^A-Za-z0-9_.,=-]/) do |i|
-        hex = i.unpack('H2').first
+        hex = i.unpack1('H2')
         "%#{hex}"
       end
       # handle empty case