sinew 1.0.4 → 2.0.0

data/lib/sinew/output.rb

@@ -0,0 +1,126 @@
+require 'csv'
+require 'stringex'
+
+#
+# CSV output.
+#
+
+module Sinew
+  class Output
+    attr_reader :sinew, :columns, :rows, :csv
+
+    def initialize(sinew)
+      @sinew = sinew
+      @rows = []
+    end
+
+    def filename
+      @filename ||= begin
+        recipe = sinew.options[:recipe]
+        ext = File.extname(recipe)
+        if ext.empty?
+          "#{recipe}.csv"
+        else
+          recipe.gsub(ext, '.csv')
+        end
+      end
+    end
+
+    def header(columns)
+      sinew.banner("Writing to #{filename}...") if !sinew.quiet?
+
+      columns = columns.flatten
+      @columns = columns
+
+      # open csv, write header row
+      @csv = CSV.open(filename, 'wb')
+      csv << columns
+    end
+
+    def emit(row)
+      # implicit header if necessary
+      header(row.keys) if !csv
+
+      rows << row.dup
+
+      # map columns to row, and normalize along the way
+      print = {}
+      row = columns.map do |i|
+        value = normalize(row[i])
+        print[i] = value if value.present?
+        value
+      end
+
+      # print
+      sinew.vputs print.ai
+
+      csv << row
+      csv.flush
+    end
+
+    def count
+      rows.length
+    end
+
+    def report
+      return if count == 0
+
+      sinew.banner("Got #{count} rows.")
+
+      # calculate counts
+      counts = Hash.new(0)
+      rows.each do |row|
+        row.each_pair { |k, v| counts[k] += 1 if v.present? }
+      end
+      # sort by counts
+      cols = columns.sort_by { |i| [ -counts[i], i ] }
+
+      # report
+      len = cols.map { |i| i.to_s.length }.max
+      fmt = "  %-#{len + 1}s %7d / %-7d %6.1f%%\n"
+      cols.each do |col|
+        $stderr.printf(fmt, col, counts[col], count, counts[col] * 100.0 / count)
+      end
+    end
+
+    def normalize(s)
+      # noko/array/misc => string
+      s = case s
+      when Nokogiri::XML::Element, Nokogiri::XML::NodeSet
+        s.inner_html
+      when Array
+        s.map(&:to_s).join('|')
+      else
+        s.to_s
+      end
+
+      #
+      # Below uses stringex
+      #
+      # github.com/rsl/stringex/blob/master/lib/stringex/string_extensions.rb
+      # github.com/rsl/stringex/blob/master/lib/stringex/localization/conversion_expressions.rb
+      #
+
+      # <a>b</a> => b
+      s = s.strip_html_tags
+
+      # Converts MS Word 'smart punctuation' to ASCII
+      s = s.convert_smart_punctuation
+
+      # "&aacute;".convert_accented_html_entities # => "a"
+      s = s.convert_accented_html_entities
+
+      # &amp, &frac, etc.
+      s = s.convert_miscellaneous_html_entities
+
+      # convert unicode => regular characters
+      s = s.to_ascii
+
+      # squish
+      s = s.squish
+
+      s
+    end
+    protected :normalize
+  end
+end

data/lib/sinew/request.rb

@@ -0,0 +1,148 @@
+require 'digest/md5'
+require 'httparty'
+require 'htmlentities'
+
+#
+# Process a single HTTP request. Mostly a wrapper around HTTParty.
+#
+
+module Sinew
+  class Error < StandardError; end
+
+  class Request
+    HTML_ENTITIES = HTMLEntities.new
+    VALID_METHODS = %w[get post patch put delete head options].freeze
+
+    attr_reader :sinew, :method, :uri, :options, :cache_key
+
+    # Options are largely compatible with HTTParty, except for :method.
+    def initialize(sinew, method, url, options = {})
+      @sinew = sinew
+      @method = method
+      @options = options.dup
+      @uri = parse_url(url)
+      @cache_key = calculate_cache_key
+    end
+
+    # run the request, return the result
+    def perform
+      validate!
+
+      # merge global/options headers
+      headers = sinew.runtime_options.headers
+      headers = headers.merge(options[:headers]) if options[:headers]
+      options[:headers] = headers
+
+      party_response = HTTParty.send(method, uri, options)
+      Response.from_network(self, party_response)
+    end
+
+    # We accept sloppy urls and attempt to clean them up
+    def parse_url(url)
+      s = url
+
+      # remove entities
+      s = HTML_ENTITIES.decode(s)
+
+      # fix a couple of common encoding bugs
+      s = s.gsub(' ', '%20')
+      s = s.gsub("'", '%27')
+
+      # append query manually (instead of letting HTTParty handle it) so we can
+      # include it in cache_key
+      query = options.delete(:query)
+      if query.present?
+        q = HTTParty::HashConversions.to_params(query)
+        separator = s.include?('?') ? '&' : '?'
+        s = "#{s}#{separator}#{q}"
+      end
+
+      URI.parse(s)
+    end
+    protected :parse_url
+
+    def calculate_cache_key
+      dir = pathify(uri.host)
+
+      body_key = if body.is_a?(Hash)
+        HTTParty::HashConversions.to_params(body)
+      else
+        body&.dup
+      end
+
+      # build key, as a hash for before_generate_cache_key
+      key = {
+        method: method.dup,
+        path: uri.path,
+        query: uri.query,
+        body: body_key,
+      }
+      key = sinew.runtime_options.before_generate_cache_key.call(key)
+
+      # strip method for gets
+      key.delete(:method) if key[:method] == 'get'
+
+      # pull out the values, join and pathify
+      path = key.values.select(&:present?).join(',')
+      path = pathify(path)
+
+      # shorten long paths
+      if path.length > 250
+        path = Digest::MD5.hexdigest(path)
+      end
+
+      "#{dir}/#{path}"
+    end
+    protected :calculate_cache_key
+
+    def validate!
+      raise "invalid method #{method}" if !VALID_METHODS.include?(method)
+      raise "invalid url #{uri}" if uri.scheme !~ /^http/
+      raise "can't get with a body" if method == 'get' && body
+      raise "Content-Type doesn't make sense without a body" if content_type && !body
+    end
+    protected :validate!
+
+    def body
+      options[:body]
+    end
+    protected :body
+
+    def headers
+      options[:headers]
+    end
+    protected :headers
+
+    def content_type
+      headers && headers['Content-Type']
+    end
+    protected :content_type
+
+    def form?
+      content_type == 'application/x-www-form-urlencoded'
+    end
+    protected :form?
+
+    def pathify(s)
+      # remove leading slash
+      s = s.gsub(/^\//, '')
+      # .. => comma
+      s = s.gsub('..', ',')
+      # query separators => comma
+      s = s.gsub(/[?\/&]/, ',')
+      # ,, => comma
+      s = s.gsub(',,', ',')
+      # encode invalid path chars
+      s = s.gsub(/[^A-Za-z0-9_.,=-]/) do |i|
+        hex = i.unpack('H2').first
+        "%#{hex}"
+      end
+      # handle empty case
+      s = '_root_' if s.blank?
+      # always downcase
+      s = s.downcase
+      s
+    end
+    protected :pathify
+  end
+end

data/lib/sinew/response.rb

@@ -0,0 +1,75 @@
+#
+# An HTTP response. Mostly a wrapper around HTTParty.
+#
+
+module Sinew
+  class Response
+    attr_accessor :request, :uri, :body, :code, :headers
+
+    def self.from_network(request, party_response)
+      Response.new.tap do |response|
+        response.request = request
+        response.uri = party_response.request.last_uri
+        response.code = party_response.code
+        response.headers = party_response.headers.to_h
+
+        # force to utf-8 as best we can
+        body = party_response.body
+        if body.encoding != Encoding::UTF_8
+          body = body.encode('UTF-8', invalid: :replace, undef: :replace, replace: '?')
+        end
+        response.body = body
+      end
+    end
+
+    def self.from_cache(request, body, head)
+      Response.new.tap do |response|
+        response.request = request
+        response.body = body
+
+        # defaults
+        response.uri = request.uri
+        response.code = 200
+        response.headers = {}
+
+        # overwrite with cached response headers
+        if head
+          head = JSON.parse(head, symbolize_names: true)
+          response.uri = URI.parse(head[:uri])
+          response.code = head[:code]
+          response.headers = head[:headers]
+        end
+      end
+    end
+
+    def self.from_timeout(request)
+      Response.new.tap do |response|
+        response.request = request
+        response.uri = request.uri
+        response.body = 'timeout'
+        response.code = 999
+        response.headers = {}
+      end
+    end
+
+    def error?
+      code >= 400
+    end
+
+    def error_500?
+      code / 100 >= 5
+    end
+
+    def redirected?
+      request.uri != uri
+    end
+
+    def head_as_json
+      {
+        uri: uri,
+        code: code,
+        headers: headers,
+      }
+    end
+  end
+end

data/lib/sinew/runtime_options.rb

@@ -0,0 +1,26 @@
+#
+# Runtime options that sinew files can modify.
+#
+
+module Sinew
+  class RuntimeOptions
+    attr_accessor :retries
+    attr_accessor :rate_limit
+    attr_accessor :headers
+    attr_accessor :before_generate_cache_key
+
+    def initialize
+      self.retries = 3
+      self.rate_limit = 1
+      self.headers = {
+        'User-Agent' => "sinew/#{VERSION}",
+      }
+      self.before_generate_cache_key = ->(i) { i }
+
+      # for testing
+      if ENV['SINEW_TEST']
+        self.rate_limit = 0
+      end
+    end
+  end
+end

data/lib/sinew/version.rb

@@ -1,4 +1,4 @@
 module Sinew
   # Gem version
-  VERSION = "1.0.4"
+  VERSION = '2.0.0'.freeze
 end

data/sample.sinew

@@ -1,7 +1,9 @@
-get "http://httpbin.org"
-noko.css("ul li a").each do |a|
-  row = { }
+get 'http://httpbin.org'
+noko.css('ul li a').each do |a|
+  row = {}
   row[:url] = a[:href]
   row[:title] = a.text
   csv_emit(row)
 end
+
+get 'http://httpbin.org/redirect/2'

data/sinew.gemspec

@@ -1,29 +1,34 @@
-$LOAD_PATH << File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift("#{__dir__}/lib")
 
-require "sinew/version"
+require 'sinew/version'
 
 Gem::Specification.new do |s|
-  s.name        = "sinew"
+  s.name        = 'sinew'
   s.version     = Sinew::VERSION
   s.platform    = Gem::Platform::RUBY
-  s.authors     = ["Adam Doppelt"]
-  s.email       = ["amd@gurge.com"]
-  s.homepage    = "http://github.com/gurgeous/sinew"
-  s.summary     = "Sinew - structured web crawling using recipes."
-  s.description = "Crawl web sites easily using ruby recipes, with caching and nokogiri."
+  s.license     = 'MIT'
+  s.authors     = [ 'Adam Doppelt' ]
+  s.email       = [ 'amd@gurge.com' ]
+  s.homepage    = 'http://github.com/gurgeous/sinew'
+  s.summary     = 'Sinew - structured web crawling using recipes.'
+  s.description = 'Crawl web sites easily using ruby recipes, with caching and nokogiri.'
+  s.required_ruby_version = '~> 2.3'
 
-  s.rubyforge_project = "sinew"
+  s.rubyforge_project = 'sinew'
 
-  s.add_runtime_dependency "activesupport", "~> 3.0"
-  s.add_runtime_dependency "awesome_print"
-  s.add_runtime_dependency "htmlentities"
-  s.add_runtime_dependency "nokogiri"
-  s.add_runtime_dependency "stringex", "~> 2.0"
-  s.add_runtime_dependency "trollop"
-  s.add_development_dependency "rake"
+  s.add_runtime_dependency 'awesome_print', '~> 1.8'
+  s.add_runtime_dependency 'htmlentities', '~> 4.3'
+  s.add_runtime_dependency 'httparty', '~> 0.16'
+  s.add_runtime_dependency 'nokogiri', '~> 1.8'
+  s.add_runtime_dependency 'scripto', '~> 0'
+  s.add_runtime_dependency 'slop', '~> 4.6'
+  s.add_runtime_dependency 'stringex', '~> 2.8'
+  s.add_development_dependency 'minitest', '~> 5.11'
+  s.add_development_dependency 'rake', '~> 12.3'
+  s.add_development_dependency 'webmock', '~> 3.4'
 
   s.files         = `git ls-files`.split("\n")
-  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
-  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
-  s.require_paths = ["lib"]
+  s.test_files    = `git ls-files -- test/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
+  s.require_paths = [ 'lib' ]
 end