sinew 2.0.3 → 4.0.0

data/lib/sinew/core_ext.rb

@@ -1,59 +0,0 @@
-#
-# A few core extensions brought over from ActiveSupport. These are handy for
-# parsing.
-#
-
-class String
-  def squish
-    dup.squish!
-  end
-
-  def squish!
-    strip!
-    gsub!(/\s+/, ' ')
-    self
-  end
-
-  def first(limit = 1)
-    if limit == 0
-      ''
-    elsif limit >= size
-      dup
-    else
-      self[0..limit - 1]
-    end
-  end
-
-  def last(limit = 1)
-    if limit == 0
-      ''
-    elsif limit >= size
-      dup
-    else
-      self[-limit..-1]
-    end
-  end
-
-  alias starts_with? start_with?
-  alias ends_with? end_with?
-end
-
-#
-# blank?/present?
-#
-
-class Object
-  def blank?
-    respond_to?(:empty?) ? !!empty? : !self
-  end
-
-  def present?
-    !blank?
-  end
-end
-
-class String
-  def blank?
-    !!(self =~ /\A\s*\z/)
-  end
-end

data/lib/sinew/dsl.rb

@@ -1,114 +0,0 @@
-require 'awesome_print'
-require 'cgi'
-
-#
-# The DSL available to .sinew files.
-#
-
-module Sinew
-  class DSL
-    # this is used to break out of --limit
-    class LimitError < StandardError; end
-
-    attr_reader :sinew, :raw, :uri, :elapsed
-
-    def initialize(sinew)
-      @sinew = sinew
-    end
-
-    def run
-      tm = Time.now
-      begin
-        recipe = sinew.options[:recipe]
-        instance_eval(File.read(recipe, mode: 'rb'), recipe)
-      rescue LimitError
-        # ignore - this is flow control for --limit
-      end
-      @elapsed = Time.now - tm
-    end
-
-    #
-    # request
-    #
-
-    def get(url, query = {})
-      http('get', url, query: query)
-    end
-
-    def post(url, form = {})
-      body = form
-      headers = {
-        'Content-Type' => 'application/x-www-form-urlencoded',
-      }
-      http('post', url, body: body, headers: headers)
-    end
-
-    def post_json(url, json = {})
-      body = json.to_json
-      headers = {
-        'Content-Type' => 'application/json',
-      }
-      http('post', url, body: body, headers: headers)
-    end
-
-    def http(method, url, options = {})
-      # these need to be cleared before each request
-      %i[@html @noko @xml @json].each do |i|
-        instance_variable_set(i, nil)
-      end
-
-      # fetch and make response available to callers
-      response = sinew.http(method, url, options)
-      @uri, @raw = response.uri, response.body
-
-      # don't confuse the user
-      nil
-    end
-
-    #
-    # response
-    #
-
-    def html
-      @html ||= begin
-        s = raw.dup
-        # squish!
-        s.squish!
-        # kill whitespace around tags
-        s.gsub!(/ ?<([^>]+)> ?/, '<\\1>')
-        s
-      end
-    end
-
-    def noko
-      @noko ||= Nokogiri::HTML(html)
-    end
-
-    def xml
-      @xml ||= Nokogiri::XML(html)
-    end
-
-    def json
-      @json ||= JSON.parse(raw, symbolize_names: true)
-    end
-
-    def url
-      uri.to_s
-    end
-
-    #
-    # csv
-    #
-
-    def csv_header(*args)
-      sinew.output.header(args)
-    end
-
-    def csv_emit(row)
-      sinew.output.emit(row)
-      if sinew.output.count == sinew.options[:limit]
-        raise LimitError.new
-      end
-    end
-  end
-end

data/lib/sinew/output.rb

@@ -1,149 +0,0 @@
-require 'csv'
-require 'set'
-require 'stringex'
-
-#
-# Stringex customizations
-#
-
-# turn '&' into '&', not 'and'
-Stringex::Localization::DefaultConversions::HTML_ENTITIES[:amp] = '&'
-
-#
-# CSV output.
-#
-
-module Sinew
-  class Output
-    attr_reader :sinew, :columns, :rows, :urls, :csv
-
-    def initialize(sinew)
-      @sinew = sinew
-      @rows = []
-      @urls = Set.new
-    end
-
-    def filename
-      @filename ||= begin
-        recipe = sinew.options[:recipe]
-        ext = File.extname(recipe)
-        if ext.empty?
-          "#{recipe}.csv"
-        else
-          recipe.gsub(ext, '.csv')
-        end
-      end
-    end
-
-    def header(columns)
-      sinew.banner("Writing to #{filename}...") if !sinew.quiet?
-
-      columns = columns.flatten
-      @columns = columns
-
-      # open csv, write header row
-      @csv = CSV.open(filename, 'wb')
-      csv << columns
-    end
-
-    def emit(row)
-      # implicit header if necessary
-      header(row.keys) if !csv
-
-      # don't allow duplicate urls
-      return if dup_url?(row)
-      rows << row.dup
-
-      # map columns to row, and normalize along the way
-      print = {}
-      row = columns.map do |i|
-        value = normalize(row[i])
-        print[i] = value if value.present?
-        value
-      end
-
-      # print
-      sinew.vputs print.ai
-
-      csv << row
-      csv.flush
-    end
-
-    def count
-      rows.length
-    end
-
-    def report
-      return if count == 0
-
-      sinew.banner("Got #{count} rows.")
-
-      # calculate counts
-      counts = Hash.new(0)
-      rows.each do |row|
-        row.each_pair { |k, v| counts[k] += 1 if v.present? }
-      end
-      # sort by counts
-      cols = columns.sort_by { |i| [ -counts[i], i ] }
-
-      # report
-      len = cols.map { |i| i.to_s.length }.max
-      fmt = "  %-#{len + 1}s %7d / %-7d %6.1f%%\n"
-      cols.each do |col|
-        $stderr.printf(fmt, col, counts[col], count, counts[col] * 100.0 / count)
-      end
-    end
-
-    def normalize(s)
-      # noko/array/misc => string
-      s = case s
-      when Nokogiri::XML::Element, Nokogiri::XML::NodeSet
-        s.inner_html
-      when Array
-        s.map(&:to_s).join('|')
-      else
-        s.to_s
-      end
-
-      # strip html tags. Note that we replace tags with spaces
-      s = s.gsub(/<[^>]+>/, ' ')
-
-      #
-      # Below uses stringex
-      #
-      # github.com/rsl/stringex/blob/master/lib/stringex/string_extensions.rb
-      # github.com/rsl/stringex/blob/master/lib/stringex/localization/conversion_expressions.rb
-      #
-
-      # Converts MS Word 'smart punctuation' to ASCII
-      s = s.convert_smart_punctuation
-
-      # "&aacute;".convert_accented_html_entities # => "a"
-      s = s.convert_accented_html_entities
-
-      # &amp, &frac, etc.
-      s = s.convert_miscellaneous_html_entities
-
-      # convert unicode => regular characters
-      s = s.to_ascii
-
-      # squish
-      s = s.squish
-
-      s
-    end
-    protected :normalize
-
-    def dup_url?(row)
-      if url = row[:url]
-        if urls.include?(url)
-          sinew.warning("duplicate url: #{url}") if !sinew.quiet?
-          return true
-        end
-        urls << url
-      end
-      false
-    end
-    protected :dup_url?
-  end
-end

data/lib/sinew/request.rb

@@ -1,151 +0,0 @@
-require 'digest/md5'
-require 'httparty'
-require 'htmlentities'
-
-#
-# Process a single HTTP request. Mostly a wrapper around HTTParty.
-#
-
-module Sinew
-  class Error < StandardError; end
-
-  class Request
-    HTML_ENTITIES = HTMLEntities.new
-    VALID_METHODS = %w[get post patch put delete head options].freeze
-
-    attr_reader :sinew, :method, :uri, :options, :cache_key
-
-    # Options are largely compatible with HTTParty, except for :method.
-    def initialize(sinew, method, url, options = {})
-      @sinew = sinew
-      @method = method
-      @options = options.dup
-      @uri = parse_url(url)
-      @cache_key = calculate_cache_key
-    end
-
-    # run the request, return the result
-    def perform
-      validate!
-
-      # merge optons
-      options = self.options.merge(sinew.runtime_options.httparty_options)
-
-      # merge headers
-      headers = sinew.runtime_options.headers
-      headers = headers.merge(options[:headers]) if options[:headers]
-      options[:headers] = headers
-
-      party_response = HTTParty.send(method, uri, options)
-      Response.from_network(self, party_response)
-    end
-
-    # We accept sloppy urls and attempt to clean them up
-    def parse_url(url)
-      s = url
-
-      # remove entities
-      s = HTML_ENTITIES.decode(s)
-
-      # fix a couple of common encoding bugs
-      s = s.gsub(' ', '%20')
-      s = s.gsub("'", '%27')
-
-      # append query manually (instead of letting HTTParty handle it) so we can
-      # include it in cache_key
-      query = options.delete(:query)
-      if query.present?
-        q = HTTParty::HashConversions.to_params(query)
-        separator = s.include?('?') ? '&' : '?'
-        s = "#{s}#{separator}#{q}"
-      end
-
-      URI.parse(s)
-    end
-    protected :parse_url
-
-    def calculate_cache_key
-      dir = pathify(uri.host)
-
-      body_key = if body.is_a?(Hash)
-        HTTParty::HashConversions.to_params(body)
-      else
-        body&.dup
-      end
-
-      # build key, as a hash for before_generate_cache_key
-      key = {
-        method: method.dup,
-        path: uri.path,
-        query: uri.query,
-        body: body_key,
-      }
-      key = sinew.runtime_options.before_generate_cache_key.call(key)
-
-      # strip method for gets
-      key.delete(:method) if key[:method] == 'get'
-
-      # pull out the values, join and pathify
-      path = key.values.select(&:present?).join(',')
-      path = pathify(path)
-
-      # shorten long paths
-      if path.length > 250
-        path = Digest::MD5.hexdigest(path)
-      end
-
-      "#{dir}/#{path}"
-    end
-    protected :calculate_cache_key
-
-    def validate!
-      raise "invalid method #{method}" if !VALID_METHODS.include?(method)
-      raise "invalid url #{uri}" if uri.scheme !~ /^http/
-      raise "can't get with a body" if method == 'get' && body
-      raise "Content-Type doesn't make sense without a body" if content_type && !body
-    end
-    protected :validate!
-
-    def body
-      options[:body]
-    end
-    protected :body
-
-    def headers
-      options[:headers]
-    end
-    protected :headers
-
-    def content_type
-      headers && headers['Content-Type']
-    end
-    protected :content_type
-
-    def form?
-      content_type == 'application/x-www-form-urlencoded'
-    end
-    protected :form?
-
-    def pathify(s)
-      # remove leading slash
-      s = s.gsub(/^\//, '')
-      # .. => comma
-      s = s.gsub('..', ',')
-      # query separators => comma
-      s = s.gsub(/[?\/&]/, ',')
-      # ,, => comma
-      s = s.gsub(',,', ',')
-      # encode invalid path chars
-      s = s.gsub(/[^A-Za-z0-9_.,=-]/) do |i|
-        hex = i.unpack('H2').first
-        "%#{hex}"
-      end
-      # handle empty case
-      s = '_root_' if s.blank?
-      # always downcase
-      s = s.downcase
-      s
-    end
-    protected :pathify
-  end
-end