sinew 3.0.1 → 4.0.1

data/lib/sinew/dsl.rb

@@ -1,115 +0,0 @@
-require 'amazing_print'
-require 'cgi'
-require 'json'
-
-#
-# The DSL available to .sinew files.
-#
-
-module Sinew
-  class DSL
-    # this is used to break out of --limit
-    class LimitError < StandardError; end
-
-    attr_reader :sinew, :uri, :raw, :code, :elapsed
-
-    def initialize(sinew)
-      @sinew = sinew
-    end
-
-    def run
-      tm = Time.now
-      begin
-        recipe = sinew.options[:recipe]
-        instance_eval(File.read(recipe, mode: 'rb'), recipe)
-      rescue LimitError
-        # ignore - this is flow control for --limit
-      end
-      @elapsed = Time.now - tm
-    end
-
-    #
-    # request
-    #
-
-    def get(url, query = {})
-      http('get', url, query: query)
-    end
-
-    def post(url, form = {})
-      body = form
-      headers = {
-        'Content-Type' => 'application/x-www-form-urlencoded',
-      }
-      http('post', url, body: body, headers: headers)
-    end
-
-    def post_json(url, json = {})
-      body = json.to_json
-      headers = {
-        'Content-Type' => 'application/json',
-      }
-      http('post', url, body: body, headers: headers)
-    end
-
-    def http(method, url, options = {})
-      # these need to be cleared before each request
-      %i[@html @noko @xml @json].each do |i|
-        instance_variable_set(i, nil)
-      end
-
-      # fetch and make response available to callers
-      response = sinew.http(method, url, options)
-      @uri, @raw, @code = response.uri, response.body, response.code
-
-      # don't confuse the user
-      nil
-    end
-
-    #
-    # response
-    #
-
-    def html
-      @html ||= begin
-        s = raw.dup
-        # squish!
-        s.squish!
-        # kill whitespace around tags
-        s.gsub!(/ ?<([^>]+)> ?/, '<\\1>')
-        s
-      end
-    end
-
-    def noko
-      @noko ||= Nokogiri::HTML(html)
-    end
-
-    def xml
-      @xml ||= Nokogiri::XML(html)
-    end
-
-    def json
-      @json ||= JSON.parse(raw, symbolize_names: true)
-    end
-
-    def url
-      uri.to_s
-    end
-
-    #
-    # csv
-    #
-
-    def csv_header(*args)
-      sinew.output.header(args)
-    end
-
-    def csv_emit(row)
-      sinew.output.emit(row)
-      if sinew.output.count == sinew.options[:limit]
-        raise LimitError.new
-      end
-    end
-  end
-end

data/lib/sinew/output.rb

@@ -1,133 +0,0 @@
-require 'csv'
-require 'set'
-require 'sterile'
-
-#
-# CSV output.
-#
-
-module Sinew
-  class Output
-    attr_reader :sinew, :columns, :rows, :urls, :csv
-
-    def initialize(sinew)
-      @sinew = sinew
-      @rows = []
-      @urls = Set.new
-    end
-
-    def filename
-      @filename ||= begin
-        recipe = sinew.options[:recipe]
-        ext = File.extname(recipe)
-        if ext.empty?
-          "#{recipe}.csv"
-        else
-          recipe.gsub(ext, '.csv')
-        end
-      end
-    end
-
-    def header(columns)
-      sinew.banner("Writing to #{filename}...") if !sinew.quiet?
-
-      columns = columns.flatten
-      @columns = columns
-
-      # open csv, write header row
-      @csv = CSV.open(filename, 'wb')
-      csv << columns
-    end
-
-    def emit(row)
-      # implicit header if necessary
-      header(row.keys) if !csv
-
-      # don't allow duplicate urls
-      return if dup_url?(row)
-
-      rows << row.dup
-
-      # map columns to row, and normalize along the way
-      print = {}
-      row = columns.map do |i|
-        value = normalize(row[i])
-        print[i] = value if value.present?
-        value
-      end
-
-      # print
-      sinew.vputs print.ai
-
-      csv << row
-      csv.flush
-    end
-
-    def count
-      rows.length
-    end
-
-    def report
-      return if count == 0
-
-      sinew.banner("Got #{count} rows.")
-
-      # calculate counts
-      counts = Hash.new(0)
-      rows.each do |row|
-        row.each_pair { |k, v| counts[k] += 1 if v.present? }
-      end
-      # sort by counts
-      cols = columns.sort_by { |i| [ -counts[i], i ] }
-
-      # report
-      len = cols.map { |i| i.to_s.length }.max
-      fmt = "  %-#{len + 1}s %7d / %-7d %6.1f%%\n"
-      cols.each do |col|
-        $stderr.printf(fmt, col, counts[col], count, counts[col] * 100.0 / count)
-      end
-    end
-
-    def normalize(s)
-      # noko/array/misc => string
-      s = case s
-      when Nokogiri::XML::Element, Nokogiri::XML::NodeSet
-        s.inner_html
-      when Array
-        s.map(&:to_s).join('|')
-      else
-        s.to_s
-      end
-
-      # strip html tags. Note that we replace tags with spaces
-      s = s.gsub(/<[^>]+>/, ' ')
-
-      # Converts MS Word 'smart punctuation' to ASCII
-      s = Sterile.plain_format(s)
-
-      # &aacute; &amp; etc.
-      s = Sterile.decode_entities(s)
-
-      # "šţɽĩɳģ" => "string"
-      s = Sterile.transliterate(s)
-
-      # squish
-      s = s.squish
-
-      s
-    end
-    protected :normalize
-
-    def dup_url?(row)
-      if url = row[:url]
-        if urls.include?(url)
-          sinew.warning("duplicate url: #{url}") if !sinew.quiet?
-          return true
-        end
-        urls << url
-      end
-      false
-    end
-    protected :dup_url?
-  end
-end

data/lib/sinew/request.rb

@@ -1,86 +0,0 @@
-require 'sterile'
-
-#
-# Process a single HTTP request.
-#
-
-module Sinew
-  class Error < StandardError; end
-
-  class Request
-    VALID_METHODS = %w[get post patch put delete head options].freeze
-    METHODS_WITH_BODY = %w[patch post put].freeze
-
-    attr_reader :method, :options, :uri
-
-    # Supported options:
-    #  body: Body of http post
-    #  headers: Hash of HTTP headers (combined with runtime_options.headers)
-    #  query: Hash of query parameters to add to url
-    def initialize(method, url, options = {})
-      @method = method
-      @options = options.dup
-      @uri = parse_url(url)
-    end
-
-    # run the request, return the result
-    def perform(connection)
-      validate!
-
-      body = options.delete(:body)
-      fday_response = connection.send(method, uri, body) do
-        _1.headers.update(options[:headers]) if options[:headers]
-        _1.options[:proxy] = options[:proxy]
-      end
-
-      Response.from_network(self, fday_response)
-    end
-
-    # We accept sloppy urls and attempt to clean them up
-    def parse_url(url)
-      s = url.to_s
-
-      # remove entities
-      s = Sterile.decode_entities(s)
-
-      # fix a couple of common encoding bugs
-      s = s.gsub(' ', '%20')
-      s = s.gsub("'", '%27')
-
-      # append query manually (instead of letting Faraday handle it) for consistent
-      # Request#uri and Response#uri
-      query = options.delete(:query)
-      if query.present?
-        q = Faraday::Utils.default_params_encoder.encode(query)
-        separator = s.include?('?') ? '&' : '?'
-        s = "#{s}#{separator}#{q}"
-      end
-
-      URI.parse(s)
-    end
-    protected :parse_url
-
-    def validate!
-      raise "invalid method #{method}" if !VALID_METHODS.include?(method)
-      raise "invalid url #{uri}" if uri.scheme !~ /^http/
-      raise "can't #{method} with a body" if body && !METHODS_WITH_BODY.include?(method)
-      raise "Content-Type doesn't make sense without a body" if content_type && !body
-    end
-    protected :validate!
-
-    def body
-      options[:body]
-    end
-    protected :body
-
-    def headers
-      options[:headers]
-    end
-    protected :headers
-
-    def content_type
-      headers && headers['Content-Type']
-    end
-    protected :content_type
-  end
-end

data/lib/sinew/runtime_options.rb

@@ -1,28 +0,0 @@
-#
-# Runtime options that sinew files can modify.
-#
-
-module Sinew
-  class RuntimeOptions
-    attr_accessor :retries
-    attr_accessor :rate_limit
-    attr_accessor :headers
-    attr_accessor :httpdisk_options
-    attr_accessor :insecure
-
-    def initialize
-      self.retries = 3
-      self.rate_limit = 1
-      self.headers = {
-        'User-Agent' => "sinew/#{VERSION}",
-      }
-      self.httpdisk_options = {}
-      self.insecure = false
-
-      # for testing
-      if ENV['SINEW_TEST']
-        self.rate_limit = 0
-      end
-    end
-  end
-end