sinew 2.0.3 → 4.0.0

data/lib/sinew/csv.rb

@@ -0,0 +1,89 @@
+require 'csv'
+require 'sterile'
+
+module Sinew
+  class CSV
+    attr_reader :columns, :count, :csv, :path, :tally
+
+    def initialize(path)
+      @count = 0
+      @csv = nil
+      @path = path
+    end
+
+    # start writing the csv
+    def start(columns)
+      raise 'started twice' if started?
+
+      @columns = columns
+      @tally = columns.map { [_1, 0] }.to_h
+      @csv = ::CSV.open(path, 'wb').tap do
+        _1 << columns
+      end
+    end
+
+    # has this csv been started?
+    def started?
+      @csv != nil
+    end
+
+    # append a row
+    def emit(row)
+      # convert row to cols, and construct print (our return value)
+      print = {}
+      row = columns.map do
+        value = normalize(row[_1])
+        if value
+          print[_1] = value
+          tally[_1] += 1
+        end
+        value
+      end
+      @count += 1
+
+      # emit
+      csv << row
+      csv.flush
+
+      # return in case someone wants to pretty print this
+      print
+    end
+
+    ASCII_ONLY = begin
+      chars = (33..126).map(&:chr) - ['&']
+      /\A[#{Regexp.escape(chars.join)}\s]+\Z/
+    end.freeze
+
+    def normalize(s)
+      # nokogiri/array/misc => string
+      s = if s.respond_to?(:inner_html)
+        s.inner_html
+      elsif s.is_a?(Array)
+        s.join('|')
+      else
+        s.to_s
+      end
+      return if s.empty?
+
+      # simple attempt to strip tags. Note that we replace tags with spaces
+      s = s.gsub(/<[^>]+>/, ' ')
+
+      if s !~ ASCII_ONLY
+        # Converts MS Word 'smart punctuation' to ASCII
+        s = Sterile.plain_format(s)
+
+        # &aacute; &amp; etc.
+        s = Sterile.decode_entities(s)
+
+        # "šţɽĩɳģ" => "string"
+        s = Sterile.transliterate(s)
+      end
+
+      # squish
+      s = s.strip.gsub(/\s+/, ' ')
+      return if s.empty?
+
+      s
+    end
+  end
+end

data/lib/sinew/main.rb

@@ -1,125 +1,72 @@
-require 'scripto'
-
-#
-# Main sinew entry point.
-#
-
 module Sinew
-  class Main < Scripto::Main
-    attr_reader :runtime_options, :request_tm, :request_count
+  # Helper class used by sinew bin. This exists as an independent class solely
+  # for testing, otherwise it would be built into the bin script.
+  class Main
+    attr_reader :sinew
 
     def initialize(options)
-      super(options)
-
-      # init
-      @runtime_options = RuntimeOptions.new
-      @request_tm = Time.at(0)
-      @request_count = 0
-
-      if options[:proxy]
-        addr, port = options[:proxy].split(':')
-        runtime_options.httparty_options[:http_proxyaddr] = addr
-        runtime_options.httparty_options[:http_proxyport] = port || 80
+      options[:output] ||= begin
+        src = options[:recipe]
+        dst = File.join(File.dirname(src), "#{File.basename(src, File.extname(src))}.csv")
+        dst = dst.sub(%r{^./}, '') # nice to clean this up
+        dst
       end
-    end
 
-    def run
-      dsl.run
-      footer if !quiet?
+      @sinew = Sinew::Base.new(options)
     end
 
-    def quiet?
-      options[:quiet]
+    def run
+      tm = Time.now
+      header if !sinew.options[:silent]
+      recipe = sinew.options[:recipe]
+      dsl = DSL.new(sinew)
+      begin
+        dsl.instance_eval(File.read(recipe, mode: 'rb'), recipe)
+      rescue LimitError
+        # ignore - this is flow control for --limit
+      end
+      footer(Time.now - tm) if !sinew.options[:silent]
     end
 
-    def dsl
-      @dsl ||= DSL.new(self)
-    end
+    protected
 
     #
-    # http requests and caching
+    # header/footer
     #
 
-    def cache
-      @cache ||= Cache.new(self)
+    def header
+      sinew.banner("Writing to #{sinew.csv.path}...")
     end
 
-    def http(method, url, options = {})
-      request = Request.new(self, method, url, options)
-
-      # try to get from cache
-      response = cache.get(request)
+    def footer(elapsed)
+      csv = sinew.csv
+      count = csv.count
 
-      # perform if necessary
-      if !response
-        response = perform(request)
-        cache.set(response)
+      if count == 0
+        sinew.banner(format('Done in %ds. Nothing written.', elapsed))
+        return
       end
 
-      # always log error messages
-      if response.error?
-        puts "xxx http request failed with #{response.code}"
-      end
-
-      response
-    end
-
-    def perform(request)
-      before_perform_request(request)
-
-      response = nil
+      # summary
+      msg = format('Done in %ds. Wrote %d rows to %s. Summary:', elapsed, count, csv.path)
+      sinew.banner(msg)
 
-      tries = runtime_options.retries + 1
-      while tries > 0
-        tries -= 1
-        begin
-          @request_count += 1
-          response = request.perform
-        rescue Timeout::Error
-          response = Response.from_timeout(request)
-        end
-        break if !response.error_500?
+      # tally
+      tally = csv.tally.sort_by { [-_2, _1.to_s] }.to_h
+      len = tally.keys.map { _1.to_s.length }.max
+      fmt = "  %-#{len + 1}s %7d/%-7d %5.1f%%\n"
+      tally.each do
+        printf(fmt, _1, _2, count, _2 * 100.0 / count)
       end
-
-      response
     end
-    protected :perform
 
-    #
-    # output
-    #
-
-    def output
-      @output ||= Output.new(self)
-    end
+    # simple DSL for .sinew files
+    class DSL
+      attr_reader :sinew
 
-    #
-    # helpers
-    #
-
-    def before_perform_request(request)
-      # log
-      if !quiet?
-        msg = if request.method != 'get'
-          "req #{request.uri} (#{request.method})"
-        else
-          "req #{request.uri}"
-        end
-        $stderr.puts msg
+      def initialize(sinew)
+        @sinew = sinew
       end
-
-      # rate limit
-      sleep = (request_tm + runtime_options.rate_limit) - Time.now
-      sleep(sleep) if sleep > 0
-      @request_tm = Time.now
-    end
-    protected :before_perform_request
-
-    def footer
-      output.report
-      finished = output.count > 0 ? "Finished #{output.filename}" : 'Finished'
-      banner("#{finished} in #{dsl.elapsed.to_i}s.")
     end
-    protected :footer
   end
 end

data/lib/sinew/middleware/log_formatter.rb

@@ -0,0 +1,23 @@
+module Sinew
+  module Middleware
+    # Minimalist Formatter that logs proxy if present.
+    class LogFormatter < Faraday::Logging::Formatter
+      def request(env)
+        info('req') do
+          # Only log the initial request, not the redirects
+          return if env[:redirect]
+
+          msg = apply_filters(env.url.to_s)
+          msg = "#{msg} (#{env.method})" if env.method != :get
+          msg = "#{msg} => #{env.request.proxy.uri}" if env.request.proxy
+
+          msg
+        end
+      end
+
+      def response(env)
+        # silent
+      end
+    end
+  end
+end

data/lib/sinew/nokogiri_ext.rb

@@ -1,28 +1,19 @@
 require 'nokogiri'
 
 # modify NodeSet to join with SPACE instead of empty string
-class Nokogiri::XML::NodeSet
-  alias old_inner_html inner_html
-  alias old_inner_text inner_text
+module Nokogiri
+  module XML
+    class NodeSet
+      alias old_inner_html inner_html
+      alias old_inner_text inner_text
 
-  def inner_text
-    map(&:inner_text).join(' ')
-  end
-
-  def inner_html(*args)
-    map { |i| i.inner_html(*args) }.join(' ')
-  end
-end
+      def inner_text
+        map(&:inner_text).join(' ')
+      end
 
-# text_just_me
-class Nokogiri::XML::Node
-  def text_just_me
-    t = children.find { |i| i.node_type == Nokogiri::XML::Node::TEXT_NODE }
-    t&.text
-  end
-end
-class Nokogiri::XML::NodeSet
-  def text_just_me
-    map(&:text_just_me).join(' ')
+      def inner_html(*args)
+        map { _1.inner_html(*args) }.join(' ')
+      end
+    end
   end
 end

data/lib/sinew/response.rb

@@ -1,121 +1,61 @@
-require 'stringio'
-require 'zlib'
-
-#
-# An HTTP response. Mostly a wrapper around HTTParty.
-#
+require 'delegate'
+require 'hashie/mash'
+require 'json'
+require 'nokogiri'
 
 module Sinew
-  class Response
-    attr_accessor :request, :uri, :body, :code, :headers
-
-    #
-    # factory methods
-    #
-
-    def self.from_network(request, party_response)
-      Response.new.tap do |response|
-        response.request = request
-        response.uri = party_response.request.last_uri
-        response.code = party_response.code
-        response.headers = party_response.headers.to_h
-        response.body = process_body(party_response)
-      end
-    end
-
-    def self.from_cache(request, body, head)
-      Response.new.tap do |response|
-        response.request = request
-        response.body = body
+  # A wrapper around Faraday::Response, with some parsing helpers.
+  class Response < SimpleDelegator
+    # Like body, but tries to cleanup whitespace around HTML for easier parsing.
+    def html
+      @html ||= body.dup.tap do
+        # fix invalid utf8
+        if _1.encoding == Encoding::UTF_8
+          _1.encode!('UTF-8', invalid: :replace, undef: :replace, replace: '?')
+        end
 
-        # defaults
-        response.uri = request.uri
-        response.code = 200
-        response.headers = {}
+        # squish
+        _1.strip!
+        _1.gsub!(/\s+/, ' ')
 
-        # overwrite with cached response headers
-        if head
-          if head !~ /^{/
-            return from_legacy_head(response, head)
-          end
-          head = JSON.parse(head, symbolize_names: true)
-          response.uri = URI.parse(head[:uri])
-          response.code = head[:code]
-          response.headers = head[:headers]
-        end
+        # kill whitespace around tags
+        _1.gsub!(/ ?<([^>]+)> ?/, '<\\1>')
       end
     end
 
-    def self.from_timeout(request)
-      Response.new.tap do |response|
-        response.request = request
-        response.uri = request.uri
-        response.body = 'timeout'
-        response.code = 999
-        response.headers = {}
-      end
+    # Return body as JSON
+    def json
+      @json ||= JSON.parse(body, symbolize_names: true)
     end
 
-    def self.from_legacy_head(response, head)
-      response.tap do |r|
-        case head
-        when /\ACURLER_ERROR/
-          # error
-          r.code = 999
-        when /\AHTTP/
-          # redirect
-          location = head.scan(/Location: ([^\r\n]+)/).flatten.last
-          r.uri += location
-        else
-          $stderr.puts "unknown cached /head for #{r.uri}"
-        end
-      end
+    # Return JSON body as Hashie::Mash
+    def mash
+      @mash ||= Hashie::Mash.new(json)
     end
 
-    # helper for decoding bodies before parsing
-    def self.process_body(response)
-      body = response.body
-
-      # inflate if necessary
-      bits = body[0, 10].force_encoding('BINARY')
-      if bits =~ /\A\x1f\x8b/n
-        body = Zlib::GzipReader.new(StringIO.new(body)).read
-      end
-
-      # force to utf-8 if we think this could be text
-      if body.encoding != Encoding::UTF_8
-        if content_type = response.headers['content-type']
-          if content_type =~ /\b(html|javascript|json|text|xml)\b/
-            body = body.encode('UTF-8', invalid: :replace, undef: :replace, replace: '?')
-          end
-        end
-      end
-
-      body
+    # Return body HTML as Nokogiri document
+    def noko
+      @noko ||= Nokogiri::HTML(html)
     end
 
-    #
-    # accessors
-    #
-
-    def error?
-      code >= 400
+    # Return body XML as Nokogiri document
+    def xml
+      @xml ||= Nokogiri::XML(html)
     end
 
-    def error_500?
-      code / 100 >= 5
+    # Return the final URI for the request, after redirects
+    def url
+      env.url
     end
 
-    def redirected?
-      request.uri != uri
+    # Return the cache diskpath for this response
+    def diskpath
+      env[:httpdisk_diskpath]
     end
 
-    def head_as_json
-      {
-        uri: uri,
-        code: code,
-        headers: headers,
-      }
+    # Remove cached response from disk, if any
+    def uncache
+      File.unlink(diskpath) if File.exist?(diskpath)
     end
   end
 end