sinew 3.0.1 → 4.0.0

data/lib/sinew/csv.rb

@@ -0,0 +1,89 @@
+require 'csv'
+require 'sterile'
+
+module Sinew
+  class CSV
+    attr_reader :columns, :count, :csv, :path, :tally
+
+    def initialize(path)
+      @count = 0
+      @csv = nil
+      @path = path
+    end
+
+    # start writing the csv
+    def start(columns)
+      raise 'started twice' if started?
+
+      @columns = columns
+      @tally = columns.map { [_1, 0] }.to_h
+      @csv = ::CSV.open(path, 'wb').tap do
+        _1 << columns
+      end
+    end
+
+    # has this csv been started?
+    def started?
+      @csv != nil
+    end
+
+    # append a row
+    def emit(row)
+      # convert row to cols, and construct print (our return value)
+      print = {}
+      row = columns.map do
+        value = normalize(row[_1])
+        if value
+          print[_1] = value
+          tally[_1] += 1
+        end
+        value
+      end
+      @count += 1
+
+      # emit
+      csv << row
+      csv.flush
+
+      # return in case someone wants to pretty print this
+      print
+    end
+
+    ASCII_ONLY = begin
+      chars = (33..126).map(&:chr) - ['&']
+      /\A[#{Regexp.escape(chars.join)}\s]+\Z/
+    end.freeze
+
+    def normalize(s)
+      # nokogiri/array/misc => string
+      s = if s.respond_to?(:inner_html)
+        s.inner_html
+      elsif s.is_a?(Array)
+        s.join('|')
+      else
+        s.to_s
+      end
+      return if s.empty?
+
+      # simple attempt to strip tags. Note that we replace tags with spaces
+      s = s.gsub(/<[^>]+>/, ' ')
+
+      if s !~ ASCII_ONLY
+        # Converts MS Word 'smart punctuation' to ASCII
+        s = Sterile.plain_format(s)
+
+        # &aacute; &amp; etc.
+        s = Sterile.decode_entities(s)
+
+        # "šţɽĩɳģ" => "string"
+        s = Sterile.transliterate(s)
+      end
+
+      # squish
+      s = s.strip.gsub(/\s+/, ' ')
+      return if s.empty?
+
+      s
+    end
+  end
+end

data/lib/sinew/main.rb

@@ -1,98 +1,72 @@
-require 'scripto'
-require 'sinew/connection'
-
-#
-# Main sinew entry point.
-#
-
 module Sinew
-  class Main < Scripto::Main
-    attr_reader :runtime_options
+  # Helper class used by sinew bin. This exists as an independent class solely
+  # for testing, otherwise it would be built into the bin script.
+  class Main
+    attr_reader :sinew
 
     def initialize(options)
-      super(options)
+      options[:output] ||= begin
+        src = options[:recipe]
+        dst = File.join(File.dirname(src), "#{File.basename(src, File.extname(src))}.csv")
+        dst = dst.sub(%r{^./}, '') # nice to clean this up
+        dst
+      end
 
-      # init
-      @runtime_options = RuntimeOptions.new
+      @sinew = Sinew::Base.new(options)
     end
 
     def run
-      dsl.run
-      footer if !quiet?
-    end
-
-    def quiet?
-      options[:quiet]
-    end
-
-    def dsl
-      @dsl ||= DSL.new(self)
-    end
-
-    #
-    # http requests
-    #
-
-    def http(method, url, options = {})
-      request = Request.new(method, url, request_options(options))
-      response = request.perform(connection)
-
-      # always log error messages
-      if response.error?
-        puts "xxx http request failed with #{response.code}"
+      tm = Time.now
+      header if !sinew.options[:silent]
+      recipe = sinew.options[:recipe]
+      dsl = DSL.new(sinew)
+      begin
+        dsl.instance_eval(File.read(recipe, mode: 'rb'), recipe)
+      rescue LimitError
+        # ignore - this is flow control for --limit
       end
-
-      response
+      footer(Time.now - tm) if !sinew.options[:silent]
     end
 
-    def connection
-      @connection ||= Connection.create(options: options, runtime_options: runtime_options)
-    end
-    protected :connection
+    protected
 
     #
-    # output
+    # header/footer
     #
 
-    def output
-      @output ||= Output.new(self)
+    def header
+      sinew.banner("Writing to #{sinew.csv.path}...")
     end
 
-    #
-    # helpers
-    #
+    def footer(elapsed)
+      csv = sinew.csv
+      count = csv.count
 
-    def request_options(options)
-      options.dup.tap do |req|
-        req[:headers] = {}.tap do |h|
-          [ runtime_options.headers, options[:headers]].each do
-            h.merge!(_1) if _1
-          end
-        end
-        req[:proxy] = random_proxy
+      if count == 0
+        sinew.banner(format('Done in %ds. Nothing written.', elapsed))
+        return
       end
-    end
-    protected :request_options
-
-    PROXY_RE = /\A#{URI::PATTERN::HOST}(:\d+)?\Z/.freeze
 
-    def random_proxy
-      return if !options[:proxy]
+      # summary
+      msg = format('Done in %ds. Wrote %d rows to %s. Summary:', elapsed, count, csv.path)
+      sinew.banner(msg)
 
-      proxy = options[:proxy].split(',').sample
-      if proxy !~ PROXY_RE
-        raise ArgumentError, "invalid proxy #{proxy.inspect}, should be host[:port]"
+      # tally
+      tally = csv.tally.sort_by { [-_2, _1.to_s] }.to_h
+      len = tally.keys.map { _1.to_s.length }.max
+      fmt = "  %-#{len + 1}s %7d/%-7d %5.1f%%\n"
+      tally.each do
+        printf(fmt, _1, _2, count, _2 * 100.0 / count)
       end
-
-      "http://#{proxy}"
     end
-    protected :random_proxy
 
-    def footer
-      output.report
-      finished = output.count > 0 ? "Finished #{output.filename}" : 'Finished'
-      banner("#{finished} in #{dsl.elapsed.to_i}s.")
+    # simple DSL for .sinew files
+    class DSL
+      attr_reader :sinew
+
+      def initialize(sinew)
+        @sinew = sinew
+      end
     end
-    protected :footer
   end
 end

data/lib/sinew/{connection → middleware}/log_formatter.rb

@@ -1,5 +1,6 @@
 module Sinew
-  module Connection
+  module Middleware
+    # Minimalist Formatter that logs proxy if present.
     class LogFormatter < Faraday::Logging::Formatter
       def request(env)
         info('req') do

data/lib/sinew/nokogiri_ext.rb

@@ -1,28 +1,19 @@
 require 'nokogiri'
 
 # modify NodeSet to join with SPACE instead of empty string
-class Nokogiri::XML::NodeSet
-  alias old_inner_html inner_html
-  alias old_inner_text inner_text
+module Nokogiri
+  module XML
+    class NodeSet
+      alias old_inner_html inner_html
+      alias old_inner_text inner_text
 
-  def inner_text
-    map(&:inner_text).join(' ')
-  end
-
-  def inner_html(*args)
-    map { |i| i.inner_html(*args) }.join(' ')
-  end
-end
+      def inner_text
+        map(&:inner_text).join(' ')
+      end
 
-# text_just_me
-class Nokogiri::XML::Node
-  def text_just_me
-    t = children.find { |i| i.node_type == Nokogiri::XML::Node::TEXT_NODE }
-    t&.text
-  end
-end
-class Nokogiri::XML::NodeSet
-  def text_just_me
-    map(&:text_just_me).join(' ')
+      def inner_html(*args)
+        map { _1.inner_html(*args) }.join(' ')
+      end
+    end
   end
 end

data/lib/sinew/response.rb

@@ -1,72 +1,61 @@
-require 'stringio'
-require 'zlib'
-
-#
-# An HTTP response.
-#
+require 'delegate'
+require 'hashie/mash'
+require 'json'
+require 'nokogiri'
 
 module Sinew
-  class Response
-    attr_accessor :request, :uri, :body, :code, :headers
+  # A wrapper around Faraday::Response, with some parsing helpers.
+  class Response < SimpleDelegator
+    # Like body, but tries to cleanup whitespace around HTML for easier parsing.
+    def html
+      @html ||= body.dup.tap do
+        # fix invalid utf8
+        if _1.encoding == Encoding::UTF_8
+          _1.encode!('UTF-8', invalid: :replace, undef: :replace, replace: '?')
+        end
 
-    #
-    # factory methods
-    #
+        # squish
+        _1.strip!
+        _1.gsub!(/\s+/, ' ')
 
-    def self.from_network(request, fday_response)
-      Response.new.tap do
-        _1.request = request
-        _1.uri = fday_response.env.url
-        _1.code = fday_response.status
-        _1.headers = fday_response.headers.to_h
-        _1.body = process_body(fday_response)
+        # kill whitespace around tags
+        _1.gsub!(/ ?<([^>]+)> ?/, '<\\1>')
       end
     end
 
-    # helper for decoding bodies before parsing
-    def self.process_body(response)
-      body = response.body
-
-      # inflate if necessary
-      bits = body[0, 10].force_encoding('BINARY')
-      if bits =~ /\A\x1f\x8b/n
-        body = Zlib::GzipReader.new(StringIO.new(body)).read
-      end
-
-      # force to utf-8 if we think this could be text
-      if body.encoding != Encoding::UTF_8
-        if content_type = response.headers['content-type']
-          if content_type =~ /\b(html|javascript|json|text|xml)\b/
-            body = body.encode('UTF-8', invalid: :replace, undef: :replace, replace: '?')
-          end
-        end
-      end
+    # Return body as JSON
+    def json
+      @json ||= JSON.parse(body, symbolize_names: true)
+    end
 
-      body
+    # Return JSON body as Hashie::Mash
+    def mash
+      @mash ||= Hashie::Mash.new(json)
     end
 
-    #
-    # accessors
-    #
+    # Return body HTML as Nokogiri document
+    def noko
+      @noko ||= Nokogiri::HTML(html)
+    end
 
-    def error?
-      code >= 400
+    # Return body XML as Nokogiri document
+    def xml
+      @xml ||= Nokogiri::XML(html)
     end
 
-    def error_500?
-      code / 100 >= 5
+    # Return the final URI for the request, after redirects
+    def url
+      env.url
     end
 
-    def redirected?
-      request.uri != uri
+    # Return the cache diskpath for this response
+    def diskpath
+      env[:httpdisk_diskpath]
     end
 
-    def head_as_json
-      {
-        uri: uri,
-        code: code,
-        headers: headers,
-      }
+    # Remove cached response from disk, if any
+    def uncache
+      File.unlink(diskpath) if File.exist?(diskpath)
     end
   end
 end

data/lib/sinew/version.rb

@@ -1,4 +1,4 @@
 module Sinew
   # Gem version
-  VERSION = '3.0.1'.freeze
+  VERSION = '4.0.0'.freeze
 end

data/sample.rb

@@ -0,0 +1,13 @@
+require_relative 'lib/sinew'
+
+sinew = Sinew.new(output: 'sample.csv', verbose: true)
+
+response = sinew.get 'http://httpbingo.org'
+response.noko.css('ul li a').each do |a|
+  row = {}
+  row[:url] = a[:href]
+  row[:title] = a.text
+  sinew.csv_emit(row)
+end
+
+sinew.get 'http://httpbingo.org/redirect/2'

data/sample.sinew

@@ -1,9 +1,9 @@
-get 'http://httpbingo.org'
-noko.css('ul li a').each do |a|
+response = sinew.get 'http://httpbingo.org'
+response.noko.css('ul li a').each do |a|
   row = {}
   row[:url] = a[:href]
   row[:title] = a.text
-  csv_emit(row)
+  sinew.csv_emit(row)
 end
 
-get 'http://httpbingo.org/redirect/2'
+sinew.get 'http://httpbingo.org/redirect/2'

data/sinew.gemspec

@@ -3,14 +3,15 @@ $LOAD_PATH.unshift("#{__dir__}/lib")
 require 'sinew/version'
 
 Gem::Specification.new do |s|
-  s.name        = 'sinew'
-  s.version     = Sinew::VERSION
-  s.license     = 'MIT'
-  s.authors     = [ 'Adam Doppelt', 'Nathan Kriege' ]
-  s.email       = [ 'amd@gurge.com' ]
-  s.homepage    = 'http://github.com/gurgeous/sinew'
-  s.summary     = 'Sinew - structured web crawling using recipes.'
+  s.name = 'sinew'
+  s.version = Sinew::VERSION
+  s.authors = ['Adam Doppelt', 'Nathan Kriege']
+  s.email = ['amd@gurge.com']
+
+  s.summary = 'Sinew - structured web crawling using recipes.'
   s.description = 'Crawl web sites easily using ruby recipes, with caching and nokogiri.'
+  s.homepage = 'http://github.com/gurgeous/sinew'
+  s.license = 'MIT'
   s.required_ruby_version = '>= 2.7'
 
   # what's in the gem?
@@ -19,14 +20,16 @@ Gem::Specification.new do |s|
   end
   s.bindir = 'bin'
   s.executables = s.files.grep(%r{^#{s.bindir}/}) { File.basename(_1) }
-  s.require_paths = [ 'lib' ]
+  s.require_paths = ['lib']
 
-  s.add_runtime_dependency 'amazing_print', '~> 1.3'
-  s.add_runtime_dependency 'faraday', '~> 1.4'
-  s.add_runtime_dependency 'faraday-encoding', '~> 0'
-  s.add_runtime_dependency 'httpdisk', '~> 0'
-  s.add_runtime_dependency 'nokogiri', '~> 1.11'
-  s.add_runtime_dependency 'scripto', '~> 0'
-  s.add_runtime_dependency 'slop', '~> 4.8'
-  s.add_runtime_dependency 'sterile', '~> 1.0'
+  # gem dependencies
+  s.add_dependency 'amazing_print', '~> 1.3'
+  s.add_dependency 'faraday', '~> 1.4'
+  s.add_dependency 'faraday-encoding', '~> 0'
+  s.add_dependency 'faraday-rate_limiter', '~> 0.0'
+  s.add_dependency 'hashie', '~> 4.1'
+  s.add_dependency 'httpdisk', '~> 0.5'
+  s.add_dependency 'nokogiri', '~> 1.11'
+  s.add_dependency 'slop', '~> 4.8'
+  s.add_dependency 'sterile', '~> 1.0'
 end