sinew 3.0.1 → 4.0.1

data/justfile

@@ -0,0 +1,59 @@
+
+# read gem version
+gemver := `cat lib/sinew/version.rb | grep -Eo "[0-9]+\.[0-9]+\.[0-9]+"`
+
+#
+# dev
+#
+
+default: test
+
+check: lint test
+
+fmt:
+  bundle exec rubocop -a
+
+lint:
+  @just banner lint...
+  bundle exec rubocop
+
+pry:
+  bundle exec pry -I lib -r sinew.rb
+
+test:
+  @just banner test...
+  bundle exec rake test
+
+watch:
+  @watchexec --watch lib --watch test --clear bundle exec rake test
+
+#
+# ci
+#
+
+ci:
+  bundle install
+  just check
+
+#
+# gem tasks
+#
+
+gem-push: check-git-status
+  @just banner gem build...
+  gem build sinew.gemspec
+  @just banner tag...
+  git tag -a "v{{gemver}}" -m "Tagging {{gemver}}"
+  git push --tags
+  @just banner gem push...
+  gem push "sinew-{{gemver}}.gem"
+
+#
+# util
+#
+
+banner *ARGS:
+  @printf '\e[42;37;1m[%s] %-72s \e[m\n' "$(date +%H:%M:%S)" "{{ARGS}}"
+
+check-git-status:
+  @if [ ! -z "$(git status --porcelain)" ]; then echo "git status is dirty, bailing."; exit 1; fi

data/lib/sinew/args.rb

@@ -0,0 +1,53 @@
+# manually load dependencies here since this is loaded standalone by bin
+require "httpdisk/slop_duration"
+require "sinew/version"
+require "slop"
+
+#
+# This is used to parse command line arguments with Slop. We don't set any
+# defaults in here, relying instead on Sloptions in Sinew::Base. That way
+# defaults are applied for both command line and embedded usage of Sinew::Base.
+#
+
+module Sinew
+  module Args
+    def self.slop(args)
+      slop = Slop.parse(args) do |o|
+        o.banner = "Usage: sinew [options] [recipe.sinew]"
+        o.integer "-l", "--limit", "quit after emitting this many rows"
+        o.string "--proxy", "use host[:port] as HTTP proxy (can be a comma-delimited list)"
+        o.integer "--timeout", "maximum time allowed for the transfer"
+        o.bool "-s", "--silent", "suppress some output"
+        o.bool "-v", "--verbose", "dump emitted rows while running"
+
+        o.separator "From httpdisk:"
+        o.string "--dir", "set custom cache directory"
+        # note: uses slop_duration from HTTPDisk
+        o.duration "--expires", "when to expire cached requests (ex: 1h, 2d, 3w)"
+        o.bool "--force", "don't read anything from cache (but still write)"
+        o.bool "--force-errors", "don't read errors from cache (but still write)"
+
+        # generic
+        o.boolean "--version", "show version" do
+          puts "sinew #{Sinew::VERSION}"
+          exit
+        end
+        o.on("--help", "show this help") do
+          puts o
+          exit
+        end
+      end
+
+      # recipe argument
+      recipe = slop.args.first
+      raise Slop::Error, "" if args.empty?
+      raise Slop::Error, "no RECIPE specified" if !recipe
+      raise Slop::Error, "more than one RECIPE specified" if slop.args.length > 1
+      raise Slop::Error, "#{recipe} not found" if !File.exist?(recipe)
+
+      slop.to_h.tap do
+        _1[:recipe] = recipe
+      end
+    end
+  end
+end

data/lib/sinew/base.rb

@@ -0,0 +1,252 @@
+require "amazing_print"
+require "faraday-encoding"
+require "faraday-rate_limiter"
+require "faraday/logging/formatter"
+require "faraday/retry"
+require "httpdisk"
+
+module Sinew
+  # Sinew base class, for in standalone scripts or via the sinew binary.
+  class Base
+    attr_reader :csv, :mutex, :options
+
+    def initialize(opts = {})
+      @mutex = Mutex.new
+
+      #
+      # defaults for Sloptions
+      #
+
+      # default :rate_limit, typically 1
+      default_rate_limit = ENV["SINEW_TEST"] ? 0 : 1
+
+      #
+      # note: uses HTTPDisk::Sloptions
+      #
+
+      @options = HTTPDisk::Sloptions.parse(opts) do
+        # cli
+        _1.integer :limit
+        _1.integer :timeout, default: 30
+        _1.boolean :silent
+        _1.on :proxy, type: [:string, Array]
+        _1.boolean :verbose
+
+        # httpdisk
+        _1.string :dir, default: File.join(ENV["HOME"], ".sinew")
+        _1.integer :expires
+        _1.boolean :force
+        _1.boolean :force_errors
+        _1.array :ignore_params
+
+        # more runtime options
+        _1.hash :headers
+        _1.boolean :insecure
+        _1.string :output, required: true
+        _1.hash :params
+        _1.float :rate_limit, default: default_rate_limit
+        _1.integer :retries, default: 2
+        _1.on :url_prefix, type: [:string, URI]
+        _1.boolean :utf8, default: true
+      end
+
+      @csv = CSV.new(opts[:output])
+    end
+
+    #
+    # requests
+    #
+
+    # http get, returns a Response
+    def get(url, params = nil, headers = nil)
+      faraday_response = faraday.get(url, params, headers) do
+        _1.options[:proxy] = random_proxy
+      end
+      Response.new(faraday_response)
+    end
+
+    # http post, returns a Response. Defaults to form body type.
+    def post(url, body = nil, headers = nil)
+      faraday_response = faraday.post(url, body, headers) do
+        _1.options[:proxy] = random_proxy
+      end
+      Response.new(faraday_response)
+    end
+
+    # http post json, returns a Response
+    def post_json(url, body = nil, headers = nil)
+      body = body.to_json
+      headers = (headers || {}).merge("Content-Type" => "application/json")
+      post(url, body, headers)
+    end
+
+    # Faraday connection for this recipe
+    def faraday
+      mutex.synchronize do
+        @faraday ||= create_faraday
+      end
+    end
+
+    #
+    # httpdisk
+    #
+
+    # Returns true if request is cached. Defaults to form body type.
+    def cached?(method, url, params = nil, body = nil)
+      status = status(method, url, params, body)
+      status[:status] != "miss"
+    end
+
+    # Remove cache file, if any. Defaults to form body type.
+    def uncache(method, url, params = nil, body = nil)
+      status = status(method, url, params, body)
+      path = status[:path]
+      File.unlink(path) if File.exist?(path)
+    end
+
+    # Check httpdisk status for this request. Defaults to form body type.
+    def status(method, url, params = nil, body = nil)
+      # if hash, default to url encoded form
+      # see lib/faraday/request/url_encoded.rb
+      if body.is_a?(Hash)
+        body = Faraday::Utils::ParamsHash[body].to_query
+      end
+
+      env = Faraday::Env.new.tap do
+        _1.method = method.to_s.downcase.to_sym
+        _1.request_headers = {}
+        _1.request_body = body
+        _1.url = faraday.build_url(url, params)
+      end
+      httpdisk.status(env)
+    end
+
+    #
+    # csv
+    #
+
+    # Output a csv header. This usually happens automatically, but you can call
+    # this method directly to ensure a consistent set of columns.
+    def csv_header(*columns)
+      csv.start(columns.flatten)
+    end
+
+    # Output a csv row. Row should be any object that can turn into a hash - a
+    # hash, OpenStruct, etc.
+    def csv_emit(row)
+      row = row.to_h
+      mutex.synchronize do
+        # header if necessary
+        csv_header(row.keys) if !csv.started?
+
+        # emit
+        print = csv.emit(row)
+        puts print.ai if options[:verbose]
+
+        # this is caught by Sinew::Main
+        if csv.count == options[:limit]
+          raise LimitError
+        end
+      end
+    end
+
+    #
+    # stdout
+    #
+
+    RESET = "\e[0m".freeze
+    RED = "\e[1;37;41m".freeze
+    GREEN = "\e[1;37;42m".freeze
+
+    # Print a nice green banner.
+    def banner(msg, color: GREEN)
+      msg = "#{msg} ".ljust(72, " ")
+      msg = "[#{Time.new.strftime("%H:%M:%S")}] #{msg}"
+      msg = "#{color}#{msg}#{RESET}" if $stdout.tty?
+      puts msg
+    end
+
+    # Print a scary red banner and exit.
+    def fatal(msg)
+      banner(msg, color: RED)
+      exit 1
+    end
+
+    protected
+
+    # Return a random proxy.
+    def random_proxy
+      return if !options[:proxy]
+
+      proxies = options[:proxy]
+      proxies = proxies.split(",") if !proxies.is_a?(Array)
+      proxies.sample
+    end
+
+    # Create the Faraday connection for making requests.
+    def create_faraday
+      faraday_options = options.slice(:headers, :params)
+      if options[:insecure]
+        faraday_options[:ssl] = {verify: false}
+      end
+      Faraday.new(nil, faraday_options) do
+        # options
+        if options[:url_prefix]
+          _1.url_prefix = options[:url_prefix]
+        end
+        _1.options.timeout = options[:timeout]
+
+        #
+        # middleware that runs on both disk/network requests
+        #
+
+        # cookie middleware
+        _1.use :cookie_jar
+
+        # auto-encode form bodies
+        _1.request :url_encoded
+
+        # Before httpdisk so each redirect segment is cached
+        # Keep track of redirect status for logger
+        _1.response :follow_redirects, callback: ->(_old_env, new_env) { new_env[:redirect] = true }
+
+        #
+        # httpdisk
+        #
+
+        httpdisk_options = options.slice(:dir, :expires, :force, :force_errors, :ignore_params, :utf8)
+        _1.use :httpdisk, httpdisk_options
+
+        #
+        # middleware below only used it httpdisk uses the network
+        #
+
+        # rate limit
+        rate_limit = options[:rate_limit]
+        _1.request :rate_limiter, interval: rate_limit
+
+        # After httpdisk so that only non-cached requests are logged.
+        # Before retry so that we don't log each retry attempt.
+        _1.response :logger, nil, formatter: Middleware::LogFormatter if !options[:silent]
+
+        retry_options = {
+          max_interval: rate_limit, # very important, negates Retry-After: 86400
+          max: options[:retries],
+          methods: %w[delete get head options patch post put trace],
+          retry_statuses: (500..600).to_a,
+          retry_if: ->(_env, _err) { true }
+        }
+        _1.request :retry, retry_options
+      end
+    end
+
+    # find connection's httpdisk instance
+    def httpdisk
+      @httpdisk ||= begin
+        app = faraday.app
+        app = app.app until app.is_a?(HTTPDisk::Client)
+        app
+      end
+    end
+  end
+end

data/lib/sinew/csv.rb

@@ -0,0 +1,89 @@
+require "csv"
+require "sterile"
+
+module Sinew
+  class CSV
+    attr_reader :columns, :count, :csv, :path, :tally
+
+    def initialize(path)
+      @count = 0
+      @csv = nil
+      @path = path
+    end
+
+    # start writing the csv
+    def start(columns)
+      raise "started twice" if started?
+
+      @columns = columns
+      @tally = columns.map { [_1, 0] }.to_h
+      @csv = ::CSV.open(path, "wb").tap do
+        _1 << columns
+      end
+    end
+
+    # has this csv been started?
+    def started?
+      @csv != nil
+    end
+
+    # append a row
+    def emit(row)
+      # convert row to cols, and construct print (our return value)
+      print = {}
+      row = columns.map do
+        value = normalize(row[_1])
+        if value
+          print[_1] = value
+          tally[_1] += 1
+        end
+        value
+      end
+      @count += 1
+
+      # emit
+      csv << row
+      csv.flush
+
+      # return in case someone wants to pretty print this
+      print
+    end
+
+    ASCII_ONLY = begin
+      chars = (33..126).map(&:chr) - ["&"]
+      /\A[#{Regexp.escape(chars.join)}\s]+\Z/
+    end.freeze
+
+    def normalize(s)
+      # nokogiri/array/misc => string
+      s = if s.respond_to?(:inner_html)
+        s.inner_html
+      elsif s.is_a?(Array)
+        s.join("|")
+      else
+        s.to_s
+      end
+      return if s.empty?
+
+      # simple attempt to strip tags. Note that we replace tags with spaces
+      s = s.gsub(/<[^>]+>/, " ")
+
+      if s !~ ASCII_ONLY
+        # Converts MS Word 'smart punctuation' to ASCII
+        s = Sterile.plain_format(s)
+
+        # &aacute; &amp; etc.
+        s = Sterile.decode_entities(s)
+
+        # "šţɽĩɳģ" => "string"
+        s = Sterile.transliterate(s)
+      end
+
+      # squish
+      s = s.strip.gsub(/\s+/, " ")
+      return if s.empty?
+
+      s
+    end
+  end
+end

data/lib/sinew/main.rb

@@ -1,98 +1,72 @@
-require 'scripto'
-require 'sinew/connection'
-
-#
-# Main sinew entry point.
-#
-
 module Sinew
-  class Main < Scripto::Main
-    attr_reader :runtime_options
+  # Helper class used by sinew bin. This exists as an independent class solely
+  # for testing, otherwise it would be built into the bin script.
+  class Main
+    attr_reader :sinew
 
     def initialize(options)
-      super(options)
+      options[:output] ||= begin
+        src = options[:recipe]
+        dst = File.join(File.dirname(src), "#{File.basename(src, File.extname(src))}.csv")
+        dst = dst.sub(%r{^./}, "") # nice to clean this up
+        dst
+      end
 
-      # init
-      @runtime_options = RuntimeOptions.new
+      @sinew = Sinew::Base.new(options)
     end
 
     def run
-      dsl.run
-      footer if !quiet?
-    end
-
-    def quiet?
-      options[:quiet]
-    end
-
-    def dsl
-      @dsl ||= DSL.new(self)
-    end
-
-    #
-    # http requests
-    #
-
-    def http(method, url, options = {})
-      request = Request.new(method, url, request_options(options))
-      response = request.perform(connection)
-
-      # always log error messages
-      if response.error?
-        puts "xxx http request failed with #{response.code}"
+      tm = Time.now
+      header if !sinew.options[:silent]
+      recipe = sinew.options[:recipe]
+      dsl = DSL.new(sinew)
+      begin
+        dsl.instance_eval(File.read(recipe, mode: "rb"), recipe)
+      rescue LimitError
+        # ignore - this is flow control for --limit
       end
-
-      response
+      footer(Time.now - tm) if !sinew.options[:silent]
     end
 
-    def connection
-      @connection ||= Connection.create(options: options, runtime_options: runtime_options)
-    end
-    protected :connection
+    protected
 
     #
-    # output
+    # header/footer
     #
 
-    def output
-      @output ||= Output.new(self)
+    def header
+      sinew.banner("Writing to #{sinew.csv.path}...")
     end
 
-    #
-    # helpers
-    #
+    def footer(elapsed)
+      csv = sinew.csv
+      count = csv.count
 
-    def request_options(options)
-      options.dup.tap do |req|
-        req[:headers] = {}.tap do |h|
-          [ runtime_options.headers, options[:headers]].each do
-            h.merge!(_1) if _1
-          end
-        end
-        req[:proxy] = random_proxy
+      if count == 0
+        sinew.banner(format("Done in %ds. Nothing written.", elapsed))
+        return
       end
-    end
-    protected :request_options
-
-    PROXY_RE = /\A#{URI::PATTERN::HOST}(:\d+)?\Z/.freeze
 
-    def random_proxy
-      return if !options[:proxy]
+      # summary
+      msg = format("Done in %ds. Wrote %d rows to %s. Summary:", elapsed, count, csv.path)
+      sinew.banner(msg)
 
-      proxy = options[:proxy].split(',').sample
-      if proxy !~ PROXY_RE
-        raise ArgumentError, "invalid proxy #{proxy.inspect}, should be host[:port]"
+      # tally
+      tally = csv.tally.sort_by { [-_2, _1.to_s] }.to_h
+      len = tally.keys.map { _1.to_s.length }.max
+      fmt = "  %-#{len + 1}s %7d/%-7d %5.1f%%\n"
+      tally.each do
+        printf(fmt, _1, _2, count, _2 * 100.0 / count)
       end
-
-      "http://#{proxy}"
     end
-    protected :random_proxy
 
-    def footer
-      output.report
-      finished = output.count > 0 ? "Finished #{output.filename}" : 'Finished'
-      banner("#{finished} in #{dsl.elapsed.to_i}s.")
+    # simple DSL for .sinew files
+    class DSL
+      attr_reader :sinew
+
+      def initialize(sinew)
+        @sinew = sinew
+      end
     end
-    protected :footer
   end
 end

data/lib/sinew/{connection → middleware}/log_formatter.rb

@@ -1,8 +1,9 @@
 module Sinew
-  module Connection
+  module Middleware
+    # Minimalist Formatter that logs proxy if present.
     class LogFormatter < Faraday::Logging::Formatter
       def request(env)
-        info('req') do
+        info("req") do
           # Only log the initial request, not the redirects
           return if env[:redirect]
 

data/lib/sinew/nokogiri_ext.rb

@@ -1,28 +1,19 @@
-require 'nokogiri'
+require "nokogiri"
 
 # modify NodeSet to join with SPACE instead of empty string
-class Nokogiri::XML::NodeSet
-  alias old_inner_html inner_html
-  alias old_inner_text inner_text
+module Nokogiri
+  module XML
+    class NodeSet
+      alias_method :old_inner_html, :inner_html
+      alias_method :old_inner_text, :inner_text
 
-  def inner_text
-    map(&:inner_text).join(' ')
-  end
-
-  def inner_html(*args)
-    map { |i| i.inner_html(*args) }.join(' ')
-  end
-end
+      def inner_text
+        map(&:inner_text).join(" ")
+      end
 
-# text_just_me
-class Nokogiri::XML::Node
-  def text_just_me
-    t = children.find { |i| i.node_type == Nokogiri::XML::Node::TEXT_NODE }
-    t&.text
-  end
-end
-class Nokogiri::XML::NodeSet
-  def text_just_me
-    map(&:text_just_me).join(' ')
+      def inner_html(*args)
+        map { _1.inner_html(*args) }.join(" ")
+      end
+    end
   end
 end