sinew 2.0.3 → 4.0.0

data/Rakefile

@@ -1,38 +1,54 @@
-require 'bundler'
 require 'bundler/setup'
-
-require 'rake'
 require 'rake/testtask'
-require 'sinew/version'
+
+# load the spec, we use it below
+spec = Gem::Specification.load('sinew.gemspec')
 
 #
-# gem
+# testing
+# don't forget about TESTOPTS="--verbose" rake
+# also: rake install && rm -rf ~/.sinew/www.amazon.com && /usr/local/bin/sinew sample.sinew
 #
 
-task gem: :build
-task :build do
-  system 'gem build --quiet sinew.gemspec'
+# test (default)
+Rake::TestTask.new
+task default: :test
+
+# Watch rb files, run tests whenever something changes
+task :watch do
+  sh "find . -name '*.rb' -o -name '*.sinew' | entr -c rake"
 end
 
-task install: :build do
-  system "sudo gem install --quiet sinew-#{Sinew::VERSION}.gem"
+#
+# pry
+#
+
+task :pry do
+  sh 'pry -I lib -r sinew.rb'
 end
 
-task release: :build do
-  system "git tag -a #{Sinew::VERSION} -m 'Tagging #{Sinew::VERSION}'"
-  system 'git push --tags'
-  system "gem push sinew-#{Sinew::VERSION}.gem"
+#
+# rubocop
+#
+
+task :rubocop do
+  sh 'bundle exec rubocop -A .'
 end
 
 #
-# minitest
+# gem
 #
 
-Rake::TestTask.new(:test) do |t|
-  t.warning = false
+task :build do
+  sh 'gem build --quiet sinew.gemspec'
 end
 
-task default: :test
+task install: :build do
+  sh "gem install --quiet sinew-#{spec.version}.gem"
+end
 
-# to test:
-# block ; rake install && rm -rf ~/.sinew/www.amazon.com && /usr/local/bin/sinew sample.sinew
+task release: %i[rubocop test build] do
+  sh "git tag -a #{spec.version} -m 'Tagging #{spec.version}'"
+  sh 'git push --tags'
+  sh "gem push sinew-#{spec.version}.gem"
+end

data/bin/sinew

@@ -1,51 +1,25 @@
 #!/usr/bin/env ruby
 
-$LOAD_PATH.unshift("#{__dir__}/../lib")
+$LOAD_PATH.unshift(File.join(__dir__, '../lib'))
 
-require 'sinew'
-require 'slop'
+BIN = File.basename($PROGRAM_NAME)
 
 #
-# options
+# Load the bare minimum and parse args with slop. For speed.
 #
 
-options = Slop.parse do |o|
-  o.banner = 'Usage: sinew [options] <gub.sinew>'
-  o.bool '-v', '--verbose', 'dump emitted rows while running'
-  o.bool '-q', '--quiet', 'suppress some output'
-  o.integer '-l', '--limit', 'quit after emitting this many rows'
-  o.string '-c', '--cache', 'set custom cache directory', default: "#{ENV['HOME']}/.sinew"
-  o.string '--proxy', 'use host[:port] as HTTP proxy'
-  o.bool '--version', 'show version and exit'
-  o.on('--help', 'show this help') do
-    puts o
-    exit
-  end
-end
-
-if options[:version]
-  puts Sinew::VERSION
-  exit
+require 'sinew/args'
+begin
+  slop = Sinew::Args.slop(ARGV)
+rescue Slop::Error => e
+  $stderr.puts "#{BIN}: #{e}" if e.message != ''
+  $stderr.puts("#{BIN}: try '#{BIN} --help' for more information")
+  exit 1
 end
 
 #
-# recipe
+# now load everything and run
 #
 
-recipe = options.arguments.first
-if !recipe
-  Scripto.fatal('need a .sinew file to run against')
-end
-if !File.exist?(recipe)
-  Scripto.fatal("#{recipe} not found")
-end
-if options.arguments.length > 1
-  Scripto.fatal('can only run on one .sinew file')
-end
-options = options.to_h.merge(recipe: recipe)
-
-#
-# main
-#
-
-Sinew::Main.new(options).run
+require 'sinew'
+Sinew::Main.new(slop).run

data/lib/sinew.rb

@@ -1,10 +1,23 @@
-require_relative 'sinew/cache'
-require_relative 'sinew/core_ext'
-require_relative 'sinew/dsl'
-require_relative 'sinew/main'
-require_relative 'sinew/nokogiri_ext'
-require_relative 'sinew/output'
-require_relative 'sinew/request'
-require_relative 'sinew/response'
-require_relative 'sinew/runtime_options'
-require_relative 'sinew/version'
+# sinew
+require 'sinew/args'
+require 'sinew/base'
+require 'sinew/csv'
+require 'sinew/main'
+require 'sinew/nokogiri_ext'
+require 'sinew/response'
+require 'sinew/version'
+
+# custom faraday middleware
+require 'sinew/middleware/log_formatter'
+
+module Sinew
+  # flow control for --limit
+  class LimitError < StandardError; end
+
+  # shortcut for Sinew::Base.new
+  class << self
+    def new(**args)
+      Sinew::Base.new(**args)
+    end
+  end
+end

data/lib/sinew/args.rb

@@ -0,0 +1,53 @@
+# manually load dependencies here since this is loaded standalone by bin
+require 'httpdisk/slop_duration'
+require 'sinew/version'
+require 'slop'
+
+#
+# This is used to parse command line arguments with Slop. We don't set any
+# defaults in here, relying instead on Sloptions in Sinew::Base. That way
+# defaults are applied for both command line and embedded usage of Sinew::Base.
+#
+
+module Sinew
+  module Args
+    def self.slop(args)
+      slop = Slop.parse(args) do |o|
+        o.banner = 'Usage: sinew [options] [recipe.sinew]'
+        o.integer '-l', '--limit', 'quit after emitting this many rows'
+        o.string '--proxy', 'use host[:port] as HTTP proxy (can be a comma-delimited list)'
+        o.integer '--timeout', 'maximum time allowed for the transfer'
+        o.bool '-s', '--silent', 'suppress some output'
+        o.bool '-v', '--verbose', 'dump emitted rows while running'
+
+        o.separator 'From httpdisk:'
+        o.string '--dir', 'set custom cache directory'
+        # note: uses slop_duration from HTTPDisk
+        o.duration '--expires', 'when to expire cached requests (ex: 1h, 2d, 3w)'
+        o.bool '--force', "don't read anything from cache (but still write)"
+        o.bool '--force-errors', "don't read errors from cache (but still write)"
+
+        # generic
+        o.boolean '--version', 'show version' do
+          puts "sinew #{Sinew::VERSION}"
+          exit
+        end
+        o.on('--help', 'show this help') do
+          puts o
+          exit
+        end
+      end
+
+      # recipe argument
+      recipe = slop.args.first
+      raise Slop::Error, '' if args.empty?
+      raise Slop::Error, 'no RECIPE specified' if !recipe
+      raise Slop::Error, 'more than one RECIPE specified' if slop.args.length > 1
+      raise Slop::Error, "#{recipe} not found" if !File.exist?(recipe)
+
+      slop.to_h.tap do
+        _1[:recipe] = recipe
+      end
+    end
+  end
+end

data/lib/sinew/base.rb

@@ -0,0 +1,251 @@
+require 'amazing_print'
+require 'faraday-encoding'
+require 'faraday/logging/formatter'
+require 'faraday-rate_limiter'
+require 'httpdisk'
+
+module Sinew
+  # Sinew base class, for in standalone scripts or via the sinew binary.
+  class Base
+    attr_reader :csv, :mutex, :options
+
+    def initialize(opts = {})
+      @mutex = Mutex.new
+
+      #
+      # defaults for Sloptions
+      #
+
+      # default :rate_limit, typically 1
+      default_rate_limit = ENV['SINEW_TEST'] ? 0 : 1
+
+      #
+      # note: uses HTTPDisk::Sloptions
+      #
+
+      @options = HTTPDisk::Sloptions.parse(opts) do
+        # cli
+        _1.integer :limit
+        _1.integer :timeout, default: 30
+        _1.boolean :silent
+        _1.on :proxy, type: [:string, Array]
+        _1.boolean :verbose
+
+        # httpdisk
+        _1.string :dir, default: File.join(ENV['HOME'], '.sinew')
+        _1.integer :expires
+        _1.boolean :force
+        _1.boolean :force_errors
+        _1.array :ignore_params
+
+        # more runtime options
+        _1.hash :headers
+        _1.boolean :insecure
+        _1.string :output, required: true
+        _1.hash :params
+        _1.float :rate_limit, default: default_rate_limit
+        _1.integer :retries, default: 2
+        _1.on :url_prefix, type: [:string, URI]
+        _1.boolean :utf8, default: true
+      end
+
+      @csv = CSV.new(opts[:output])
+    end
+
+    #
+    # requests
+    #
+
+    # http get, returns a Response
+    def get(url, params = nil, headers = nil)
+      faraday_response = faraday.get(url, params, headers) do
+        _1.options[:proxy] = random_proxy
+      end
+      Response.new(faraday_response)
+    end
+
+    # http post, returns a Response. Defaults to form body type.
+    def post(url, body = nil, headers = nil)
+      faraday_response = faraday.post(url, body, headers) do
+        _1.options[:proxy] = random_proxy
+      end
+      Response.new(faraday_response)
+    end
+
+    # http post json, returns a Response
+    def post_json(url, body = nil, headers = nil)
+      body = body.to_json
+      headers = (headers || {}).merge('Content-Type' => 'application/json')
+      post(url, body, headers)
+    end
+
+    # Faraday connection for this recipe
+    def faraday
+      mutex.synchronize do
+        @faraday ||= create_faraday
+      end
+    end
+
+    #
+    # httpdisk
+    #
+
+    # Returns true if request is cached. Defaults to form body type.
+    def cached?(method, url, params = nil, body = nil)
+      status = status(method, url, params, body)
+      status[:status] != 'miss'
+    end
+
+    # Remove cache file, if any. Defaults to form body type.
+    def uncache(method, url, params = nil, body = nil)
+      status = status(method, url, params, body)
+      path = status[:path]
+      File.unlink(path) if File.exist?(path)
+    end
+
+    # Check httpdisk status for this request. Defaults to form body type.
+    def status(method, url, params = nil, body = nil)
+      # if hash, default to url encoded form
+      # see lib/faraday/request/url_encoded.rb
+      if body.is_a?(Hash)
+        body = Faraday::Utils::ParamsHash[body].to_query
+      end
+
+      env = Faraday::Env.new.tap do
+        _1.method = method.to_s.downcase.to_sym
+        _1.request_headers = {}
+        _1.request_body = body
+        _1.url = faraday.build_url(url, params)
+      end
+      httpdisk.status(env)
+    end
+
+    #
+    # csv
+    #
+
+    # Output a csv header. This usually happens automatically, but you can call
+    # this method directly to ensure a consistent set of columns.
+    def csv_header(*columns)
+      csv.start(columns.flatten)
+    end
+
+    # Output a csv row. Row should be any object that can turn into a hash - a
+    # hash, OpenStruct, etc.
+    def csv_emit(row)
+      row = row.to_h
+      mutex.synchronize do
+        # header if necessary
+        csv_header(row.keys) if !csv.started?
+
+        # emit
+        print = csv.emit(row)
+        puts print.ai if options[:verbose]
+
+        # this is caught by Sinew::Main
+        if csv.count == options[:limit]
+          raise LimitError
+        end
+      end
+    end
+
+    #
+    # stdout
+    #
+
+    RESET = "\e[0m".freeze
+    RED = "\e[1;37;41m".freeze
+    GREEN = "\e[1;37;42m".freeze
+
+    # Print a nice green banner.
+    def banner(msg, color: GREEN)
+      msg = "#{msg} ".ljust(72, ' ')
+      msg = "[#{Time.new.strftime('%H:%M:%S')}] #{msg}"
+      msg = "#{color}#{msg}#{RESET}" if $stdout.tty?
+      puts msg
+    end
+
+    # Print a scary red banner and exit.
+    def fatal(msg)
+      banner(msg, color: RED)
+      exit 1
+    end
+
+    protected
+
+    # Return a random proxy.
+    def random_proxy
+      return if !options[:proxy]
+
+      proxies = options[:proxy]
+      proxies = proxies.split(',') if !proxies.is_a?(Array)
+      proxies.sample
+    end
+
+    # Create the Faraday connection for making requests.
+    def create_faraday
+      faraday_options = options.slice(:headers, :params)
+      if options[:insecure]
+        faraday_options[:ssl] = { verify: false }
+      end
+      Faraday.new(nil, faraday_options) do
+        # options
+        if options[:url_prefix]
+          _1.url_prefix = options[:url_prefix]
+        end
+        _1.options.timeout = options[:timeout]
+
+        #
+        # middleware that runs on both disk/network requests
+        #
+
+        # cookie middleware
+        _1.use :cookie_jar
+
+        # auto-encode form bodies
+        _1.request :url_encoded
+
+        # Before httpdisk so each redirect segment is cached
+        # Keep track of redirect status for logger
+        _1.response :follow_redirects, callback: ->(_old_env, new_env) { new_env[:redirect] = true }
+
+        #
+        # httpdisk
+        #
+
+        httpdisk_options = options.slice(:dir, :expires, :force, :force_errors, :ignore_params, :utf8)
+        _1.use :httpdisk, httpdisk_options
+
+        #
+        # middleware below only used it httpdisk uses the network
+        #
+
+        # rate limit
+        rate_limit = options[:rate_limit]
+        _1.request :rate_limiter, interval: rate_limit
+
+        # After httpdisk so that only non-cached requests are logged.
+        # Before retry so that we don't log each retry attempt.
+        _1.response :logger, nil, formatter: Middleware::LogFormatter if !options[:silent]
+
+        retry_options = {
+          max_interval: rate_limit, # very important, negates Retry-After: 86400
+          max: options[:retries],
+          methods: %w[delete get head options patch post put trace],
+          retry_statuses: (500..600).to_a,
+          retry_if: ->(_env, _err) { true },
+        }
+        _1.request :retry, retry_options
+      end
+    end
+
+    # find connection's httpdisk instance
+    def httpdisk
+      @httpdisk ||= begin
+        app = faraday.app
+        app = app.app until app.is_a?(HTTPDisk::Client)
+        app
+      end
+    end
+  end
+end