sinew 2.0.5 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b383fb9d0a1d57acfafd78d8e3ff0185b81acb5bcd368d4c0cca9a8999aa0a52
-  data.tar.gz: 0dede0f01c7a53056a38705c6fc134cd33a2f15868a9b4fee1b6f9fa85361d31
+  metadata.gz: df55f2168ff4242fceb31d083b8d16f1046139fa7acb8a9c4fc3f06f7884e113
+  data.tar.gz: 520967eba4ea2d8446690736f2c28d34642b452c0f4e5003dcb89ce373c116e5
 SHA512:
-  metadata.gz: e867516dd43bed9f6dd524475c70b933b30029788521224eed08d269ea1264f5d52a3740e967a2cc61ec96861e1875ecaf2f43af403bd917cac05ce2fd394119
-  data.tar.gz: 719ab64ac523e9cf553171bf318a59938655e4a40fd4c6544ec5c5188c7a81227a30b9ba7c70a6ace90a41b476a62081f2ac4cc88a490d0be2e05ade7e8b3dce
+  metadata.gz: 7443bccc5fc4e1bd112ce50b3d17445f0c21f5b351a6b5be586aadd63f36396312370ec6115d8116701165b3af19fcb852f85d18d0fbe7b4bf0d797312d3fa40
+  data.tar.gz: 9ca4f3c424e021100f518ca4f2231f515b38dbeb402ff4fce07c13a2440f19b0fabcc2a2920e51aa3f6228507550d3c94cb70a46fd22711ba3add90e4fc28004

data/.github/workflows/test.yml

@@ -0,0 +1,26 @@
+name: test
+
+on:
+  push:
+    paths-ignore:
+      - '**.md'
+  pull_request:
+    paths-ignore:
+      - '**.md'
+  workflow_dispatch:
+
+jobs:
+  test:
+    strategy:
+      max-parallel: 3
+      matrix:
+        os: [ubuntu, macos]
+        ruby-version: [3.0, 2.7]
+    runs-on: ${{ matrix.os }}-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+      - run: bundle install
+      - run: bundle exec rake test

data/.rubocop.yml

@@ -1,6 +1,6 @@
 AllCops:
-  Exclude:
-  TargetRubyVersion: 2.3
+  TargetRubyVersion: 2.7
+  NewCops: enable
 
 # amd: customizations
 Layout/SpaceInsideArrayLiteralBrackets:
@@ -22,18 +22,19 @@ Style/TrailingCommaInHashLiteral:
 
 # amd: these seem extreme
 Lint/AssignmentInCondition: { Enabled: false } # I do this all the time
-Lint/HandleExceptions: { Enabled: false } # blank rescues are useful
+Lint/SuppressedException: { Enabled: false } # blank rescues are useful
 Naming/BinaryOperatorParameterName: { Enabled: false } # silly
 Naming/HeredocDelimiterNaming: { Enabled: false } # silly
-Naming/UncommunicativeMethodParamName: { Enabled: false } # silly
-Performance/RegexpMatch: { Enabled: false } # =~ is fine
-Performance/TimesMap: { Enabled: false } # silly
+Naming/MethodParameterName: { Enabled: false } # silly
+Style/AccessorGrouping: { Enabled: false } # silly
+Style/AsciiComments: { Enabled: false } # silly
 Style/ClassAndModuleChildren: { Enabled: false } # silly
 Style/Documentation: { Enabled: false } # we don't need this
 Style/DoubleNegation: { Enabled: false } # silly
 Style/FormatStringToken: { Enabled: false } # we like printf here
 Style/FrozenStringLiteralComment: { Enabled: false } # seems excessive
 Style/GuardClause: { Enabled: false } # confusing
+Style/HashTransformValues: { Enabled: false } # breaks code by trying to apply to an array
 Style/IfUnlessModifier: { Enabled: false } # personally I hate unless
 Style/NegatedIf: { Enabled: false } # these are fine
 Style/Next: { Enabled: false } # these are fine
@@ -41,7 +42,9 @@ Style/NumericPredicate: { Enabled: false } # silly
 Style/ParallelAssignment: { Enabled: false } # these are fine
 Style/PerlBackrefs: { Enabled: false } # these are fine
 Style/RaiseArgs: { Enabled: false } # silly
+Style/RedundantAssignment: { Enabled: false } # these are usually on purpose
 Style/RegexpLiteral: { Enabled: false } # these are fine
+Style/SoleNestedConditional: { Enabled: false } # these are fine
 Style/StderrPuts: { Enabled: false } # this is awful
 
 # amd: these Metric rules are annoying, disable

data/.vscode/settings.json

@@ -1,15 +1,5 @@
 {
-  "editor.formatOnSave": true,
-  "editor.formatOnSaveTimeout": 1500,
-  "editor.tabSize": 2,
-  "editor.wordSeparators": "`~#$%^&*()-=+[{]}\\|;:'\",.<>/",
   "files.associations": {
     "*.sinew": "ruby"
-  },
-  "files.insertFinalNewline": true,
-  "files.trimTrailingWhitespace": true,
-  "ruby.format": "rubocop",
-  "ruby.lint": {
-    "rubocop": true
   }
 }

data/Gemfile

@@ -1,2 +1,11 @@
 source 'http://rubygems.org'
+
+group :development do
+  gem 'minitest'
+  gem 'mocha'
+  gem 'rake'
+  gem 'rubocop', '~> 0.91.0', require: false
+  gem 'webmock'
+end
+
 gemspec

data/README.md

@@ -1,4 +1,4 @@
-![Travis](https://travis-ci.org/gurgeous/sinew.svg?branch=master)
+[![Build Status](https://github.com/gurgeous/sinew/workflows/test/badge.svg?branch=master)](https://github.com/gurgeous/sinew/action)
 
 ## Welcome to Sinew
 
@@ -20,7 +20,7 @@ gem 'sinew'
 
 <!--- markdown-toc --no-firsth1 --maxdepth 1 readme.md -->
 
-- [Sinew 2](#sinew-2-may-2018)
+- [Sinew 3](#sinew-3-may-2021)
 - [Quick Example](#quick-example)
 - [How it Works](#how-it-works)
 - [DSL Reference](#dsl-reference)
@@ -29,27 +29,21 @@ gem 'sinew'
 - [Changelog](#changelog)
 - [License](#license)
 
-## Sinew 2 (May 2018)
+## Sinew 3 (May 2021)
 
-I am pleased to announce the release of Sinew 2.0, a complete rewrite of Sinew for the modern era. Enhancements include:
-
-- Remove dependencies on active_support, curl and tidy. We use HTTParty now.
-- Much easier to customize requests in `.sinew` files. For example, setting User-Agent or Bearer tokens.
-- More operations like `post_json` or the generic `http`. These methods are thin wrappers around HTTParty.
-- New end-of-run report.
-- Tests, rubocop, vscode settings, travis, etc.
+I am pleased to announce the release of Sinew 3.0. Sinew has been streamlined and updated to use the [Faraday](https://lostisland.github.io/faraday/) HTTP client with [sinew](https://github.com/gurgeous/sinew/) middleware for caching.
 
 **Breaking change**
 
-Sinew uses a new format for cached responses. Old Sinew 1 cache directories must be removed before running Sinew again. Sinew 2 might choke on Sinew 1 cache directores when reading `head/`. This is not tested or supported.
+Sinew 3 uses a new format for cached responses. Old Sinew 2 cache directories should be removed before running Sinew again.
 
 ## Quick Example
 
-Here's an example for collecting the links from httpbin.org:
+Here's an example for collecting the links from httpbingo.org:
 
 ```ruby
 # get the url
-get "http://httpbin.org"
+get "http://httpbingo.org"
 
 # use nokogiri to collect links
 noko.css("ul li a").each do |a|
@@ -114,9 +108,9 @@ Sinew creates a CSV file with the same name as the recipe, and `csv_emit(hash)`
 
 #### Caching
 
-Requests are made using HTTParty, and all responses are cached on disk in `~/.sinew`. Error responses are cached as well. Each URL will be hit exactly once, and requests are rate limited to one per second. Sinew tries to be polite.
+Sinew uses [sinew](https://github.com/gurgeous/sinew/) to aggressively cache all HTTP responses to disk in `~/.sinew`. Error responses are cached as well. Each URL will be hit exactly once, and requests are rate limited to one per second. Sinew tries to be polite.
 
-The files in `~/.sinew` have nice names and are designed to be human readable. This helps when writing recipes. Sinew never deletes files from the cache - that's up to you!
+Sinew never deletes files from the cache - that's up to you!
 
 Because all requests are cached, you can run Sinew repeatedly with confidence. Run it over and over again while you build up your recipe.
 
@@ -162,7 +156,7 @@ Writing Sinew recipes is fun and easy. The builtin caching means you can iterate
 noko.css("table")[4].css("td").select { |i| i[:width].to_i > 80 }.map(&:text)
 ```
 
-- Debug your recipes using plain old `puts`, or better yet use `ap` from [awesome_print](https://github.com/michaeldv/awesome_print).
+- Debug your recipes using plain old `puts`, or better yet use `ap` from [amazing_print](https://github.com/amazing-print/amazing_print).
 - Run `sinew -v` to get a report on every `csv_emit`. Very handy.
 - Add the CSV files to your git repo. That way you can version them and get diffs!
 
@@ -173,8 +167,10 @@ noko.css("table")[4].css("td").select { |i| i[:width].to_i > 80 }.map(&:text)
 
 ## Changelog
 
-#### 2.0.5 (unreleased)
+#### 3.0.0 (May 2021)
 
+- Major rewrite of network and caching layer. See above.
+- Use Faraday HTTP client with sinew middleware for caching.
 - Supports multiple proxies (`--proxy host1,host2,...`)
 
 #### 2.0.4 (May 2018)

data/Rakefile

@@ -1,38 +1,53 @@
-require 'bundler'
 require 'bundler/setup'
 
-require 'rake'
 require 'rake/testtask'
 require 'sinew/version'
 
+# load the spec, we use it below
+spec = Gem::Specification.load('sinew.gemspec')
+
 #
-# gem
+# testing
+# don't forget about TESTOPTS="--verbose" rake
+# also: rake install && rm -rf ~/.sinew/www.amazon.com && /usr/local/bin/sinew sample.sinew
 #
 
-task gem: :build
-task :build do
-  system 'gem build --quiet sinew.gemspec'
+# test (default)
+task default: :test
+
+Rake::TestTask.new do
+  _1.libs << 'test'
+  _1.warning = false # sterile has a few issues here
 end
 
-task install: :build do
-  system "sudo gem install --quiet sinew-#{Sinew::VERSION}.gem"
+# Watch rb files, run tests whenever something changes
+task :watch do
+  # https://superuser.com/a/665208 / https://unix.stackexchange.com/a/42288
+  system("while true; do find . -name '*.rb' | entr -c -d rake; test $? -gt 128 && break; done")
 end
 
-task release: :build do
-  system "git tag -a #{Sinew::VERSION} -m 'Tagging #{Sinew::VERSION}'"
-  system 'git push --tags'
-  system "gem push sinew-#{Sinew::VERSION}.gem"
+#
+# rubocop
+#
+
+task :rubocop do
+  system('bundle exec rubocop -A .', exception: true)
 end
 
 #
-# minitest
+# gem
 #
 
-Rake::TestTask.new(:test) do |t|
-  t.warning = false
+task :build do
+  system 'gem build --quiet sinew.gemspec', exception: true
 end
 
-task default: :test
+task install: :build do
+  system "gem install --quiet sinew-#{spec.version}.gem", exception: true
+end
 
-# to test:
-# block ; rake install && rm -rf ~/.sinew/www.amazon.com && /usr/local/bin/sinew sample.sinew
+task release: %i[rubocop test build] do
+  system "git tag -a #{spec.version} -m 'Tagging #{spec.version}'", exception: true
+  system 'git push --tags', exception: true
+  system "gem push sinew-#{spec.version}.gem", exception: true
+end

data/bin/sinew

@@ -15,6 +15,8 @@ options = Slop.parse do |o|
   o.bool '-q', '--quiet', 'suppress some output'
   o.integer '-l', '--limit', 'quit after emitting this many rows'
   o.string '-c', '--cache', 'set custom cache directory', default: "#{ENV['HOME']}/.sinew"
+  o.bool '--force', "don't read anything from cache (but still write)"
+  o.bool '--force-errors', "don't read errors from cache (but still write)"
   o.string '--proxy', 'use host[:port] as HTTP proxy'
   o.bool '--version', 'show version and exit'
   o.on('--help', 'show this help') do

data/lib/sinew.rb

@@ -1,4 +1,3 @@
-require_relative 'sinew/cache'
 require_relative 'sinew/core_ext'
 require_relative 'sinew/dsl'
 require_relative 'sinew/main'

data/lib/sinew/connection.rb

@@ -0,0 +1,52 @@
+require 'faraday'
+require 'faraday-encoding'
+require 'faraday/logging/formatter'
+require 'httpdisk'
+require 'sinew/connection/log_formatter'
+require 'sinew/connection/rate_limit'
+
+module Sinew
+  module Connection
+    def self.create(options:, runtime_options:)
+      connection_options = {}
+      connection_options[:ssl] = { verify: false } if runtime_options.insecure
+
+      Faraday.new(nil, connection_options) do
+        _1.use RateLimit, rate_limit: runtime_options.rate_limit
+
+        # auto-encode form bodies
+        _1.request :url_encoded
+
+        # Before httpdisk so each redirect segment is cached
+        # Keep track of redirect status for logger
+        _1.response :follow_redirects, callback: ->(_old_env, new_env) { new_env[:redirect] = true }
+
+        # set Ruby string encoding based on Content-Type (should be above httpdisk)
+        _1.response :encoding
+
+        # disk caching
+        httpdisk_options = {
+          dir: options[:cache],
+          force: options[:force],
+          force_errors: options[:force_errors],
+        }.merge(runtime_options.httpdisk_options)
+
+        _1.use :httpdisk, httpdisk_options
+
+        # After httpdisk so that only non-cached requests are logged.
+        # Before retry so that we don't log each retry attempt.
+        _1.response :logger, nil, formatter: LogFormatter if !options[:quiet]
+
+        # After httpdisk so transient failures are not cached
+        retry_options = {
+          interval: runtime_options.rate_limit,
+          max: runtime_options.retries,
+          methods: %w[delete get head options patch post put trace],
+          retry_statuses: (500..600).to_a,
+          retry_if: ->(_env, _err) { true },
+        }
+        _1.request :retry, retry_options
+      end
+    end
+  end
+end

data/lib/sinew/connection/log_formatter.rb

@@ -0,0 +1,22 @@
+module Sinew
+  module Connection
+    class LogFormatter < Faraday::Logging::Formatter
+      def request(env)
+        info('req') do
+          # Only log the initial request, not the redirects
+          return if env[:redirect]
+
+          msg = apply_filters(env.url.to_s)
+          msg = "#{msg} (#{env.method})" if env.method != :get
+          msg = "#{msg} => #{env.request.proxy.uri}" if env.request.proxy
+
+          msg
+        end
+      end
+
+      def response(env)
+        # silent
+      end
+    end
+  end
+end

data/lib/sinew/connection/rate_limit.rb

@@ -0,0 +1,29 @@
+module Sinew
+  module Connection
+    class RateLimit < Faraday::Middleware
+      attr_reader :rate_limit
+
+      def initialize(app, options = {})
+        super(app)
+
+        @last_request_tm = @current_request_tm = nil
+        @rate_limit = options.fetch(:rate_limit, 1)
+      end
+
+      def on_request(_env)
+        if @last_request_tm
+          sleep = (@last_request_tm + rate_limit) - Time.now
+          sleep(sleep) if sleep > 0
+        end
+
+        @current_request_tm = Time.now
+      end
+
+      def on_complete(env)
+        # Only rate limit on uncached requests
+        @last_request_tm = @current_request_tm unless env[:httpdisk]
+        @current_request_tm = nil
+      end
+    end
+  end
+end

data/lib/sinew/core_ext.rb

@@ -30,7 +30,7 @@ class String
     elsif limit >= size
       dup
     else
-      self[-limit..-1]
+      self[-limit..]
     end
   end
 

data/lib/sinew/dsl.rb

@@ -1,5 +1,6 @@
-require 'awesome_print'
+require 'amazing_print'
 require 'cgi'
+require 'json'
 
 #
 # The DSL available to .sinew files.

data/lib/sinew/main.rb

@@ -1,4 +1,5 @@
 require 'scripto'
+require 'sinew/connection'
 
 #
 # Main sinew entry point.
@@ -6,15 +7,13 @@ require 'scripto'
 
 module Sinew
   class Main < Scripto::Main
-    attr_reader :runtime_options, :request_tm, :request_count
+    attr_reader :runtime_options
 
     def initialize(options)
       super(options)
 
       # init
       @runtime_options = RuntimeOptions.new
-      @request_tm = Time.at(0)
-      @request_count = 0
     end
 
     def run
@@ -31,24 +30,12 @@ module Sinew
     end
 
     #
-    # http requests and caching
+    # http requests
     #
 
-    def cache
-      @cache ||= Cache.new(self)
-    end
-
     def http(method, url, options = {})
       request = Request.new(self, method, url, options)
-
-      # try to get from cache
-      response = cache.get(request)
-
-      # perform if necessary
-      if !response
-        response = perform(request)
-        cache.set(response)
-      end
+      response = request.perform(connection)
 
       # always log error messages
       if response.error?
@@ -58,26 +45,10 @@ module Sinew
       response
     end
 
-    def perform(request)
-      before_perform_request(request)
-
-      response = nil
-
-      tries = runtime_options.retries + 1
-      while tries > 0
-        tries -= 1
-        begin
-          @request_count += 1
-          response = request.perform
-        rescue HTTParty::RedirectionTooDeep, OpenSSL::SSL::SSLError, SocketError, SystemCallError, Timeout::Error => e
-          response = Response.from_error(request, e)
-        end
-        break if !response.error_500?
-      end
-
-      response
+    def connection
+      @connection ||= Connection.create(options: options, runtime_options: runtime_options)
     end
-    protected :perform
+    protected :connection
 
     #
     # output
@@ -91,25 +62,6 @@ module Sinew
     # helpers
     #
 
-    def before_perform_request(request)
-      # log
-      if !quiet?
-        msg = if request.method != 'get'
-          "req #{request.uri} (#{request.method})"
-        else
-          "req #{request.uri}"
-        end
-        msg = "#{msg} => #{request.proxy}" if request.proxy
-        $stderr.puts msg
-      end
-
-      # rate limit
-      sleep = (request_tm + runtime_options.rate_limit) - Time.now
-      sleep(sleep) if sleep > 0
-      @request_tm = Time.now
-    end
-    protected :before_perform_request
-
     def footer
       output.report
       finished = output.count > 0 ? "Finished #{output.filename}" : 'Finished'