sinew 2.0.2 → 3.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 33506a03f47a88cae5bf7e0f4675d7cf83d86ba3c96f0880f5c473a7b23b167b
-  data.tar.gz: 990bd4690f9fe799774c349314a32ab2c08979d555f03891316c5e0be8a4ad3d
+  metadata.gz: ba5558019816540d71e1bb44029f2733aff53649a3f4da72a9905e0a67d06ad9
+  data.tar.gz: 446c245782cad55f1caa36e01b0e8c98748295b47eb5b1911175fff2f79589b2
 SHA512:
-  metadata.gz: 9644097a2e11d8cba59a7985dfe770f27b00d5d18b676d0cacdee3e73a21f1b6c237b3bb58d68489d2a67fc981f7a7f8bb27a6e6fb23781f318cde78b392d7cd
-  data.tar.gz: 667c301e7896b27162a77cff5165f264a0c2b73afbe5c35f541181709118185a13241d187ff8f8d5964e302537064ff927d9fa64ece2cb10ca65ba7dd89ce807
+  metadata.gz: cef8c1145a21e84f560b44821071ffc7b57ef965167b633e1c837f7b7d9dbfce340b14d1afb2a35891c7b1ed4aa4f08e47ca7405cf382acca7eae855a47d3a71
+  data.tar.gz: 8dc7b67511fc541cccef23b69463abd9a8081f9c75583c1d6c6756dda0561ce7a91be35ee06f7d886ea7488b9005c969e14fa10794cac92475a3063c8968abec

data/.github/workflows/test.yml

@@ -0,0 +1,26 @@
+name: test
+
+on:
+  push:
+    paths-ignore:
+      - '**.md'
+  pull_request:
+    paths-ignore:
+      - '**.md'
+  workflow_dispatch:
+
+jobs:
+  test:
+    strategy:
+      max-parallel: 3
+      matrix:
+        os: [ubuntu, macos]
+        ruby-version: [3.0, 2.7]
+    runs-on: ${{ matrix.os }}-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+      - run: bundle install
+      - run: bundle exec rake test

data/.rubocop.yml

@@ -1,6 +1,6 @@
 AllCops:
-  Exclude:
-  TargetRubyVersion: 2.3
+  TargetRubyVersion: 2.7
+  NewCops: enable
 
 # amd: customizations
 Layout/SpaceInsideArrayLiteralBrackets:
@@ -22,18 +22,19 @@ Style/TrailingCommaInHashLiteral:
 
 # amd: these seem extreme
 Lint/AssignmentInCondition: { Enabled: false } # I do this all the time
-Lint/HandleExceptions: { Enabled: false } # blank rescues are useful
+Lint/SuppressedException: { Enabled: false } # blank rescues are useful
 Naming/BinaryOperatorParameterName: { Enabled: false } # silly
 Naming/HeredocDelimiterNaming: { Enabled: false } # silly
-Naming/UncommunicativeMethodParamName: { Enabled: false } # silly
-Performance/RegexpMatch: { Enabled: false } # =~ is fine
-Performance/TimesMap: { Enabled: false } # silly
+Naming/MethodParameterName: { Enabled: false } # silly
+Style/AccessorGrouping: { Enabled: false } # silly
+Style/AsciiComments: { Enabled: false } # silly
 Style/ClassAndModuleChildren: { Enabled: false } # silly
 Style/Documentation: { Enabled: false } # we don't need this
 Style/DoubleNegation: { Enabled: false } # silly
 Style/FormatStringToken: { Enabled: false } # we like printf here
 Style/FrozenStringLiteralComment: { Enabled: false } # seems excessive
 Style/GuardClause: { Enabled: false } # confusing
+Style/HashTransformValues: { Enabled: false } # breaks code by trying to apply to an array
 Style/IfUnlessModifier: { Enabled: false } # personally I hate unless
 Style/NegatedIf: { Enabled: false } # these are fine
 Style/Next: { Enabled: false } # these are fine
@@ -41,7 +42,9 @@ Style/NumericPredicate: { Enabled: false } # silly
 Style/ParallelAssignment: { Enabled: false } # these are fine
 Style/PerlBackrefs: { Enabled: false } # these are fine
 Style/RaiseArgs: { Enabled: false } # silly
+Style/RedundantAssignment: { Enabled: false } # these are usually on purpose
 Style/RegexpLiteral: { Enabled: false } # these are fine
+Style/SoleNestedConditional: { Enabled: false } # these are fine
 Style/StderrPuts: { Enabled: false } # this is awful
 
 # amd: these Metric rules are annoying, disable

data/.vscode/settings.json

@@ -1,15 +1,5 @@
 {
-  "editor.formatOnSave": true,
-  "editor.formatOnSaveTimeout": 1500,
-  "editor.tabSize": 2,
-  "editor.wordSeparators": "`~#$%^&*()-=+[{]}\\|;:'\",.<>/",
   "files.associations": {
     "*.sinew": "ruby"
-  },
-  "files.insertFinalNewline": true,
-  "files.trimTrailingWhitespace": true,
-  "ruby.format": "rubocop",
-  "ruby.lint": {
-    "rubocop": true
   }
 }

data/Gemfile

@@ -1,2 +1,11 @@
 source 'http://rubygems.org'
+
+group :development do
+  gem 'minitest'
+  gem 'mocha'
+  gem 'rake'
+  gem 'rubocop', '~> 0.91.0', require: false
+  gem 'webmock'
+end
+
 gemspec

data/README.md

@@ -1,4 +1,4 @@
-![Travis](https://travis-ci.org/gurgeous/sinew.svg?branch=master)
+[![Build Status](https://github.com/gurgeous/sinew/workflows/test/badge.svg?branch=master)](https://github.com/gurgeous/sinew/action)
 
 ## Welcome to Sinew
 
@@ -20,36 +20,30 @@ gem 'sinew'
 
 <!--- markdown-toc --no-firsth1 --maxdepth 1 readme.md -->
 
-* [Sinew 2](#sinew-2-may-2018)
-* [Quick Example](#quick-example)
-* [How it Works](#how-it-works)
-* [DSL Reference](#dsl-reference)
-* [Hints](#hints)
-* [Limitations](#limitations)
-* [Changelog](#changelog)
-* [License](#license)
+- [Sinew 3](#sinew-3-may-2021)
+- [Quick Example](#quick-example)
+- [How it Works](#how-it-works)
+- [DSL Reference](#dsl-reference)
+- [Hints](#hints)
+- [Limitations](#limitations)
+- [Changelog](#changelog)
+- [License](#license)
 
-## Sinew 2 (May 2018)
+## Sinew 3 (May 2021)
 
-I am pleased to announce the release of Sinew 2.0, a complete rewrite of Sinew for the modern era. Enhancements include:
-
-* Remove dependencies on active_support, curl and tidy. We use HTTParty now.
-* Much easier to customize requests in `.sinew` files. For example, setting User-Agent or Bearer tokens.
-* More operations like `post_json` or the generic `http`. These methods are thin wrappers around HTTParty.
-* New end-of-run report.
-* Tests, rubocop, vscode settings, travis, etc.
+I am pleased to announce the release of Sinew 3.0. Sinew has been streamlined and updated to use the [Faraday](https://lostisland.github.io/faraday/) HTTP client with [sinew](https://github.com/gurgeous/sinew/) middleware for caching.
 
 **Breaking change**
 
-Sinew uses a new format for cached responses. Old Sinew 1 cache directories must be removed before running Sinew again. Sinew 2 might choke on Sinew 1 cache directores when reading `head/`. This is not tested or supported.
+Sinew 3 uses a new format for cached responses. Old Sinew 2 cache directories should be removed before running Sinew again.
 
 ## Quick Example
 
-Here's an example for collecting the links from httpbin.org:
+Here's an example for collecting the links from httpbingo.org. Paste this into a file called `sample.sinew` and run `sinew sample.sinew`. It will create a `sample.csv` file containing the href and text for each link:
 
 ```ruby
 # get the url
-get "http://httpbin.org"
+get "http://httpbingo.org"
 
 # use nokogiri to collect links
 noko.css("ul li a").each do |a|
@@ -62,8 +56,6 @@ noko.css("ul li a").each do |a|
 end
 ```
 
-If you paste this into a file called `sample.sinew` and run `sinew sample.sinew`, it will create a `sample.csv` file containing the href and text for each link.
-
 ## How it Works
 
 There are three main features provided by Sinew.
@@ -114,9 +106,9 @@ Sinew creates a CSV file with the same name as the recipe, and `csv_emit(hash)`
 
 #### Caching
 
-Requests are made using HTTParty, and all responses are cached on disk in `~/.sinew`. Error responses are cached as well. Each URL will be hit exactly once, and requests are rate limited to one per second. Sinew tries to be polite.
+Sinew uses [httpdisk](https://github.com/gurgeous/httpdisk/) to aggressively cache all HTTP responses to disk in `~/.sinew`. Error responses are cached as well. Each URL will be hit exactly once, and requests are rate limited to one per second. Sinew tries to be polite.
 
-The files in `~/.sinew` have nice names and are designed to be human readable. This helps when writing recipes. Sinew never deletes files from the cache - that's up to you!
+Sinew never deletes files from the cache - that's up to you!
 
 Because all requests are cached, you can run Sinew repeatedly with confidence. Run it over and over again while you build up your recipe.
 
@@ -124,68 +116,84 @@ Because all requests are cached, you can run Sinew repeatedly with confidence. R
 
 #### Making requests
 
-* `get(url, query = {})` - fetch a url with HTTP GET. URL parameters can be added using `query.
-* `post(url, form = {})` - fetch a url with HTTP POST, using `form` as the URL encoded POST body.
-* `post_json(url, json = {})` - fetch a url with HTTP POST, using `json` as the POST body.
-* `http(method, url, options = {})` - use this for more complex requests
+- `get(url, query = {})` - fetch a url with HTTP GET. URL parameters can be added using `query.
+- `post(url, form = {})` - fetch a url with HTTP POST, using `form` as the URL encoded POST body.
+- `post_json(url, json = {})` - fetch a url with HTTP POST, using `json` as the POST body.
+- `http(method, url, options = {})` - use this for more complex requests
 
 #### Parsing the response
 
 These variables are set after each HTTP request.
 
-* `raw` - the raw response from the last request
-* `html` - like `raw`, but with a handful of HTML-specific whitespace cleanups
-* `noko` - parse the response as HTML and return a [Nokogiri](http://nokogiri.org) document
-* `xml` - parse the response as XML and return a [Nokogiri](http://nokogiri.org) document
-* `json` - parse the response as JSON, with symbolized keys
-* `url` - the url of the last request. If the request goes through a redirect, `url` will reflect the final url.
-* `uri` - the URI of the last request. This is useful for resolving relative URLs.
+- `raw` - the raw response from the last request
+- `html` - like `raw`, but with a handful of HTML-specific whitespace cleanups
+- `noko` - parse the response as HTML and return a [Nokogiri](http://nokogiri.org) document
+- `xml` - parse the response as XML and return a [Nokogiri](http://nokogiri.org) document
+- `json` - parse the response as JSON, with symbolized keys
+- `url` - the url of the last request. If the request goes through a redirect, `url` will reflect the final url.
+- `uri` - the URI of the last request. This is useful for resolving relative URLs.
 
 #### Writing CSV
 
-* `csv_header(keys)` - specify the columns for CSV output. If you don't call this, Sinew will use the keys from the first call to `csv_emit`.
-* `csv_emit(hash)` - append a row to the CSV file
+- `csv_header(keys)` - specify the columns for CSV output. If you don't call this, Sinew will use the keys from the first call to `csv_emit`.
+- `csv_emit(hash)` - append a row to the CSV file
 
 ## Hints
 
 Writing Sinew recipes is fun and easy. The builtin caching means you can iterate quickly, since you won't have to re-fetch the data. Here are some hints for writing idiomatic recipes:
 
-* Sinew doesn't (yet) check robots.txt - please check it manually.
-* Prefer Nokogiri over regular expressions wherever possible. Learn [CSS selectors](http://www.w3schools.com/cssref/css_selectors.asp).
-* In Chrome, `$` in the console is your friend.
-* Fallback to regular expressions if you're desperate. Depending on the site, use either `raw` or `html`. `html` is probably your best bet. `raw` is good for crawling Javascript, but it's fragile if the site changes.
-* Learn to love `String#[regexp]`, which is an obscure operator but incredibly handy for Sinew.
-* Laziness is useful. Keep your CSS selectors and regular expressions simple, so maybe they'll work again the next time you need to crawl a site.
-* Don't be afraid to mix CSS selectors, regular expressions, and Ruby:
+- Sinew doesn't (yet) check robots.txt - please check it manually.
+- Prefer Nokogiri over regular expressions wherever possible. Learn [CSS selectors](http://www.w3schools.com/cssref/css_selectors.asp).
+- In Chrome, `$` in the console is your friend.
+- Fallback to regular expressions if you're desperate. Depending on the site, use either `raw` or `html`. `html` is probably your best bet. `raw` is good for crawling Javascript, but it's fragile if the site changes.
+- Learn to love `String#[regexp]`, which is an obscure operator but incredibly handy for Sinew.
+- Laziness is useful. Keep your CSS selectors and regular expressions simple, so maybe they'll work again the next time you need to crawl a site.
+- Don't be afraid to mix CSS selectors, regular expressions, and Ruby:
 
 ```ruby
 noko.css("table")[4].css("td").select { |i| i[:width].to_i > 80 }.map(&:text)
 ```
 
-* Debug your recipes using plain old `puts`, or better yet use `ap` from [awesome_print](https://github.com/michaeldv/awesome_print).
-* Run `sinew -v` to get a report on every `csv_emit`. Very handy.
-* Add the CSV files to your git repo. That way you can version them and get diffs!
+- Debug your recipes using plain old `puts`, or better yet use `ap` from [amazing_print](https://github.com/amazing-print/amazing_print).
+- Run `sinew -v` to get a report on every `csv_emit`. Very handy.
+- Add the CSV files to your git repo. That way you can version them and get diffs!
 
 ## Limitations
 
-* Caching is based on URL, so use caution with cookies and other forms of authentication
-* Almost no support for international (non-english) characters
+- Caching is based on URL, so use caution with cookies and other forms of authentication
+- Almost no support for international (non-english) characters
 
 ## Changelog
 
+#### 3.0.0 (May 2021)
+
+- Major rewrite of network and caching layer. See above.
+- Use Faraday HTTP client with sinew middleware for caching.
+- Supports multiple proxies (`--proxy host1,host2,...`)
+
+#### 2.0.4 (May 2018)
+
+- Handle and cache more errors (too many redirects, connection failures, etc.)
+- Support for adding uri.scheme in generate_cache_key
+- Added status `code`, a peer to `uri`, `raw`, etc.
+
+#### 2.0.3 (May 2018)
+
+- &amp; now normalizes to & (not and)
+
 #### 2.0.2 (May 2018)
 
-* Support for `--limit`, `--proxy` and the `xml` variable
-* Dedup - warn and ignore if row[:url] has already been emitted
-* Auto gunzip if contents are compressed
+- Support for `--limit`, `--proxy` and the `xml` variable
+- Dedup - warn and ignore if row[:url] has already been emitted
+- Auto gunzip if contents are compressed
 
 #### 2.0.1 (May 2018)
 
-* Support for legacy cached `head` files from Sinew 1
+- Support for legacy cached `head` files from Sinew 1
 
 #### 2.0.0 (May 2018)
 
-* Complete rewrite. See above.
+- Complete rewrite. See above.
 
 #### 1.0.3 (June 2012)
 

data/Rakefile

@@ -1,38 +1,53 @@
-require 'bundler'
 require 'bundler/setup'
 
-require 'rake'
 require 'rake/testtask'
 require 'sinew/version'
 
+# load the spec, we use it below
+spec = Gem::Specification.load('sinew.gemspec')
+
 #
-# gem
+# testing
+# don't forget about TESTOPTS="--verbose" rake
+# also: rake install && rm -rf ~/.sinew/www.amazon.com && /usr/local/bin/sinew sample.sinew
 #
 
-task gem: :build
-task :build do
-  system 'gem build --quiet sinew.gemspec'
+# test (default)
+task default: :test
+
+Rake::TestTask.new do
+  _1.libs << 'test'
+  _1.warning = false # sterile has a few issues here
 end
 
-task install: :build do
-  system "sudo gem install --quiet sinew-#{Sinew::VERSION}.gem"
+# Watch rb files, run tests whenever something changes
+task :watch do
+  # https://superuser.com/a/665208 / https://unix.stackexchange.com/a/42288
+  system("while true; do find . -name '*.rb' | entr -c -d rake; test $? -gt 128 && break; done")
 end
 
-task release: :build do
-  system "git tag -a #{Sinew::VERSION} -m 'Tagging #{Sinew::VERSION}'"
-  system 'git push --tags'
-  system "gem push sinew-#{Sinew::VERSION}.gem"
+#
+# rubocop
+#
+
+task :rubocop do
+  system('bundle exec rubocop -A .', exception: true)
 end
 
 #
-# minitest
+# gem
 #
 
-Rake::TestTask.new(:test) do |t|
-  t.warning = false
+task :build do
+  system 'gem build --quiet sinew.gemspec', exception: true
 end
 
-task default: :test
+task install: :build do
+  system "gem install --quiet sinew-#{spec.version}.gem", exception: true
+end
 
-# to test:
-# block ; rake install && rm -rf ~/.sinew/www.amazon.com && /usr/local/bin/sinew sample.sinew
+task release: %i[rubocop test build] do
+  system "git tag -a #{spec.version} -m 'Tagging #{spec.version}'", exception: true
+  system 'git push --tags', exception: true
+  system "gem push sinew-#{spec.version}.gem", exception: true
+end

data/bin/sinew

@@ -15,6 +15,8 @@ options = Slop.parse do |o|
   o.bool '-q', '--quiet', 'suppress some output'
   o.integer '-l', '--limit', 'quit after emitting this many rows'
   o.string '-c', '--cache', 'set custom cache directory', default: "#{ENV['HOME']}/.sinew"
+  o.bool '--force', "don't read anything from cache (but still write)"
+  o.bool '--force-errors', "don't read errors from cache (but still write)"
   o.string '--proxy', 'use host[:port] as HTTP proxy'
   o.bool '--version', 'show version and exit'
   o.on('--help', 'show this help') do

data/lib/sinew.rb

@@ -1,4 +1,3 @@
-require_relative 'sinew/cache'
 require_relative 'sinew/core_ext'
 require_relative 'sinew/dsl'
 require_relative 'sinew/main'

data/lib/sinew/connection.rb

@@ -0,0 +1,52 @@
+require 'faraday'
+require 'faraday-encoding'
+require 'faraday/logging/formatter'
+require 'httpdisk'
+require 'sinew/connection/log_formatter'
+require 'sinew/connection/rate_limit'
+
+module Sinew
+  module Connection
+    def self.create(options:, runtime_options:)
+      connection_options = {}
+      connection_options[:ssl] = { verify: false } if runtime_options.insecure
+
+      Faraday.new(nil, connection_options) do
+        _1.use RateLimit, rate_limit: runtime_options.rate_limit
+
+        # auto-encode form bodies
+        _1.request :url_encoded
+
+        # Before httpdisk so each redirect segment is cached
+        # Keep track of redirect status for logger
+        _1.response :follow_redirects, callback: ->(_old_env, new_env) { new_env[:redirect] = true }
+
+        # set Ruby string encoding based on Content-Type (should be above httpdisk)
+        _1.response :encoding
+
+        # disk caching
+        httpdisk_options = {
+          dir: options[:cache],
+          force: options[:force],
+          force_errors: options[:force_errors],
+        }.merge(runtime_options.httpdisk_options)
+
+        _1.use :httpdisk, httpdisk_options
+
+        # After httpdisk so that only non-cached requests are logged.
+        # Before retry so that we don't log each retry attempt.
+        _1.response :logger, nil, formatter: LogFormatter if !options[:quiet]
+
+        # After httpdisk so transient failures are not cached
+        retry_options = {
+          interval: runtime_options.rate_limit,
+          max: runtime_options.retries,
+          methods: %w[delete get head options patch post put trace],
+          retry_statuses: (500..600).to_a,
+          retry_if: ->(_env, _err) { true },
+        }
+        _1.request :retry, retry_options
+      end
+    end
+  end
+end

data/lib/sinew/connection/log_formatter.rb

@@ -0,0 +1,22 @@
+module Sinew
+  module Connection
+    class LogFormatter < Faraday::Logging::Formatter
+      def request(env)
+        info('req') do
+          # Only log the initial request, not the redirects
+          return if env[:redirect]
+
+          msg = apply_filters(env.url.to_s)
+          msg = "#{msg} (#{env.method})" if env.method != :get
+          msg = "#{msg} => #{env.request.proxy.uri}" if env.request.proxy
+
+          msg
+        end
+      end
+
+      def response(env)
+        # silent
+      end
+    end
+  end
+end