sinew 2.0.2 → 3.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/test.yml +26 -0
  3. data/.rubocop.yml +9 -6
  4. data/.vscode/settings.json +0 -10
  5. data/Gemfile +9 -0
  6. data/README.md +62 -54
  7. data/Rakefile +33 -18
  8. data/bin/sinew +2 -0
  9. data/lib/sinew.rb +0 -1
  10. data/lib/sinew/connection.rb +52 -0
  11. data/lib/sinew/connection/log_formatter.rb +22 -0
  12. data/lib/sinew/connection/rate_limit.rb +29 -0
  13. data/lib/sinew/core_ext.rb +1 -1
  14. data/lib/sinew/dsl.rb +10 -6
  15. data/lib/sinew/main.rb +29 -56
  16. data/lib/sinew/output.rb +7 -16
  17. data/lib/sinew/request.rb +22 -87
  18. data/lib/sinew/response.rb +8 -57
  19. data/lib/sinew/runtime_options.rb +4 -4
  20. data/lib/sinew/version.rb +1 -1
  21. data/sample.sinew +2 -2
  22. data/sinew.gemspec +16 -18
  23. metadata +38 -110
  24. data/.travis.yml +0 -4
  25. data/lib/sinew/cache.rb +0 -79
  26. data/test/legacy/eu.httpbin.org/head/redirect,3 +0 -51
  27. data/test/legacy/eu.httpbin.org/head/status,500 +0 -1
  28. data/test/legacy/eu.httpbin.org/redirect,3 +0 -11
  29. data/test/legacy/eu.httpbin.org/status,500 +0 -1
  30. data/test/legacy/legacy.sinew +0 -2
  31. data/test/recipes/array_header.sinew +0 -6
  32. data/test/recipes/basic.sinew +0 -8
  33. data/test/recipes/dups.sinew +0 -7
  34. data/test/recipes/implicit_header.sinew +0 -5
  35. data/test/recipes/limit.sinew +0 -11
  36. data/test/recipes/noko.sinew +0 -9
  37. data/test/recipes/uri.sinew +0 -11
  38. data/test/recipes/xml.sinew +0 -8
  39. data/test/test.html +0 -45
  40. data/test/test_cache.rb +0 -69
  41. data/test/test_helper.rb +0 -123
  42. data/test/test_legacy.rb +0 -23
  43. data/test/test_main.rb +0 -34
  44. data/test/test_nokogiri_ext.rb +0 -18
  45. data/test/test_output.rb +0 -56
  46. data/test/test_recipes.rb +0 -60
  47. data/test/test_requests.rb +0 -135
  48. data/test/test_utf8.rb +0 -39
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 33506a03f47a88cae5bf7e0f4675d7cf83d86ba3c96f0880f5c473a7b23b167b
4
- data.tar.gz: 990bd4690f9fe799774c349314a32ab2c08979d555f03891316c5e0be8a4ad3d
3
+ metadata.gz: ba5558019816540d71e1bb44029f2733aff53649a3f4da72a9905e0a67d06ad9
4
+ data.tar.gz: 446c245782cad55f1caa36e01b0e8c98748295b47eb5b1911175fff2f79589b2
5
5
  SHA512:
6
- metadata.gz: 9644097a2e11d8cba59a7985dfe770f27b00d5d18b676d0cacdee3e73a21f1b6c237b3bb58d68489d2a67fc981f7a7f8bb27a6e6fb23781f318cde78b392d7cd
7
- data.tar.gz: 667c301e7896b27162a77cff5165f264a0c2b73afbe5c35f541181709118185a13241d187ff8f8d5964e302537064ff927d9fa64ece2cb10ca65ba7dd89ce807
6
+ metadata.gz: cef8c1145a21e84f560b44821071ffc7b57ef965167b633e1c837f7b7d9dbfce340b14d1afb2a35891c7b1ed4aa4f08e47ca7405cf382acca7eae855a47d3a71
7
+ data.tar.gz: 8dc7b67511fc541cccef23b69463abd9a8081f9c75583c1d6c6756dda0561ce7a91be35ee06f7d886ea7488b9005c969e14fa10794cac92475a3063c8968abec
@@ -0,0 +1,26 @@
1
+ name: test
2
+
3
+ on:
4
+ push:
5
+ paths-ignore:
6
+ - '**.md'
7
+ pull_request:
8
+ paths-ignore:
9
+ - '**.md'
10
+ workflow_dispatch:
11
+
12
+ jobs:
13
+ test:
14
+ strategy:
15
+ max-parallel: 3
16
+ matrix:
17
+ os: [ubuntu, macos]
18
+ ruby-version: [3.0, 2.7]
19
+ runs-on: ${{ matrix.os }}-latest
20
+ steps:
21
+ - uses: actions/checkout@v2
22
+ - uses: ruby/setup-ruby@v1
23
+ with:
24
+ ruby-version: ${{ matrix.ruby-version }}
25
+ - run: bundle install
26
+ - run: bundle exec rake test
data/.rubocop.yml CHANGED
@@ -1,6 +1,6 @@
1
1
  AllCops:
2
- Exclude:
3
- TargetRubyVersion: 2.3
2
+ TargetRubyVersion: 2.7
3
+ NewCops: enable
4
4
 
5
5
  # amd: customizations
6
6
  Layout/SpaceInsideArrayLiteralBrackets:
@@ -22,18 +22,19 @@ Style/TrailingCommaInHashLiteral:
22
22
 
23
23
  # amd: these seem extreme
24
24
  Lint/AssignmentInCondition: { Enabled: false } # I do this all the time
25
- Lint/HandleExceptions: { Enabled: false } # blank rescues are useful
25
+ Lint/SuppressedException: { Enabled: false } # blank rescues are useful
26
26
  Naming/BinaryOperatorParameterName: { Enabled: false } # silly
27
27
  Naming/HeredocDelimiterNaming: { Enabled: false } # silly
28
- Naming/UncommunicativeMethodParamName: { Enabled: false } # silly
29
- Performance/RegexpMatch: { Enabled: false } # =~ is fine
30
- Performance/TimesMap: { Enabled: false } # silly
28
+ Naming/MethodParameterName: { Enabled: false } # silly
29
+ Style/AccessorGrouping: { Enabled: false } # silly
30
+ Style/AsciiComments: { Enabled: false } # silly
31
31
  Style/ClassAndModuleChildren: { Enabled: false } # silly
32
32
  Style/Documentation: { Enabled: false } # we don't need this
33
33
  Style/DoubleNegation: { Enabled: false } # silly
34
34
  Style/FormatStringToken: { Enabled: false } # we like printf here
35
35
  Style/FrozenStringLiteralComment: { Enabled: false } # seems excessive
36
36
  Style/GuardClause: { Enabled: false } # confusing
37
+ Style/HashTransformValues: { Enabled: false } # breaks code by trying to apply to an array
37
38
  Style/IfUnlessModifier: { Enabled: false } # personally I hate unless
38
39
  Style/NegatedIf: { Enabled: false } # these are fine
39
40
  Style/Next: { Enabled: false } # these are fine
@@ -41,7 +42,9 @@ Style/NumericPredicate: { Enabled: false } # silly
41
42
  Style/ParallelAssignment: { Enabled: false } # these are fine
42
43
  Style/PerlBackrefs: { Enabled: false } # these are fine
43
44
  Style/RaiseArgs: { Enabled: false } # silly
45
+ Style/RedundantAssignment: { Enabled: false } # these are usually on purpose
44
46
  Style/RegexpLiteral: { Enabled: false } # these are fine
47
+ Style/SoleNestedConditional: { Enabled: false } # these are fine
45
48
  Style/StderrPuts: { Enabled: false } # this is awful
46
49
 
47
50
  # amd: these Metric rules are annoying, disable
@@ -1,15 +1,5 @@
1
1
  {
2
- "editor.formatOnSave": true,
3
- "editor.formatOnSaveTimeout": 1500,
4
- "editor.tabSize": 2,
5
- "editor.wordSeparators": "`~#$%^&*()-=+[{]}\\|;:'\",.<>/",
6
2
  "files.associations": {
7
3
  "*.sinew": "ruby"
8
- },
9
- "files.insertFinalNewline": true,
10
- "files.trimTrailingWhitespace": true,
11
- "ruby.format": "rubocop",
12
- "ruby.lint": {
13
- "rubocop": true
14
4
  }
15
5
  }
data/Gemfile CHANGED
@@ -1,2 +1,11 @@
1
1
  source 'http://rubygems.org'
2
+
3
+ group :development do
4
+ gem 'minitest'
5
+ gem 'mocha'
6
+ gem 'rake'
7
+ gem 'rubocop', '~> 0.91.0', require: false
8
+ gem 'webmock'
9
+ end
10
+
2
11
  gemspec
data/README.md CHANGED
@@ -1,4 +1,4 @@
1
- ![Travis](https://travis-ci.org/gurgeous/sinew.svg?branch=master)
1
+ [![Build Status](https://github.com/gurgeous/sinew/workflows/test/badge.svg?branch=master)](https://github.com/gurgeous/sinew/action)
2
2
 
3
3
  ## Welcome to Sinew
4
4
 
@@ -20,36 +20,30 @@ gem 'sinew'
20
20
 
21
21
  <!--- markdown-toc --no-firsth1 --maxdepth 1 readme.md -->
22
22
 
23
- * [Sinew 2](#sinew-2-may-2018)
24
- * [Quick Example](#quick-example)
25
- * [How it Works](#how-it-works)
26
- * [DSL Reference](#dsl-reference)
27
- * [Hints](#hints)
28
- * [Limitations](#limitations)
29
- * [Changelog](#changelog)
30
- * [License](#license)
23
+ - [Sinew 3](#sinew-3-may-2021)
24
+ - [Quick Example](#quick-example)
25
+ - [How it Works](#how-it-works)
26
+ - [DSL Reference](#dsl-reference)
27
+ - [Hints](#hints)
28
+ - [Limitations](#limitations)
29
+ - [Changelog](#changelog)
30
+ - [License](#license)
31
31
 
32
- ## Sinew 2 (May 2018)
32
+ ## Sinew 3 (May 2021)
33
33
 
34
- I am pleased to announce the release of Sinew 2.0, a complete rewrite of Sinew for the modern era. Enhancements include:
35
-
36
- * Remove dependencies on active_support, curl and tidy. We use HTTParty now.
37
- * Much easier to customize requests in `.sinew` files. For example, setting User-Agent or Bearer tokens.
38
- * More operations like `post_json` or the generic `http`. These methods are thin wrappers around HTTParty.
39
- * New end-of-run report.
40
- * Tests, rubocop, vscode settings, travis, etc.
34
+ I am pleased to announce the release of Sinew 3.0. Sinew has been streamlined and updated to use the [Faraday](https://lostisland.github.io/faraday/) HTTP client with [sinew](https://github.com/gurgeous/sinew/) middleware for caching.
41
35
 
42
36
  **Breaking change**
43
37
 
44
- Sinew uses a new format for cached responses. Old Sinew 1 cache directories must be removed before running Sinew again. Sinew 2 might choke on Sinew 1 cache directores when reading `head/`. This is not tested or supported.
38
+ Sinew 3 uses a new format for cached responses. Old Sinew 2 cache directories should be removed before running Sinew again.
45
39
 
46
40
  ## Quick Example
47
41
 
48
- Here's an example for collecting the links from httpbin.org:
42
+ Here's an example for collecting the links from httpbingo.org. Paste this into a file called `sample.sinew` and run `sinew sample.sinew`. It will create a `sample.csv` file containing the href and text for each link:
49
43
 
50
44
  ```ruby
51
45
  # get the url
52
- get "http://httpbin.org"
46
+ get "http://httpbingo.org"
53
47
 
54
48
  # use nokogiri to collect links
55
49
  noko.css("ul li a").each do |a|
@@ -62,8 +56,6 @@ noko.css("ul li a").each do |a|
62
56
  end
63
57
  ```
64
58
 
65
- If you paste this into a file called `sample.sinew` and run `sinew sample.sinew`, it will create a `sample.csv` file containing the href and text for each link.
66
-
67
59
  ## How it Works
68
60
 
69
61
  There are three main features provided by Sinew.
@@ -114,9 +106,9 @@ Sinew creates a CSV file with the same name as the recipe, and `csv_emit(hash)`
114
106
 
115
107
  #### Caching
116
108
 
117
- Requests are made using HTTParty, and all responses are cached on disk in `~/.sinew`. Error responses are cached as well. Each URL will be hit exactly once, and requests are rate limited to one per second. Sinew tries to be polite.
109
+ Sinew uses [httpdisk](https://github.com/gurgeous/httpdisk/) to aggressively cache all HTTP responses to disk in `~/.sinew`. Error responses are cached as well. Each URL will be hit exactly once, and requests are rate limited to one per second. Sinew tries to be polite.
118
110
 
119
- The files in `~/.sinew` have nice names and are designed to be human readable. This helps when writing recipes. Sinew never deletes files from the cache - that's up to you!
111
+ Sinew never deletes files from the cache - that's up to you!
120
112
 
121
113
  Because all requests are cached, you can run Sinew repeatedly with confidence. Run it over and over again while you build up your recipe.
122
114
 
@@ -124,68 +116,84 @@ Because all requests are cached, you can run Sinew repeatedly with confidence. R
124
116
 
125
117
  #### Making requests
126
118
 
127
- * `get(url, query = {})` - fetch a url with HTTP GET. URL parameters can be added using `query.
128
- * `post(url, form = {})` - fetch a url with HTTP POST, using `form` as the URL encoded POST body.
129
- * `post_json(url, json = {})` - fetch a url with HTTP POST, using `json` as the POST body.
130
- * `http(method, url, options = {})` - use this for more complex requests
119
+ - `get(url, query = {})` - fetch a url with HTTP GET. URL parameters can be added using `query.
120
+ - `post(url, form = {})` - fetch a url with HTTP POST, using `form` as the URL encoded POST body.
121
+ - `post_json(url, json = {})` - fetch a url with HTTP POST, using `json` as the POST body.
122
+ - `http(method, url, options = {})` - use this for more complex requests
131
123
 
132
124
  #### Parsing the response
133
125
 
134
126
  These variables are set after each HTTP request.
135
127
 
136
- * `raw` - the raw response from the last request
137
- * `html` - like `raw`, but with a handful of HTML-specific whitespace cleanups
138
- * `noko` - parse the response as HTML and return a [Nokogiri](http://nokogiri.org) document
139
- * `xml` - parse the response as XML and return a [Nokogiri](http://nokogiri.org) document
140
- * `json` - parse the response as JSON, with symbolized keys
141
- * `url` - the url of the last request. If the request goes through a redirect, `url` will reflect the final url.
142
- * `uri` - the URI of the last request. This is useful for resolving relative URLs.
128
+ - `raw` - the raw response from the last request
129
+ - `html` - like `raw`, but with a handful of HTML-specific whitespace cleanups
130
+ - `noko` - parse the response as HTML and return a [Nokogiri](http://nokogiri.org) document
131
+ - `xml` - parse the response as XML and return a [Nokogiri](http://nokogiri.org) document
132
+ - `json` - parse the response as JSON, with symbolized keys
133
+ - `url` - the url of the last request. If the request goes through a redirect, `url` will reflect the final url.
134
+ - `uri` - the URI of the last request. This is useful for resolving relative URLs.
143
135
 
144
136
  #### Writing CSV
145
137
 
146
- * `csv_header(keys)` - specify the columns for CSV output. If you don't call this, Sinew will use the keys from the first call to `csv_emit`.
147
- * `csv_emit(hash)` - append a row to the CSV file
138
+ - `csv_header(keys)` - specify the columns for CSV output. If you don't call this, Sinew will use the keys from the first call to `csv_emit`.
139
+ - `csv_emit(hash)` - append a row to the CSV file
148
140
 
149
141
  ## Hints
150
142
 
151
143
  Writing Sinew recipes is fun and easy. The builtin caching means you can iterate quickly, since you won't have to re-fetch the data. Here are some hints for writing idiomatic recipes:
152
144
 
153
- * Sinew doesn't (yet) check robots.txt - please check it manually.
154
- * Prefer Nokogiri over regular expressions wherever possible. Learn [CSS selectors](http://www.w3schools.com/cssref/css_selectors.asp).
155
- * In Chrome, `$` in the console is your friend.
156
- * Fallback to regular expressions if you're desperate. Depending on the site, use either `raw` or `html`. `html` is probably your best bet. `raw` is good for crawling Javascript, but it's fragile if the site changes.
157
- * Learn to love `String#[regexp]`, which is an obscure operator but incredibly handy for Sinew.
158
- * Laziness is useful. Keep your CSS selectors and regular expressions simple, so maybe they'll work again the next time you need to crawl a site.
159
- * Don't be afraid to mix CSS selectors, regular expressions, and Ruby:
145
+ - Sinew doesn't (yet) check robots.txt - please check it manually.
146
+ - Prefer Nokogiri over regular expressions wherever possible. Learn [CSS selectors](http://www.w3schools.com/cssref/css_selectors.asp).
147
+ - In Chrome, `$` in the console is your friend.
148
+ - Fallback to regular expressions if you're desperate. Depending on the site, use either `raw` or `html`. `html` is probably your best bet. `raw` is good for crawling Javascript, but it's fragile if the site changes.
149
+ - Learn to love `String#[regexp]`, which is an obscure operator but incredibly handy for Sinew.
150
+ - Laziness is useful. Keep your CSS selectors and regular expressions simple, so maybe they'll work again the next time you need to crawl a site.
151
+ - Don't be afraid to mix CSS selectors, regular expressions, and Ruby:
160
152
 
161
153
  ```ruby
162
154
  noko.css("table")[4].css("td").select { |i| i[:width].to_i > 80 }.map(&:text)
163
155
  ```
164
156
 
165
- * Debug your recipes using plain old `puts`, or better yet use `ap` from [awesome_print](https://github.com/michaeldv/awesome_print).
166
- * Run `sinew -v` to get a report on every `csv_emit`. Very handy.
167
- * Add the CSV files to your git repo. That way you can version them and get diffs!
157
+ - Debug your recipes using plain old `puts`, or better yet use `ap` from [amazing_print](https://github.com/amazing-print/amazing_print).
158
+ - Run `sinew -v` to get a report on every `csv_emit`. Very handy.
159
+ - Add the CSV files to your git repo. That way you can version them and get diffs!
168
160
 
169
161
  ## Limitations
170
162
 
171
- * Caching is based on URL, so use caution with cookies and other forms of authentication
172
- * Almost no support for international (non-english) characters
163
+ - Caching is based on URL, so use caution with cookies and other forms of authentication
164
+ - Almost no support for international (non-english) characters
173
165
 
174
166
  ## Changelog
175
167
 
168
+ #### 3.0.0 (May 2021)
169
+
170
+ - Major rewrite of network and caching layer. See above.
171
+ - Use Faraday HTTP client with sinew middleware for caching.
172
+ - Supports multiple proxies (`--proxy host1,host2,...`)
173
+
174
+ #### 2.0.4 (May 2018)
175
+
176
+ - Handle and cache more errors (too many redirects, connection failures, etc.)
177
+ - Support for adding uri.scheme in generate_cache_key
178
+ - Added status `code`, a peer to `uri`, `raw`, etc.
179
+
180
+ #### 2.0.3 (May 2018)
181
+
182
+ - &amp; now normalizes to & (not and)
183
+
176
184
  #### 2.0.2 (May 2018)
177
185
 
178
- * Support for `--limit`, `--proxy` and the `xml` variable
179
- * Dedup - warn and ignore if row[:url] has already been emitted
180
- * Auto gunzip if contents are compressed
186
+ - Support for `--limit`, `--proxy` and the `xml` variable
187
+ - Dedup - warn and ignore if row[:url] has already been emitted
188
+ - Auto gunzip if contents are compressed
181
189
 
182
190
  #### 2.0.1 (May 2018)
183
191
 
184
- * Support for legacy cached `head` files from Sinew 1
192
+ - Support for legacy cached `head` files from Sinew 1
185
193
 
186
194
  #### 2.0.0 (May 2018)
187
195
 
188
- * Complete rewrite. See above.
196
+ - Complete rewrite. See above.
189
197
 
190
198
  #### 1.0.3 (June 2012)
191
199
 
data/Rakefile CHANGED
@@ -1,38 +1,53 @@
1
- require 'bundler'
2
1
  require 'bundler/setup'
3
2
 
4
- require 'rake'
5
3
  require 'rake/testtask'
6
4
  require 'sinew/version'
7
5
 
6
+ # load the spec, we use it below
7
+ spec = Gem::Specification.load('sinew.gemspec')
8
+
8
9
  #
9
- # gem
10
+ # testing
11
+ # don't forget about TESTOPTS="--verbose" rake
12
+ # also: rake install && rm -rf ~/.sinew/www.amazon.com && /usr/local/bin/sinew sample.sinew
10
13
  #
11
14
 
12
- task gem: :build
13
- task :build do
14
- system 'gem build --quiet sinew.gemspec'
15
+ # test (default)
16
+ task default: :test
17
+
18
+ Rake::TestTask.new do
19
+ _1.libs << 'test'
20
+ _1.warning = false # sterile has a few issues here
15
21
  end
16
22
 
17
- task install: :build do
18
- system "sudo gem install --quiet sinew-#{Sinew::VERSION}.gem"
23
+ # Watch rb files, run tests whenever something changes
24
+ task :watch do
25
+ # https://superuser.com/a/665208 / https://unix.stackexchange.com/a/42288
26
+ system("while true; do find . -name '*.rb' | entr -c -d rake; test $? -gt 128 && break; done")
19
27
  end
20
28
 
21
- task release: :build do
22
- system "git tag -a #{Sinew::VERSION} -m 'Tagging #{Sinew::VERSION}'"
23
- system 'git push --tags'
24
- system "gem push sinew-#{Sinew::VERSION}.gem"
29
+ #
30
+ # rubocop
31
+ #
32
+
33
+ task :rubocop do
34
+ system('bundle exec rubocop -A .', exception: true)
25
35
  end
26
36
 
27
37
  #
28
- # minitest
38
+ # gem
29
39
  #
30
40
 
31
- Rake::TestTask.new(:test) do |t|
32
- t.warning = false
41
+ task :build do
42
+ system 'gem build --quiet sinew.gemspec', exception: true
33
43
  end
34
44
 
35
- task default: :test
45
+ task install: :build do
46
+ system "gem install --quiet sinew-#{spec.version}.gem", exception: true
47
+ end
36
48
 
37
- # to test:
38
- # block ; rake install && rm -rf ~/.sinew/www.amazon.com && /usr/local/bin/sinew sample.sinew
49
+ task release: %i[rubocop test build] do
50
+ system "git tag -a #{spec.version} -m 'Tagging #{spec.version}'", exception: true
51
+ system 'git push --tags', exception: true
52
+ system "gem push sinew-#{spec.version}.gem", exception: true
53
+ end
data/bin/sinew CHANGED
@@ -15,6 +15,8 @@ options = Slop.parse do |o|
15
15
  o.bool '-q', '--quiet', 'suppress some output'
16
16
  o.integer '-l', '--limit', 'quit after emitting this many rows'
17
17
  o.string '-c', '--cache', 'set custom cache directory', default: "#{ENV['HOME']}/.sinew"
18
+ o.bool '--force', "don't read anything from cache (but still write)"
19
+ o.bool '--force-errors', "don't read errors from cache (but still write)"
18
20
  o.string '--proxy', 'use host[:port] as HTTP proxy'
19
21
  o.bool '--version', 'show version and exit'
20
22
  o.on('--help', 'show this help') do
data/lib/sinew.rb CHANGED
@@ -1,4 +1,3 @@
1
- require_relative 'sinew/cache'
2
1
  require_relative 'sinew/core_ext'
3
2
  require_relative 'sinew/dsl'
4
3
  require_relative 'sinew/main'
@@ -0,0 +1,52 @@
1
+ require 'faraday'
2
+ require 'faraday-encoding'
3
+ require 'faraday/logging/formatter'
4
+ require 'httpdisk'
5
+ require 'sinew/connection/log_formatter'
6
+ require 'sinew/connection/rate_limit'
7
+
8
+ module Sinew
9
+ module Connection
10
+ def self.create(options:, runtime_options:)
11
+ connection_options = {}
12
+ connection_options[:ssl] = { verify: false } if runtime_options.insecure
13
+
14
+ Faraday.new(nil, connection_options) do
15
+ _1.use RateLimit, rate_limit: runtime_options.rate_limit
16
+
17
+ # auto-encode form bodies
18
+ _1.request :url_encoded
19
+
20
+ # Before httpdisk so each redirect segment is cached
21
+ # Keep track of redirect status for logger
22
+ _1.response :follow_redirects, callback: ->(_old_env, new_env) { new_env[:redirect] = true }
23
+
24
+ # set Ruby string encoding based on Content-Type (should be above httpdisk)
25
+ _1.response :encoding
26
+
27
+ # disk caching
28
+ httpdisk_options = {
29
+ dir: options[:cache],
30
+ force: options[:force],
31
+ force_errors: options[:force_errors],
32
+ }.merge(runtime_options.httpdisk_options)
33
+
34
+ _1.use :httpdisk, httpdisk_options
35
+
36
+ # After httpdisk so that only non-cached requests are logged.
37
+ # Before retry so that we don't log each retry attempt.
38
+ _1.response :logger, nil, formatter: LogFormatter if !options[:quiet]
39
+
40
+ # After httpdisk so transient failures are not cached
41
+ retry_options = {
42
+ interval: runtime_options.rate_limit,
43
+ max: runtime_options.retries,
44
+ methods: %w[delete get head options patch post put trace],
45
+ retry_statuses: (500..600).to_a,
46
+ retry_if: ->(_env, _err) { true },
47
+ }
48
+ _1.request :retry, retry_options
49
+ end
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,22 @@
1
+ module Sinew
2
+ module Connection
3
+ class LogFormatter < Faraday::Logging::Formatter
4
+ def request(env)
5
+ info('req') do
6
+ # Only log the initial request, not the redirects
7
+ return if env[:redirect]
8
+
9
+ msg = apply_filters(env.url.to_s)
10
+ msg = "#{msg} (#{env.method})" if env.method != :get
11
+ msg = "#{msg} => #{env.request.proxy.uri}" if env.request.proxy
12
+
13
+ msg
14
+ end
15
+ end
16
+
17
+ def response(env)
18
+ # silent
19
+ end
20
+ end
21
+ end
22
+ end