sinew 2.0.1 → 3.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (40) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/test.yml +26 -0
  3. data/.rubocop.yml +9 -6
  4. data/.vscode/settings.json +0 -10
  5. data/Gemfile +9 -0
  6. data/LICENSE +1 -1
  7. data/README.md +77 -58
  8. data/Rakefile +33 -18
  9. data/bin/sinew +8 -4
  10. data/lib/sinew.rb +0 -1
  11. data/lib/sinew/connection.rb +52 -0
  12. data/lib/sinew/connection/log_formatter.rb +22 -0
  13. data/lib/sinew/connection/rate_limit.rb +29 -0
  14. data/lib/sinew/core_ext.rb +1 -1
  15. data/lib/sinew/dsl.rb +27 -10
  16. data/lib/sinew/main.rb +7 -54
  17. data/lib/sinew/output.rb +26 -19
  18. data/lib/sinew/request.rb +28 -49
  19. data/lib/sinew/response.rb +25 -55
  20. data/lib/sinew/runtime_options.rb +4 -2
  21. data/lib/sinew/version.rb +1 -1
  22. data/sample.sinew +2 -2
  23. data/sinew.gemspec +16 -17
  24. metadata +41 -81
  25. data/.travis.yml +0 -4
  26. data/lib/sinew/cache.rb +0 -79
  27. data/test/legacy/eu.httpbin.org/head/redirect,3 +0 -51
  28. data/test/legacy/eu.httpbin.org/head/status,500 +0 -1
  29. data/test/legacy/eu.httpbin.org/redirect,3 +0 -11
  30. data/test/legacy/eu.httpbin.org/status,500 +0 -1
  31. data/test/legacy/legacy.sinew +0 -2
  32. data/test/test.html +0 -45
  33. data/test/test_cache.rb +0 -69
  34. data/test/test_helper.rb +0 -113
  35. data/test/test_legacy.rb +0 -21
  36. data/test/test_main.rb +0 -46
  37. data/test/test_nokogiri_ext.rb +0 -18
  38. data/test/test_output.rb +0 -73
  39. data/test/test_requests.rb +0 -135
  40. data/test/test_utf8.rb +0 -39
data/.travis.yml DELETED
@@ -1,4 +0,0 @@
1
- language: ruby
2
- rvm:
3
- - 2.3.7
4
- - 2.5.1
data/lib/sinew/cache.rb DELETED
@@ -1,79 +0,0 @@
1
- require 'fileutils'
2
- require 'tempfile'
3
-
4
- #
5
- # This class handles the caching of http responses on disk.
6
- #
7
-
8
- module Sinew
9
- class Cache
10
- attr_reader :sinew
11
-
12
- def initialize(sinew)
13
- @sinew = sinew
14
- end
15
-
16
- def get(request)
17
- body = read_if_exist(body_path(request))
18
- return nil if !body
19
-
20
- head = read_if_exist(head_path(request))
21
- Response.from_cache(request, body, head)
22
- end
23
-
24
- def set(response)
25
- body_path = body_path(response.request)
26
- head_path = head_path(response.request)
27
-
28
- FileUtils.mkdir_p(File.dirname(body_path))
29
- FileUtils.mkdir_p(File.dirname(head_path))
30
-
31
- # write body, and head if necessary
32
- atomic_write(body_path, response.body)
33
- if head_necessary?(response)
34
- head = JSON.pretty_generate(response.head_as_json)
35
- atomic_write(head_path, head)
36
- end
37
- end
38
-
39
- def root_dir
40
- sinew.options[:cache]
41
- end
42
- protected :root_dir
43
-
44
- def head_necessary?(response)
45
- response.error? || response.redirected?
46
- end
47
- protected :head_necessary?
48
-
49
- def body_path(request)
50
- "#{root_dir}/#{request.cache_key}"
51
- end
52
- protected :body_path
53
-
54
- def head_path(request)
55
- body_path = body_path(request)
56
- dir, base = File.dirname(body_path), File.basename(body_path)
57
- "#{dir}/head/#{base}"
58
- end
59
- protected :head_path
60
-
61
- def read_if_exist(path)
62
- if File.exist?(path)
63
- IO.read(path, mode: 'r:UTF-8')
64
- end
65
- end
66
- protected :read_if_exist
67
-
68
- def atomic_write(path, data)
69
- tmp = Tempfile.new('sinew', encoding: 'UTF-8')
70
- tmp.write(data)
71
- tmp.close
72
- FileUtils.chmod(0o644, tmp.path)
73
- FileUtils.mv(tmp.path, path)
74
- ensure
75
- FileUtils.rm(tmp.path, force: true)
76
- end
77
- protected :atomic_write
78
- end
79
- end
data/test/legacy/eu.httpbin.org/head/redirect,3 DELETED
@@ -1,51 +0,0 @@
1
- HTTP/1.1 302 FOUND
2
- Connection: keep-alive
3
- Server: gunicorn/19.7.1
4
- Date: Wed, 02 May 2018 20:55:20 GMT
5
- Content-Type: text/html; charset=utf-8
6
- Content-Length: 247
7
- Location: /relative-redirect/2
8
- Access-Control-Allow-Origin: *
9
- Access-Control-Allow-Credentials: true
10
- X-Powered-By: Flask
11
- X-Processed-Time: 0
12
- Via: 1.1 vegur
13
-
14
- HTTP/1.1 302 FOUND
15
- Connection: keep-alive
16
- Server: gunicorn/19.7.1
17
- Date: Wed, 02 May 2018 20:55:20 GMT
18
- Content-Type: text/html; charset=utf-8
19
- Content-Length: 0
20
- Location: /relative-redirect/1
21
- Access-Control-Allow-Origin: *
22
- Access-Control-Allow-Credentials: true
23
- X-Powered-By: Flask
24
- X-Processed-Time: 0
25
- Via: 1.1 vegur
26
-
27
- HTTP/1.1 302 FOUND
28
- Connection: keep-alive
29
- Server: gunicorn/19.7.1
30
- Date: Wed, 02 May 2018 20:55:20 GMT
31
- Content-Type: text/html; charset=utf-8
32
- Content-Length: 0
33
- Location: /get
34
- Access-Control-Allow-Origin: *
35
- Access-Control-Allow-Credentials: true
36
- X-Powered-By: Flask
37
- X-Processed-Time: 0
38
- Via: 1.1 vegur
39
-
40
- HTTP/1.1 200 OK
41
- Connection: keep-alive
42
- Server: gunicorn/19.7.1
43
- Date: Wed, 02 May 2018 20:55:20 GMT
44
- Content-Type: application/json
45
- Access-Control-Allow-Origin: *
46
- Access-Control-Allow-Credentials: true
47
- X-Powered-By: Flask
48
- X-Processed-Time: 0
49
- Content-Length: 220
50
- Via: 1.1 vegur
51
-
data/test/legacy/eu.httpbin.org/head/status,500 DELETED
@@ -1 +0,0 @@
1
- CURLER_ERROR curl error (22)
data/test/legacy/eu.httpbin.org/redirect,3 DELETED
@@ -1,11 +0,0 @@
1
- {
2
- "args": {},
3
- "headers": {
4
- "Accept": "*/*",
5
- "Connection": "close",
6
- "Host": "eu.httpbin.org",
7
- "User-Agent": "sinew/1.0.4"
8
- },
9
- "origin": "104.36.46.249",
10
- "url": "http://eu.httpbin.org/get"
11
- }
data/test/legacy/eu.httpbin.org/status,500 DELETED
@@ -1 +0,0 @@
1
-
data/test/legacy/legacy.sinew DELETED
@@ -1,2 +0,0 @@
1
- get 'http://eu.httpbin.org/status/500'
2
- get 'http://eu.httpbin.org/redirect/3'
data/test/test.html DELETED
@@ -1,45 +0,0 @@
1
- <html>
2
-
3
- <head>
4
- <title>Title</title>
5
- <script>
6
- alert("alert 1");
7
- alert("alert 2");
8
- </script>
9
- </head>
10
-
11
- <body>
12
- <div id="main">
13
- <span class="class1"> text1 </span>
14
- <span class="class2"> text2 </span>
15
-
16
- <!-- for test_normalize -->
17
- <div id="element"> text </div>
18
- <div class="e"> text1 </div>
19
- <div class="e"> text2 </div>
20
- </div>
21
-
22
- <div id="nokogiri_ext">
23
- <ul>
24
- <li>hello</li>
25
- <li>world</li>
26
- </ul>
27
- <div>
28
- a
29
- <p>b
30
- <span>c</span>
31
- </p>
32
- <p>b
33
- <span>c</span>
34
- </p>
35
- </div>
36
- </div>
37
-
38
- <div id="text_util">
39
- <!-- a comment that should be removed -->
40
- <div class="will_be_removed" />
41
- <a class="will_be_preserved" />
42
- </div>
43
- </body>
44
-
45
- </html>
data/test/test_cache.rb DELETED
@@ -1,69 +0,0 @@
1
- require_relative 'test_helper'
2
-
3
- class TestCache < MiniTest::Test
4
- def test_get
5
- 2.times do
6
- sinew.dsl.get('http://httpbin.org/get', c: 3, d: 4)
7
- end
8
- if !test_network?
9
- assert_requested :get, 'http://httpbin.org/get?c=3&d=4', times: 1
10
- end
11
- assert_equal 1, sinew.request_count
12
- assert_equal({ c: '3', d: '4' }, sinew.dsl.json[:args])
13
- assert File.exist?("#{TMP}/httpbin.org/get,c=3,d=4")
14
- assert !File.exist?("#{TMP}/httpbin.org/head/get,c=3,d=4")
15
- end
16
-
17
- def test_post
18
- 2.times do
19
- sinew.dsl.post('http://httpbin.org/post', c: 5, d: 6)
20
- end
21
- if !test_network?
22
- assert_requested :post, 'http://httpbin.org/post', times: 1
23
- end
24
- assert_equal 1, sinew.request_count
25
- assert_equal({ c: '5', d: '6' }, sinew.dsl.json[:form])
26
- end
27
-
28
- def test_redirect
29
- 2.times do
30
- sinew.dsl.get('http://httpbin.org/redirect/2')
31
- end
32
- if !test_network?
33
- assert_requested :get, 'http://httpbin.org/redirect/2', times: 1
34
- assert_requested :get, 'http://httpbin.org/redirect/1', times: 1
35
- assert_requested :get, 'http://httpbin.org/get', times: 1
36
- end
37
- assert_equal 1, sinew.request_count
38
- assert_equal 'http://httpbin.org/get', sinew.dsl.url
39
- end
40
-
41
- def test_error
42
- # gotta set this or the retries mess up our request counts
43
- sinew.runtime_options.retries = 0
44
- assert_output(/failed with 500/) do
45
- 2.times do
46
- sinew.dsl.get('http://httpbin.org/status/500')
47
- end
48
- end
49
- if !test_network?
50
- assert_requested :get, 'http://httpbin.org/status/500', times: 1
51
- assert_equal '500', sinew.dsl.raw
52
- end
53
- assert_equal 1, sinew.request_count
54
- end
55
-
56
- def test_timeout
57
- return if test_network?
58
-
59
- # gotta set this or the retries mess up our request counts
60
- sinew.runtime_options.retries = 0
61
- assert_output(/failed with 999/) do
62
- 2.times do
63
- sinew.dsl.get('http://httpbin.org/delay/1')
64
- end
65
- end
66
- assert_requested :get, 'http://httpbin.org/delay/1', times: 1
67
- assert_equal 'timeout', sinew.dsl.raw
68
- end
69
- end
data/test/test_helper.rb DELETED
@@ -1,113 +0,0 @@
1
- require 'minitest/autorun'
2
- require 'minitest/pride'
3
- require 'webmock/minitest' unless ENV['SINEW_TEST_NETWORK']
4
-
5
- # a hint to sinew, so that it'll do things like set rate limit to zero
6
- ENV['SINEW_TEST'] = '1'
7
-
8
- # Normally the Rakefile takes care of this, but it's handy to have it here when
9
- # running tests individually.
10
- $LOAD_PATH.unshift("#{__dir__}/../lib")
11
- require 'sinew'
12
-
13
- class MiniTest::Test
14
- TMP = '/tmp/_test_sinew'.freeze
15
- RECIPE = "#{TMP}/test.sinew".freeze
16
- CSV = "#{TMP}/test.csv".freeze
17
- HTML = File.read("#{__dir__}/test.html")
18
-
19
- def setup
20
- super
21
-
22
- # prepare TMP
23
- FileUtils.rm_rf(TMP)
24
- FileUtils.mkdir_p(TMP)
25
-
26
- stub_network unless test_network?
27
- end
28
-
29
- def sinew
30
- @sinew ||= Sinew::Main.new(cache: TMP, quiet: true, recipe: RECIPE)
31
- end
32
- protected :sinew
33
-
34
- def run_recipe(recipe)
35
- File.write(RECIPE, recipe)
36
- sinew.run
37
- end
38
- protected :run_recipe
39
-
40
- def test_network?
41
- !!ENV['SINEW_TEST_NETWORK']
42
- end
43
- protected :test_network?
44
-
45
- # mock requests, patterned on httpbin
46
- def stub_network
47
- stub_request(:get, %r{http://[^/]+/html}).to_return(method(:respond_html))
48
- stub_request(:get, %r{http://[^/]+/get\b}).to_return(method(:respond_echo))
49
- stub_request(:post, %r{http://[^/]+/post\b}).to_return(method(:respond_echo))
50
- stub_request(:get, %r{http://[^/]+/status/\d+}).to_return(method(:respond_status))
51
- stub_request(:get, %r{http://[^/]+/(relative-)?redirect/\d+}).to_return(method(:respond_redirect))
52
- stub_request(:get, %r{http://[^/]+/delay/\d+}).to_timeout
53
- end
54
- protected :stub_network
55
-
56
- #
57
- # respond_xxx helpers
58
- #
59
-
60
- def respond_html(_request)
61
- # this html was carefully chosen to match httpbin.org/html
62
- html = <<~EOF
63
- <body>
64
- <h1>Herman Melville - Moby-Dick</h1>
65
- </body>
66
- EOF
67
- { body: html }
68
- end
69
- protected :respond_html
70
-
71
- def respond_echo(request)
72
- response = {}
73
- response[:headers] = request.headers
74
-
75
- # args
76
- response[:args] = if request.uri.query
77
- CGI.parse(request.uri.query).map { |k, v| [k, v.first] }.to_h
78
- else
79
- {}
80
- end
81
-
82
- # form
83
- if request.headers['Content-Type'] == 'application/x-www-form-urlencoded'
84
- response[:form] = CGI.parse(request.body).map { |k, v| [k, v.first] }.to_h
85
- end
86
-
87
- # json
88
- if request.headers['Content-Type'] == 'application/json'
89
- response[:json] = JSON.parse(request.body)
90
- end
91
-
92
- {
93
- headers: { 'Content-Type' => 'application/json' },
94
- body: response.to_json,
95
- }
96
- end
97
- protected :respond_echo
98
-
99
- def respond_status(request)
100
- status = request.uri.to_s.split('/').last.to_i
101
- { body: status.to_s, status: status }
102
- end
103
- protected :respond_status
104
-
105
- def respond_redirect(request)
106
- parts = request.uri.to_s.split('/')
107
- path, count = parts[-2], parts[-1].to_i
108
- url = count == 1 ? '/get' : "/#{path}/#{count - 1}"
109
- url = "http://example#{url}" if path =~ /absolute/
110
- { status: 302, headers: { 'Location' => url } }
111
- end
112
- protected :respond_redirect
113
- end
data/test/test_legacy.rb DELETED
@@ -1,21 +0,0 @@
1
- require_relative 'test_helper'
2
-
3
- class TestLegacy < MiniTest::Test
4
- def setup
5
- super
6
-
7
- # These are legacy cache files, pulled from an older version of sinew. We
8
- # use them to test our legacy head parsing.
9
- src = 'legacy/eu.httpbin.org'
10
- dst = "#{TMP}/eu.httpbin.org"
11
- FileUtils.cp_r(File.expand_path(src, __dir__), dst)
12
- end
13
-
14
- def test_legacy
15
- sinew.dsl.get('http://eu.httpbin.org/status/500')
16
- assert_equal "\n", sinew.dsl.raw
17
-
18
- sinew.dsl.get('http://eu.httpbin.org/redirect/3')
19
- assert_equal 'http://eu.httpbin.org/get', sinew.dsl.url
20
- end
21
- end
data/test/test_main.rb DELETED
@@ -1,46 +0,0 @@
1
- require_relative 'test_helper'
2
-
3
- class TestMain < MiniTest::Test
4
- def test_noko
5
- run_recipe <<~'EOF'
6
- get 'http://httpbin.org/html'
7
- noko.css("h1").each do |h1|
8
- csv_emit(h1: h1.text)
9
- end
10
- EOF
11
- assert_equal("h1\nHerman Melville - Moby-Dick\n", File.read(CSV))
12
- end
13
-
14
- def test_raw
15
- run_recipe <<~'EOF'
16
- get "http://httpbin.org/html"
17
- raw.scan(/<h1>([^<]+)/) do
18
- csv_emit(h1: $1)
19
- end
20
- EOF
21
- assert_equal("h1\nHerman Melville - Moby-Dick\n", File.read(CSV))
22
- end
23
-
24
- def test_rate_limit
25
- # true network requests call sleep for timeouts, which interferes with our
26
- # instrumentation of Kernel#sleep
27
- skip if test_network?
28
-
29
- slept = false
30
-
31
- # change Kernel#sleep to not really sleep!
32
- Kernel.send(:alias_method, :old_sleep, :sleep)
33
- Kernel.send(:define_method, :sleep) do |_duration|
34
- slept = true
35
- end
36
-
37
- sinew.runtime_options.rate_limit = 1
38
- sinew.dsl.get('http://httpbin.org/html')
39
- sinew.dsl.get('http://httpbin.org/get')
40
- assert(slept)
41
-
42
- # restore old Kernel#sleep
43
- Kernel.send(:alias_method, :sleep, :old_sleep)
44
- Kernel.send(:undef_method, :old_sleep)
45
- end
46
- end