sinew 2.0.2 → 3.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/test.yml +26 -0
  3. data/.rubocop.yml +9 -6
  4. data/.vscode/settings.json +0 -10
  5. data/Gemfile +9 -0
  6. data/README.md +62 -54
  7. data/Rakefile +33 -18
  8. data/bin/sinew +2 -0
  9. data/lib/sinew.rb +0 -1
  10. data/lib/sinew/connection.rb +52 -0
  11. data/lib/sinew/connection/log_formatter.rb +22 -0
  12. data/lib/sinew/connection/rate_limit.rb +29 -0
  13. data/lib/sinew/core_ext.rb +1 -1
  14. data/lib/sinew/dsl.rb +10 -6
  15. data/lib/sinew/main.rb +29 -56
  16. data/lib/sinew/output.rb +7 -16
  17. data/lib/sinew/request.rb +22 -87
  18. data/lib/sinew/response.rb +8 -57
  19. data/lib/sinew/runtime_options.rb +4 -4
  20. data/lib/sinew/version.rb +1 -1
  21. data/sample.sinew +2 -2
  22. data/sinew.gemspec +16 -18
  23. metadata +38 -110
  24. data/.travis.yml +0 -4
  25. data/lib/sinew/cache.rb +0 -79
  26. data/test/legacy/eu.httpbin.org/head/redirect,3 +0 -51
  27. data/test/legacy/eu.httpbin.org/head/status,500 +0 -1
  28. data/test/legacy/eu.httpbin.org/redirect,3 +0 -11
  29. data/test/legacy/eu.httpbin.org/status,500 +0 -1
  30. data/test/legacy/legacy.sinew +0 -2
  31. data/test/recipes/array_header.sinew +0 -6
  32. data/test/recipes/basic.sinew +0 -8
  33. data/test/recipes/dups.sinew +0 -7
  34. data/test/recipes/implicit_header.sinew +0 -5
  35. data/test/recipes/limit.sinew +0 -11
  36. data/test/recipes/noko.sinew +0 -9
  37. data/test/recipes/uri.sinew +0 -11
  38. data/test/recipes/xml.sinew +0 -8
  39. data/test/test.html +0 -45
  40. data/test/test_cache.rb +0 -69
  41. data/test/test_helper.rb +0 -123
  42. data/test/test_legacy.rb +0 -23
  43. data/test/test_main.rb +0 -34
  44. data/test/test_nokogiri_ext.rb +0 -18
  45. data/test/test_output.rb +0 -56
  46. data/test/test_recipes.rb +0 -60
  47. data/test/test_requests.rb +0 -135
  48. data/test/test_utf8.rb +0 -39
data/test/legacy/eu.httpbin.org/head/redirect,3 DELETED
@@ -1,51 +0,0 @@
1
- HTTP/1.1 302 FOUND
2
- Connection: keep-alive
3
- Server: gunicorn/19.7.1
4
- Date: Wed, 02 May 2018 20:55:20 GMT
5
- Content-Type: text/html; charset=utf-8
6
- Content-Length: 247
7
- Location: /relative-redirect/2
8
- Access-Control-Allow-Origin: *
9
- Access-Control-Allow-Credentials: true
10
- X-Powered-By: Flask
11
- X-Processed-Time: 0
12
- Via: 1.1 vegur
13
-
14
- HTTP/1.1 302 FOUND
15
- Connection: keep-alive
16
- Server: gunicorn/19.7.1
17
- Date: Wed, 02 May 2018 20:55:20 GMT
18
- Content-Type: text/html; charset=utf-8
19
- Content-Length: 0
20
- Location: /relative-redirect/1
21
- Access-Control-Allow-Origin: *
22
- Access-Control-Allow-Credentials: true
23
- X-Powered-By: Flask
24
- X-Processed-Time: 0
25
- Via: 1.1 vegur
26
-
27
- HTTP/1.1 302 FOUND
28
- Connection: keep-alive
29
- Server: gunicorn/19.7.1
30
- Date: Wed, 02 May 2018 20:55:20 GMT
31
- Content-Type: text/html; charset=utf-8
32
- Content-Length: 0
33
- Location: /get
34
- Access-Control-Allow-Origin: *
35
- Access-Control-Allow-Credentials: true
36
- X-Powered-By: Flask
37
- X-Processed-Time: 0
38
- Via: 1.1 vegur
39
-
40
- HTTP/1.1 200 OK
41
- Connection: keep-alive
42
- Server: gunicorn/19.7.1
43
- Date: Wed, 02 May 2018 20:55:20 GMT
44
- Content-Type: application/json
45
- Access-Control-Allow-Origin: *
46
- Access-Control-Allow-Credentials: true
47
- X-Powered-By: Flask
48
- X-Processed-Time: 0
49
- Content-Length: 220
50
- Via: 1.1 vegur
51
-
data/test/legacy/eu.httpbin.org/head/status,500 DELETED
@@ -1 +0,0 @@
1
- CURLER_ERROR curl error (22)
data/test/legacy/eu.httpbin.org/redirect,3 DELETED
@@ -1,11 +0,0 @@
1
- {
2
- "args": {},
3
- "headers": {
4
- "Accept": "*/*",
5
- "Connection": "close",
6
- "Host": "eu.httpbin.org",
7
- "User-Agent": "sinew/1.0.4"
8
- },
9
- "origin": "104.36.46.249",
10
- "url": "http://eu.httpbin.org/get"
11
- }
data/test/legacy/eu.httpbin.org/status,500 DELETED
@@ -1 +0,0 @@
1
-
data/test/legacy/legacy.sinew DELETED
@@ -1,2 +0,0 @@
1
- get 'http://eu.httpbin.org/status/500'
2
- get 'http://eu.httpbin.org/redirect/3'
data/test/recipes/array_header.sinew DELETED
@@ -1,6 +0,0 @@
1
- csv_header(%i[n a p])
2
- csv_emit(n: 'n1', a: 'a1')
3
-
4
- # OUTPUT
5
- # n,a,p
6
- # n1,a1,""
data/test/recipes/basic.sinew DELETED
@@ -1,8 +0,0 @@
1
- get 'http://httpbin.org/html'
2
- raw.scan(/<h1>([^<]+)/) do
3
- csv_emit(h1: $1)
4
- end
5
-
6
- # OUTPUT
7
- # h1
8
- # Herman Melville - Moby-Dick
data/test/recipes/dups.sinew DELETED
@@ -1,7 +0,0 @@
1
- 5.times do
2
- csv_emit(url: 'https://gub')
3
- end
4
-
5
- # OUTPUT
6
- # url
7
- # https://gub
data/test/recipes/implicit_header.sinew DELETED
@@ -1,5 +0,0 @@
1
- csv_emit(name: 'bob', address: 'main')
2
-
3
- # OUTPUT
4
- # name,address
5
- # bob,main
data/test/recipes/limit.sinew DELETED
@@ -1,11 +0,0 @@
1
- # OPTIONS { limit: 3 }
2
-
3
- (1..5).each do |i|
4
- csv_emit(i: i)
5
- end
6
-
7
- # OUTPUT
8
- # i
9
- # 1
10
- # 2
11
- # 3
data/test/recipes/noko.sinew DELETED
@@ -1,9 +0,0 @@
1
- get 'http://httpbin.org/xml'
2
- noko.css('slide title').each do |title|
3
- csv_emit(title: title.text)
4
- end
5
-
6
- # OUTPUT
7
- # title
8
- # Wake up to WonderWidgets!
9
- # Overview
data/test/recipes/uri.sinew DELETED
@@ -1,11 +0,0 @@
1
- # This tests get by URI, URI math, and csv_emit with uri
2
- get(URI.parse('http://httpbin.org/html'))
3
- csv_emit(url: uri)
4
-
5
- get(uri + '../get')
6
- csv_emit(url: uri)
7
-
8
- # OUTPUT
9
- # url
10
- # http://httpbin.org/html
11
- # http://httpbin.org/get
data/test/recipes/xml.sinew DELETED
@@ -1,8 +0,0 @@
1
- get 'http://httpbin.org/html'
2
- noko.css('h1').each do |h1|
3
- csv_emit(h1: h1.text)
4
- end
5
-
6
- # OUTPUT
7
- # h1
8
- # Herman Melville - Moby-Dick
data/test/test.html DELETED
@@ -1,45 +0,0 @@
1
- <html>
2
-
3
- <head>
4
- <title>Title</title>
5
- <script>
6
- alert("alert 1");
7
- alert("alert 2");
8
- </script>
9
- </head>
10
-
11
- <body>
12
- <div id="main">
13
- <span class="class1"> text1 </span>
14
- <span class="class2"> text2 </span>
15
-
16
- <!-- for test_normalize -->
17
- <div id="element"> text </div>
18
- <div class="e"> text1 </div>
19
- <div class="e"> text2 </div>
20
- </div>
21
-
22
- <div id="nokogiri_ext">
23
- <ul>
24
- <li>hello</li>
25
- <li>world</li>
26
- </ul>
27
- <div>
28
- a
29
- <p>b
30
- <span>c</span>
31
- </p>
32
- <p>b
33
- <span>c</span>
34
- </p>
35
- </div>
36
- </div>
37
-
38
- <div id="text_util">
39
- <!-- a comment that should be removed -->
40
- <div class="will_be_removed" />
41
- <a class="will_be_preserved" />
42
- </div>
43
- </body>
44
-
45
- </html>
data/test/test_cache.rb DELETED
@@ -1,69 +0,0 @@
1
- require_relative 'test_helper'
2
-
3
- class TestCache < MiniTest::Test
4
- def test_get
5
- 2.times do
6
- sinew.dsl.get('http://httpbin.org/get', c: 3, d: 4)
7
- end
8
- if !test_network?
9
- assert_requested :get, 'http://httpbin.org/get?c=3&d=4', times: 1
10
- end
11
- assert_equal 1, sinew.request_count
12
- assert_equal({ c: '3', d: '4' }, sinew.dsl.json[:args])
13
- assert File.exist?("#{TMP}/httpbin.org/get,c=3,d=4")
14
- assert !File.exist?("#{TMP}/httpbin.org/head/get,c=3,d=4")
15
- end
16
-
17
- def test_post
18
- 2.times do
19
- sinew.dsl.post('http://httpbin.org/post', c: 5, d: 6)
20
- end
21
- if !test_network?
22
- assert_requested :post, 'http://httpbin.org/post', times: 1
23
- end
24
- assert_equal 1, sinew.request_count
25
- assert_equal({ c: '5', d: '6' }, sinew.dsl.json[:form])
26
- end
27
-
28
- def test_redirect
29
- 2.times do
30
- sinew.dsl.get('http://httpbin.org/redirect/2')
31
- end
32
- if !test_network?
33
- assert_requested :get, 'http://httpbin.org/redirect/2', times: 1
34
- assert_requested :get, 'http://httpbin.org/redirect/1', times: 1
35
- assert_requested :get, 'http://httpbin.org/get', times: 1
36
- end
37
- assert_equal 1, sinew.request_count
38
- assert_equal 'http://httpbin.org/get', sinew.dsl.url
39
- end
40
-
41
- def test_error
42
- # gotta set this or the retries mess up our request counts
43
- sinew.runtime_options.retries = 0
44
- assert_output(/failed with 500/) do
45
- 2.times do
46
- sinew.dsl.get('http://httpbin.org/status/500')
47
- end
48
- end
49
- if !test_network?
50
- assert_requested :get, 'http://httpbin.org/status/500', times: 1
51
- assert_equal '500', sinew.dsl.raw
52
- end
53
- assert_equal 1, sinew.request_count
54
- end
55
-
56
- def test_timeout
57
- return if test_network?
58
-
59
- # gotta set this or the retries mess up our request counts
60
- sinew.runtime_options.retries = 0
61
- assert_output(/failed with 999/) do
62
- 2.times do
63
- sinew.dsl.get('http://httpbin.org/delay/1')
64
- end
65
- end
66
- assert_requested :get, 'http://httpbin.org/delay/1', times: 1
67
- assert_equal 'timeout', sinew.dsl.raw
68
- end
69
- end
data/test/test_helper.rb DELETED
@@ -1,123 +0,0 @@
1
- require 'minitest/autorun'
2
- require 'minitest/pride'
3
- require 'webmock/minitest' unless ENV['SINEW_TEST_NETWORK']
4
-
5
- # a hint to sinew, so that it'll do things like set rate limit to zero
6
- ENV['SINEW_TEST'] = '1'
7
-
8
- # Normally the Rakefile takes care of this, but it's handy to have it here when
9
- # running tests individually.
10
- $LOAD_PATH.unshift("#{__dir__}/../lib")
11
- require 'sinew'
12
-
13
- class MiniTest::Test
14
- TMP = '/tmp/_test_sinew'.freeze
15
- HTML = File.read("#{__dir__}/test.html")
16
-
17
- def setup
18
- super
19
-
20
- # prepare TMP
21
- FileUtils.rm_rf(TMP)
22
- FileUtils.mkdir_p(TMP)
23
-
24
- stub_network unless test_network?
25
- end
26
-
27
- def sinew
28
- @sinew ||= Sinew::Main.new(cache: TMP, quiet: true, recipe: "#{TMP}/ignore.sinew")
29
- end
30
- protected :sinew
31
-
32
- def test_network?
33
- !!ENV['SINEW_TEST_NETWORK']
34
- end
35
- protected :test_network?
36
-
37
- # mock requests, patterned on httpbin
38
- def stub_network
39
- stub_request(:get, %r{http://[^/]+/html}).to_return(method(:respond_html))
40
- stub_request(:get, %r{http://[^/]+/get\b}).to_return(method(:respond_echo))
41
- stub_request(:post, %r{http://[^/]+/post\b}).to_return(method(:respond_echo))
42
- stub_request(:get, %r{http://[^/]+/status/\d+}).to_return(method(:respond_status))
43
- stub_request(:get, %r{http://[^/]+/(relative-)?redirect/\d+}).to_return(method(:respond_redirect))
44
- stub_request(:get, %r{http://[^/]+/delay/\d+}).to_timeout
45
- stub_request(:get, %r{http://[^/]+/xml}).to_return(method(:respond_xml))
46
- end
47
- protected :stub_network
48
-
49
- #
50
- # respond_xxx helpers
51
- #
52
-
53
- def respond_html(_request)
54
- # this html was carefully chosen to somewhat match httpbin.org/html
55
- html = <<~EOF
56
- <body>
57
- <h1>Herman Melville - Moby-Dick</h1>
58
- </body>
59
- EOF
60
- { body: html }
61
- end
62
- protected :respond_html
63
-
64
- def respond_xml(_request)
65
- # this xml was carefully chosen to somewhat match httpbin.org/xml
66
- xml = <<~EOF
67
- <!-- A SAMPLE set of slides -->
68
- <slideshow>
69
- <slide type="all">
70
- <title>Wake up to WonderWidgets!</title>
71
- </slide>
72
- <slide type="all">
73
- <title>Overview</title>
74
- </slide>
75
- </slideshow>
76
- EOF
77
- { body: xml }
78
- end
79
- protected :respond_xml
80
-
81
- def respond_echo(request)
82
- response = {}
83
- response[:headers] = request.headers
84
-
85
- # args
86
- response[:args] = if request.uri.query
87
- CGI.parse(request.uri.query).map { |k, v| [ k, v.first ] }.to_h
88
- else
89
- {}
90
- end
91
-
92
- # form
93
- if request.headers['Content-Type'] == 'application/x-www-form-urlencoded'
94
- response[:form] = CGI.parse(request.body).map { |k, v| [ k, v.first ] }.to_h
95
- end
96
-
97
- # json
98
- if request.headers['Content-Type'] == 'application/json'
99
- response[:json] = JSON.parse(request.body)
100
- end
101
-
102
- {
103
- headers: { 'Content-Type' => 'application/json' },
104
- body: response.to_json,
105
- }
106
- end
107
- protected :respond_echo
108
-
109
- def respond_status(request)
110
- status = request.uri.to_s.split('/').last.to_i
111
- { body: status.to_s, status: status }
112
- end
113
- protected :respond_status
114
-
115
- def respond_redirect(request)
116
- parts = request.uri.to_s.split('/')
117
- path, count = parts[-2], parts[-1].to_i
118
- url = count == 1 ? '/get' : "/#{path}/#{count - 1}"
119
- url = "http://example#{url}" if path =~ /absolute/
120
- { status: 302, headers: { 'Location' => url } }
121
- end
122
- protected :respond_redirect
123
- end
data/test/test_legacy.rb DELETED
@@ -1,23 +0,0 @@
1
- require_relative 'test_helper'
2
-
3
- class TestLegacy < MiniTest::Test
4
- def setup
5
- super
6
-
7
- # These are legacy cache files, pulled from an older version of sinew. We
8
- # use them to test our legacy head parsing.
9
- src = 'legacy/eu.httpbin.org'
10
- dst = "#{TMP}/eu.httpbin.org"
11
- FileUtils.cp_r(File.expand_path(src, __dir__), dst)
12
- end
13
-
14
- def test_legacy
15
- assert_output(/failed with 999/) do
16
- sinew.dsl.get('http://eu.httpbin.org/status/500')
17
- assert_equal "\n", sinew.dsl.raw
18
- end
19
-
20
- sinew.dsl.get('http://eu.httpbin.org/redirect/3')
21
- assert_equal 'http://eu.httpbin.org/get', sinew.dsl.url
22
- end
23
- end
data/test/test_main.rb DELETED
@@ -1,34 +0,0 @@
1
- require_relative 'test_helper'
2
-
3
- require 'base64'
4
-
5
- class TestMain < MiniTest::Test
6
- def test_rate_limit
7
- # true network requests call sleep for timeouts, which interferes with our
8
- # instrumentation of Kernel#sleep
9
- skip if test_network?
10
-
11
- slept = false
12
-
13
- # change Kernel#sleep to not really sleep!
14
- Kernel.send(:alias_method, :old_sleep, :sleep)
15
- Kernel.send(:define_method, :sleep) do |_duration|
16
- slept = true
17
- end
18
-
19
- sinew.runtime_options.rate_limit = 1
20
- sinew.dsl.get('http://httpbin.org/html')
21
- sinew.dsl.get('http://httpbin.org/get')
22
- assert(slept)
23
-
24
- # restore old Kernel#sleep
25
- Kernel.send(:alias_method, :sleep, :old_sleep)
26
- Kernel.send(:undef_method, :old_sleep)
27
- end
28
-
29
- def test_gunzip
30
- body = Base64.decode64('H4sICBRI61oAA2d1Yi50eHQASy9N4gIAJlqRYgQAAAA=')
31
- body = Sinew::Response.process_body(OpenStruct.new(body: body))
32
- assert_equal 'gub', body.strip
33
- end
34
- end