sinew 2.0.3 → 4.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (54) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/test.yml +26 -0
  3. data/.gitignore +3 -5
  4. data/.rubocop.yml +31 -46
  5. data/Gemfile +9 -0
  6. data/Gemfile.lock +124 -0
  7. data/README.md +146 -81
  8. data/Rakefile +36 -20
  9. data/bin/sinew +13 -39
  10. data/lib/sinew.rb +23 -10
  11. data/lib/sinew/args.rb +53 -0
  12. data/lib/sinew/base.rb +251 -0
  13. data/lib/sinew/csv.rb +89 -0
  14. data/lib/sinew/main.rb +45 -98
  15. data/lib/sinew/middleware/log_formatter.rb +23 -0
  16. data/lib/sinew/nokogiri_ext.rb +12 -21
  17. data/lib/sinew/response.rb +39 -99
  18. data/lib/sinew/version.rb +1 -1
  19. data/sample.rb +13 -0
  20. data/sample.sinew +4 -4
  21. data/sinew.gemspec +26 -25
  22. metadata +46 -108
  23. data/.travis.yml +0 -4
  24. data/.vscode/extensions.json +0 -3
  25. data/.vscode/settings.json +0 -15
  26. data/lib/sinew/cache.rb +0 -79
  27. data/lib/sinew/core_ext.rb +0 -59
  28. data/lib/sinew/dsl.rb +0 -114
  29. data/lib/sinew/output.rb +0 -149
  30. data/lib/sinew/request.rb +0 -151
  31. data/lib/sinew/runtime_options.rb +0 -28
  32. data/test/legacy/eu.httpbin.org/head/redirect,3 +0 -51
  33. data/test/legacy/eu.httpbin.org/head/status,500 +0 -1
  34. data/test/legacy/eu.httpbin.org/redirect,3 +0 -11
  35. data/test/legacy/eu.httpbin.org/status,500 +0 -1
  36. data/test/legacy/legacy.sinew +0 -2
  37. data/test/recipes/array_header.sinew +0 -6
  38. data/test/recipes/basic.sinew +0 -8
  39. data/test/recipes/dups.sinew +0 -7
  40. data/test/recipes/implicit_header.sinew +0 -5
  41. data/test/recipes/limit.sinew +0 -11
  42. data/test/recipes/noko.sinew +0 -9
  43. data/test/recipes/uri.sinew +0 -11
  44. data/test/recipes/xml.sinew +0 -8
  45. data/test/test.html +0 -45
  46. data/test/test_cache.rb +0 -69
  47. data/test/test_helper.rb +0 -123
  48. data/test/test_legacy.rb +0 -23
  49. data/test/test_main.rb +0 -34
  50. data/test/test_nokogiri_ext.rb +0 -18
  51. data/test/test_output.rb +0 -56
  52. data/test/test_recipes.rb +0 -60
  53. data/test/test_requests.rb +0 -135
  54. data/test/test_utf8.rb +0 -39
data/lib/sinew/runtime_options.rb DELETED
@@ -1,28 +0,0 @@
1
- #
2
- # Runtime options that sinew files can modify.
3
- #
4
-
5
- module Sinew
6
- class RuntimeOptions
7
- attr_accessor :retries
8
- attr_accessor :rate_limit
9
- attr_accessor :headers
10
- attr_accessor :httparty_options
11
- attr_accessor :before_generate_cache_key
12
-
13
- def initialize
14
- self.retries = 3
15
- self.rate_limit = 1
16
- self.headers = {
17
- 'User-Agent' => "sinew/#{VERSION}",
18
- }
19
- self.httparty_options = {}
20
- self.before_generate_cache_key = ->(i) { i }
21
-
22
- # for testing
23
- if ENV['SINEW_TEST']
24
- self.rate_limit = 0
25
- end
26
- end
27
- end
28
- end
data/test/legacy/eu.httpbin.org/head/redirect,3 DELETED
@@ -1,51 +0,0 @@
1
- HTTP/1.1 302 FOUND
2
- Connection: keep-alive
3
- Server: gunicorn/19.7.1
4
- Date: Wed, 02 May 2018 20:55:20 GMT
5
- Content-Type: text/html; charset=utf-8
6
- Content-Length: 247
7
- Location: /relative-redirect/2
8
- Access-Control-Allow-Origin: *
9
- Access-Control-Allow-Credentials: true
10
- X-Powered-By: Flask
11
- X-Processed-Time: 0
12
- Via: 1.1 vegur
13
-
14
- HTTP/1.1 302 FOUND
15
- Connection: keep-alive
16
- Server: gunicorn/19.7.1
17
- Date: Wed, 02 May 2018 20:55:20 GMT
18
- Content-Type: text/html; charset=utf-8
19
- Content-Length: 0
20
- Location: /relative-redirect/1
21
- Access-Control-Allow-Origin: *
22
- Access-Control-Allow-Credentials: true
23
- X-Powered-By: Flask
24
- X-Processed-Time: 0
25
- Via: 1.1 vegur
26
-
27
- HTTP/1.1 302 FOUND
28
- Connection: keep-alive
29
- Server: gunicorn/19.7.1
30
- Date: Wed, 02 May 2018 20:55:20 GMT
31
- Content-Type: text/html; charset=utf-8
32
- Content-Length: 0
33
- Location: /get
34
- Access-Control-Allow-Origin: *
35
- Access-Control-Allow-Credentials: true
36
- X-Powered-By: Flask
37
- X-Processed-Time: 0
38
- Via: 1.1 vegur
39
-
40
- HTTP/1.1 200 OK
41
- Connection: keep-alive
42
- Server: gunicorn/19.7.1
43
- Date: Wed, 02 May 2018 20:55:20 GMT
44
- Content-Type: application/json
45
- Access-Control-Allow-Origin: *
46
- Access-Control-Allow-Credentials: true
47
- X-Powered-By: Flask
48
- X-Processed-Time: 0
49
- Content-Length: 220
50
- Via: 1.1 vegur
51
-
data/test/legacy/eu.httpbin.org/head/status,500 DELETED
@@ -1 +0,0 @@
1
- CURLER_ERROR curl error (22)
data/test/legacy/eu.httpbin.org/redirect,3 DELETED
@@ -1,11 +0,0 @@
1
- {
2
- "args": {},
3
- "headers": {
4
- "Accept": "*/*",
5
- "Connection": "close",
6
- "Host": "eu.httpbin.org",
7
- "User-Agent": "sinew/1.0.4"
8
- },
9
- "origin": "104.36.46.249",
10
- "url": "http://eu.httpbin.org/get"
11
- }
data/test/legacy/eu.httpbin.org/status,500 DELETED
@@ -1 +0,0 @@
1
-
data/test/legacy/legacy.sinew DELETED
@@ -1,2 +0,0 @@
1
- get 'http://eu.httpbin.org/status/500'
2
- get 'http://eu.httpbin.org/redirect/3'
data/test/recipes/array_header.sinew DELETED
@@ -1,6 +0,0 @@
1
- csv_header(%i[n a p])
2
- csv_emit(n: 'n1', a: 'a1')
3
-
4
- # OUTPUT
5
- # n,a,p
6
- # n1,a1,""
data/test/recipes/basic.sinew DELETED
@@ -1,8 +0,0 @@
1
- get 'http://httpbin.org/html'
2
- raw.scan(/<h1>([^<]+)/) do
3
- csv_emit(h1: $1)
4
- end
5
-
6
- # OUTPUT
7
- # h1
8
- # Herman Melville - Moby-Dick
data/test/recipes/dups.sinew DELETED
@@ -1,7 +0,0 @@
1
- 5.times do
2
- csv_emit(url: 'https://gub')
3
- end
4
-
5
- # OUTPUT
6
- # url
7
- # https://gub
data/test/recipes/implicit_header.sinew DELETED
@@ -1,5 +0,0 @@
1
- csv_emit(name: 'bob', address: 'main')
2
-
3
- # OUTPUT
4
- # name,address
5
- # bob,main
data/test/recipes/limit.sinew DELETED
@@ -1,11 +0,0 @@
1
- # OPTIONS { limit: 3 }
2
-
3
- (1..5).each do |i|
4
- csv_emit(i: i)
5
- end
6
-
7
- # OUTPUT
8
- # i
9
- # 1
10
- # 2
11
- # 3
data/test/recipes/noko.sinew DELETED
@@ -1,9 +0,0 @@
1
- get 'http://httpbin.org/xml'
2
- noko.css('slide title').each do |title|
3
- csv_emit(title: title.text)
4
- end
5
-
6
- # OUTPUT
7
- # title
8
- # Wake up to WonderWidgets!
9
- # Overview
data/test/recipes/uri.sinew DELETED
@@ -1,11 +0,0 @@
1
- # This tests get by URI, URI math, and csv_emit with uri
2
- get(URI.parse('http://httpbin.org/html'))
3
- csv_emit(url: uri)
4
-
5
- get(uri + '../get')
6
- csv_emit(url: uri)
7
-
8
- # OUTPUT
9
- # url
10
- # http://httpbin.org/html
11
- # http://httpbin.org/get
data/test/recipes/xml.sinew DELETED
@@ -1,8 +0,0 @@
1
- get 'http://httpbin.org/html'
2
- noko.css('h1').each do |h1|
3
- csv_emit(h1: h1.text)
4
- end
5
-
6
- # OUTPUT
7
- # h1
8
- # Herman Melville - Moby-Dick
data/test/test.html DELETED
@@ -1,45 +0,0 @@
1
- <html>
2
-
3
- <head>
4
- <title>Title</title>
5
- <script>
6
- alert("alert 1");
7
- alert("alert 2");
8
- </script>
9
- </head>
10
-
11
- <body>
12
- <div id="main">
13
- <span class="class1"> text1 </span>
14
- <span class="class2"> text2 </span>
15
-
16
- <!-- for test_normalize -->
17
- <div id="element"> text </div>
18
- <div class="e"> text1 </div>
19
- <div class="e"> text2 </div>
20
- </div>
21
-
22
- <div id="nokogiri_ext">
23
- <ul>
24
- <li>hello</li>
25
- <li>world</li>
26
- </ul>
27
- <div>
28
- a
29
- <p>b
30
- <span>c</span>
31
- </p>
32
- <p>b
33
- <span>c</span>
34
- </p>
35
- </div>
36
- </div>
37
-
38
- <div id="text_util">
39
- <!-- a comment that should be removed -->
40
- <div class="will_be_removed" />
41
- <a class="will_be_preserved" />
42
- </div>
43
- </body>
44
-
45
- </html>
data/test/test_cache.rb DELETED
@@ -1,69 +0,0 @@
1
- require_relative 'test_helper'
2
-
3
- class TestCache < MiniTest::Test
4
- def test_get
5
- 2.times do
6
- sinew.dsl.get('http://httpbin.org/get', c: 3, d: 4)
7
- end
8
- if !test_network?
9
- assert_requested :get, 'http://httpbin.org/get?c=3&d=4', times: 1
10
- end
11
- assert_equal 1, sinew.request_count
12
- assert_equal({ c: '3', d: '4' }, sinew.dsl.json[:args])
13
- assert File.exist?("#{TMP}/httpbin.org/get,c=3,d=4")
14
- assert !File.exist?("#{TMP}/httpbin.org/head/get,c=3,d=4")
15
- end
16
-
17
- def test_post
18
- 2.times do
19
- sinew.dsl.post('http://httpbin.org/post', c: 5, d: 6)
20
- end
21
- if !test_network?
22
- assert_requested :post, 'http://httpbin.org/post', times: 1
23
- end
24
- assert_equal 1, sinew.request_count
25
- assert_equal({ c: '5', d: '6' }, sinew.dsl.json[:form])
26
- end
27
-
28
- def test_redirect
29
- 2.times do
30
- sinew.dsl.get('http://httpbin.org/redirect/2')
31
- end
32
- if !test_network?
33
- assert_requested :get, 'http://httpbin.org/redirect/2', times: 1
34
- assert_requested :get, 'http://httpbin.org/redirect/1', times: 1
35
- assert_requested :get, 'http://httpbin.org/get', times: 1
36
- end
37
- assert_equal 1, sinew.request_count
38
- assert_equal 'http://httpbin.org/get', sinew.dsl.url
39
- end
40
-
41
- def test_error
42
- # gotta set this or the retries mess up our request counts
43
- sinew.runtime_options.retries = 0
44
- assert_output(/failed with 500/) do
45
- 2.times do
46
- sinew.dsl.get('http://httpbin.org/status/500')
47
- end
48
- end
49
- if !test_network?
50
- assert_requested :get, 'http://httpbin.org/status/500', times: 1
51
- assert_equal '500', sinew.dsl.raw
52
- end
53
- assert_equal 1, sinew.request_count
54
- end
55
-
56
- def test_timeout
57
- return if test_network?
58
-
59
- # gotta set this or the retries mess up our request counts
60
- sinew.runtime_options.retries = 0
61
- assert_output(/failed with 999/) do
62
- 2.times do
63
- sinew.dsl.get('http://httpbin.org/delay/1')
64
- end
65
- end
66
- assert_requested :get, 'http://httpbin.org/delay/1', times: 1
67
- assert_equal 'timeout', sinew.dsl.raw
68
- end
69
- end
data/test/test_helper.rb DELETED
@@ -1,123 +0,0 @@
1
- require 'minitest/autorun'
2
- require 'minitest/pride'
3
- require 'webmock/minitest' unless ENV['SINEW_TEST_NETWORK']
4
-
5
- # a hint to sinew, so that it'll do things like set rate limit to zero
6
- ENV['SINEW_TEST'] = '1'
7
-
8
- # Normally the Rakefile takes care of this, but it's handy to have it here when
9
- # running tests individually.
10
- $LOAD_PATH.unshift("#{__dir__}/../lib")
11
- require 'sinew'
12
-
13
- class MiniTest::Test
14
- TMP = '/tmp/_test_sinew'.freeze
15
- HTML = File.read("#{__dir__}/test.html")
16
-
17
- def setup
18
- super
19
-
20
- # prepare TMP
21
- FileUtils.rm_rf(TMP)
22
- FileUtils.mkdir_p(TMP)
23
-
24
- stub_network unless test_network?
25
- end
26
-
27
- def sinew
28
- @sinew ||= Sinew::Main.new(cache: TMP, quiet: true, recipe: "#{TMP}/ignore.sinew")
29
- end
30
- protected :sinew
31
-
32
- def test_network?
33
- !!ENV['SINEW_TEST_NETWORK']
34
- end
35
- protected :test_network?
36
-
37
- # mock requests, patterned on httpbin
38
- def stub_network
39
- stub_request(:get, %r{http://[^/]+/html}).to_return(method(:respond_html))
40
- stub_request(:get, %r{http://[^/]+/get\b}).to_return(method(:respond_echo))
41
- stub_request(:post, %r{http://[^/]+/post\b}).to_return(method(:respond_echo))
42
- stub_request(:get, %r{http://[^/]+/status/\d+}).to_return(method(:respond_status))
43
- stub_request(:get, %r{http://[^/]+/(relative-)?redirect/\d+}).to_return(method(:respond_redirect))
44
- stub_request(:get, %r{http://[^/]+/delay/\d+}).to_timeout
45
- stub_request(:get, %r{http://[^/]+/xml}).to_return(method(:respond_xml))
46
- end
47
- protected :stub_network
48
-
49
- #
50
- # respond_xxx helpers
51
- #
52
-
53
- def respond_html(_request)
54
- # this html was carefully chosen to somewhat match httpbin.org/html
55
- html = <<~EOF
56
- <body>
57
- <h1>Herman Melville - Moby-Dick</h1>
58
- </body>
59
- EOF
60
- { body: html }
61
- end
62
- protected :respond_html
63
-
64
- def respond_xml(_request)
65
- # this xml was carefully chosen to somewhat match httpbin.org/xml
66
- xml = <<~EOF
67
- <!-- A SAMPLE set of slides -->
68
- <slideshow>
69
- <slide type="all">
70
- <title>Wake up to WonderWidgets!</title>
71
- </slide>
72
- <slide type="all">
73
- <title>Overview</title>
74
- </slide>
75
- </slideshow>
76
- EOF
77
- { body: xml }
78
- end
79
- protected :respond_xml
80
-
81
- def respond_echo(request)
82
- response = {}
83
- response[:headers] = request.headers
84
-
85
- # args
86
- response[:args] = if request.uri.query
87
- CGI.parse(request.uri.query).map { |k, v| [ k, v.first ] }.to_h
88
- else
89
- {}
90
- end
91
-
92
- # form
93
- if request.headers['Content-Type'] == 'application/x-www-form-urlencoded'
94
- response[:form] = CGI.parse(request.body).map { |k, v| [ k, v.first ] }.to_h
95
- end
96
-
97
- # json
98
- if request.headers['Content-Type'] == 'application/json'
99
- response[:json] = JSON.parse(request.body)
100
- end
101
-
102
- {
103
- headers: { 'Content-Type' => 'application/json' },
104
- body: response.to_json,
105
- }
106
- end
107
- protected :respond_echo
108
-
109
- def respond_status(request)
110
- status = request.uri.to_s.split('/').last.to_i
111
- { body: status.to_s, status: status }
112
- end
113
- protected :respond_status
114
-
115
- def respond_redirect(request)
116
- parts = request.uri.to_s.split('/')
117
- path, count = parts[-2], parts[-1].to_i
118
- url = count == 1 ? '/get' : "/#{path}/#{count - 1}"
119
- url = "http://example#{url}" if path =~ /absolute/
120
- { status: 302, headers: { 'Location' => url } }
121
- end
122
- protected :respond_redirect
123
- end
data/test/test_legacy.rb DELETED
@@ -1,23 +0,0 @@
1
- require_relative 'test_helper'
2
-
3
- class TestLegacy < MiniTest::Test
4
- def setup
5
- super
6
-
7
- # These are legacy cache files, pulled from an older version of sinew. We
8
- # use them to test our legacy head parsing.
9
- src = 'legacy/eu.httpbin.org'
10
- dst = "#{TMP}/eu.httpbin.org"
11
- FileUtils.cp_r(File.expand_path(src, __dir__), dst)
12
- end
13
-
14
- def test_legacy
15
- assert_output(/failed with 999/) do
16
- sinew.dsl.get('http://eu.httpbin.org/status/500')
17
- assert_equal "\n", sinew.dsl.raw
18
- end
19
-
20
- sinew.dsl.get('http://eu.httpbin.org/redirect/3')
21
- assert_equal 'http://eu.httpbin.org/get', sinew.dsl.url
22
- end
23
- end