sinew 2.0.1 → 3.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +26 -0
- data/.rubocop.yml +9 -6
- data/.vscode/settings.json +0 -10
- data/Gemfile +9 -0
- data/LICENSE +1 -1
- data/README.md +77 -58
- data/Rakefile +33 -18
- data/bin/sinew +8 -4
- data/lib/sinew.rb +0 -1
- data/lib/sinew/connection.rb +52 -0
- data/lib/sinew/connection/log_formatter.rb +22 -0
- data/lib/sinew/connection/rate_limit.rb +29 -0
- data/lib/sinew/core_ext.rb +1 -1
- data/lib/sinew/dsl.rb +27 -10
- data/lib/sinew/main.rb +7 -54
- data/lib/sinew/output.rb +26 -19
- data/lib/sinew/request.rb +28 -49
- data/lib/sinew/response.rb +25 -55
- data/lib/sinew/runtime_options.rb +4 -2
- data/lib/sinew/version.rb +1 -1
- data/sample.sinew +2 -2
- data/sinew.gemspec +16 -17
- metadata +41 -81
- data/.travis.yml +0 -4
- data/lib/sinew/cache.rb +0 -79
- data/test/legacy/eu.httpbin.org/head/redirect,3 +0 -51
- data/test/legacy/eu.httpbin.org/head/status,500 +0 -1
- data/test/legacy/eu.httpbin.org/redirect,3 +0 -11
- data/test/legacy/eu.httpbin.org/status,500 +0 -1
- data/test/legacy/legacy.sinew +0 -2
- data/test/test.html +0 -45
- data/test/test_cache.rb +0 -69
- data/test/test_helper.rb +0 -113
- data/test/test_legacy.rb +0 -21
- data/test/test_main.rb +0 -46
- data/test/test_nokogiri_ext.rb +0 -18
- data/test/test_output.rb +0 -73
- data/test/test_requests.rb +0 -135
- data/test/test_utf8.rb +0 -39
data/test/test_nokogiri_ext.rb
DELETED
@@ -1,18 +0,0 @@
|
|
1
|
-
require_relative 'test_helper'
|
2
|
-
|
3
|
-
class TestNokogiriExt < MiniTest::Test
|
4
|
-
def test_inner_text
|
5
|
-
assert_equal('hello world', noko.css('li').inner_text)
|
6
|
-
assert_equal('<li>hello</li> <li>world</li>', noko.css('ul').inner_html.squish)
|
7
|
-
end
|
8
|
-
|
9
|
-
def test_just_me
|
10
|
-
assert_equal('a', noko.css('div').text_just_me.squish)
|
11
|
-
assert_equal('b b', noko.css('p').text_just_me.squish)
|
12
|
-
end
|
13
|
-
|
14
|
-
def noko
|
15
|
-
@noko ||= Nokogiri::HTML(HTML).css('#nokogiri_ext')
|
16
|
-
end
|
17
|
-
protected :noko
|
18
|
-
end
|
data/test/test_output.rb
DELETED
@@ -1,73 +0,0 @@
|
|
1
|
-
require_relative 'test_helper'
|
2
|
-
|
3
|
-
class TestOutput < MiniTest::Test
|
4
|
-
def test_output
|
5
|
-
sinew.dsl.csv_header(:n, :a, :p)
|
6
|
-
sinew.dsl.csv_emit(n: 'n1', a: 'a1')
|
7
|
-
sinew.dsl.csv_emit(n: 'n2', a: 'a2')
|
8
|
-
assert_equal 2, sinew.output.count
|
9
|
-
assert_equal "n,a,p\nn1,a1,\"\"\nn2,a2,\"\"\n", File.read(CSV)
|
10
|
-
end
|
11
|
-
|
12
|
-
def test_implicit_header
|
13
|
-
sinew.dsl.csv_emit(name: 'bob', address: 'main')
|
14
|
-
assert_equal "name,address\nbob,main\n", File.read(CSV)
|
15
|
-
end
|
16
|
-
|
17
|
-
def test_array_header
|
18
|
-
sinew.dsl.csv_header(%i[n a p])
|
19
|
-
sinew.dsl.csv_emit(n: 'n1', a: 'a1')
|
20
|
-
assert_equal "n,a,p\nn1,a1,\"\"\n", File.read(CSV)
|
21
|
-
end
|
22
|
-
|
23
|
-
def test_filenames
|
24
|
-
sinew = Sinew::Main.new(recipe: 'gub.sinew')
|
25
|
-
assert_equal 'gub.csv', sinew.output.filename
|
26
|
-
sinew = Sinew::Main.new(recipe: 'gub')
|
27
|
-
assert_equal 'gub.csv', sinew.output.filename
|
28
|
-
sinew = Sinew::Main.new(recipe: '/somewhere/gub.sinew')
|
29
|
-
assert_equal '/somewhere/gub.csv', sinew.output.filename
|
30
|
-
end
|
31
|
-
|
32
|
-
def test_normalization
|
33
|
-
output = Sinew::Output.new(nil)
|
34
|
-
|
35
|
-
#
|
36
|
-
# simple types
|
37
|
-
#
|
38
|
-
|
39
|
-
assert_equal '', output.send(:normalize, nil)
|
40
|
-
assert_equal '', output.send(:normalize, '')
|
41
|
-
assert_equal 'text', output.send(:normalize, 'text')
|
42
|
-
assert_equal '123', output.send(:normalize, 123)
|
43
|
-
assert_equal('1|2', output.send(:normalize, [ 1, 2 ]))
|
44
|
-
|
45
|
-
#
|
46
|
-
# nokogiri
|
47
|
-
#
|
48
|
-
|
49
|
-
noko = Nokogiri::HTML(HTML)
|
50
|
-
|
51
|
-
# node => text
|
52
|
-
assert_equal('text', output.send(:normalize, noko.css('#element')))
|
53
|
-
# nodes => text joined with space
|
54
|
-
assert_equal('text1 text2', output.send(:normalize, noko.css('.e')))
|
55
|
-
|
56
|
-
#
|
57
|
-
# string cleanups
|
58
|
-
#
|
59
|
-
|
60
|
-
# strip_html_tags
|
61
|
-
assert_equal('gub', output.send(:normalize, '<tag>gub</tag>'))
|
62
|
-
# convert_smart_punctuation
|
63
|
-
assert_equal('"gub"', output.send(:normalize, "\302\223gub\302\224"))
|
64
|
-
# convert_accented_html_entities
|
65
|
-
assert_equal('a', output.send(:normalize, 'á'))
|
66
|
-
# convert_miscellaneous_html_entities
|
67
|
-
assert_equal('<>', output.send(:normalize, '<>'))
|
68
|
-
# to_ascii
|
69
|
-
assert_equal('cafe', output.send(:normalize, "caf\xc3\xa9"))
|
70
|
-
# squish
|
71
|
-
assert_equal('hello world', output.send(:normalize, "\nhello \t \rworld"))
|
72
|
-
end
|
73
|
-
end
|
data/test/test_requests.rb
DELETED
@@ -1,135 +0,0 @@
|
|
1
|
-
require_relative 'test_helper'
|
2
|
-
|
3
|
-
class TestRequests < MiniTest::Test
|
4
|
-
def test_user_agent
|
5
|
-
sinew.dsl.get('http://httpbin.org/get', a: 1, b: 2)
|
6
|
-
assert_match(/sinew/, sinew.dsl.json[:headers][:'User-Agent'])
|
7
|
-
end
|
8
|
-
|
9
|
-
def test_basic_methods
|
10
|
-
sinew.dsl.get('http://httpbin.org/get', a: 1, b: 2)
|
11
|
-
assert_equal({ a: '1', b: '2' }, sinew.dsl.json[:args])
|
12
|
-
|
13
|
-
sinew.dsl.post('http://httpbin.org/post', a: 1, b: 2)
|
14
|
-
assert_equal({ a: '1', b: '2' }, sinew.dsl.json[:form])
|
15
|
-
|
16
|
-
sinew.dsl.post_json('http://httpbin.org/post', a: 1, b: 2)
|
17
|
-
assert_equal({ a: 1, b: 2 }, sinew.dsl.json[:json])
|
18
|
-
end
|
19
|
-
|
20
|
-
def test_custom_headers
|
21
|
-
sinew.dsl.http('get', 'http://httpbin.org/get', headers: { "User-Agent": '007' })
|
22
|
-
assert_match(/007/, sinew.dsl.json[:headers][:'User-Agent'])
|
23
|
-
end
|
24
|
-
|
25
|
-
def test_redirects
|
26
|
-
# absolute redirect
|
27
|
-
sinew.dsl.get('http://httpbin.org/redirect/2')
|
28
|
-
assert_equal 'http://httpbin.org/get', sinew.dsl.url
|
29
|
-
|
30
|
-
# and relative redirect
|
31
|
-
sinew.dsl.get('http://httpbin.org/relative-redirect/2')
|
32
|
-
assert_equal 'http://httpbin.org/get', sinew.dsl.url
|
33
|
-
end
|
34
|
-
|
35
|
-
def test_errors
|
36
|
-
skip if test_network?
|
37
|
-
|
38
|
-
# 500
|
39
|
-
assert_output(/failed with 500/) do
|
40
|
-
sinew.dsl.get('http://httpbin.org/status/500')
|
41
|
-
assert_equal '500', sinew.dsl.raw
|
42
|
-
end
|
43
|
-
|
44
|
-
# timeout
|
45
|
-
assert_output(/failed with 999/) do
|
46
|
-
sinew.dsl.get('http://httpbin.org/delay/1')
|
47
|
-
assert_equal 'timeout', sinew.dsl.raw
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
def test_retry_timeout
|
52
|
-
skip if test_network?
|
53
|
-
|
54
|
-
errors = 2
|
55
|
-
stub_request(:get, %r{http://[^/]+/error}).to_return do
|
56
|
-
if errors > 0
|
57
|
-
errors -= 1
|
58
|
-
raise Timeout::Error
|
59
|
-
end
|
60
|
-
{ body: 'done', status: 200 }
|
61
|
-
end
|
62
|
-
sinew.dsl.get('http://httpbin.org/error')
|
63
|
-
assert_equal 0, errors
|
64
|
-
assert_equal 'done', sinew.dsl.raw
|
65
|
-
end
|
66
|
-
|
67
|
-
def test_retry_500
|
68
|
-
skip if test_network?
|
69
|
-
|
70
|
-
errors = 2
|
71
|
-
stub_request(:get, %r{http://[^/]+/error}).to_return do
|
72
|
-
if errors > 0
|
73
|
-
errors -= 1
|
74
|
-
return { status: 500 }
|
75
|
-
end
|
76
|
-
{ body: 'done', status: 200 }
|
77
|
-
end
|
78
|
-
sinew.dsl.get('http://httpbin.org/error')
|
79
|
-
assert_equal 0, errors
|
80
|
-
assert_equal 'done', sinew.dsl.raw
|
81
|
-
end
|
82
|
-
|
83
|
-
def test_cache_key
|
84
|
-
# empty
|
85
|
-
req = Sinew::Request.new(sinew, 'get', 'http://host')
|
86
|
-
assert_equal req.cache_key, 'host/_root_'
|
87
|
-
|
88
|
-
# path
|
89
|
-
req = Sinew::Request.new(sinew, 'get', 'http://host/path')
|
90
|
-
assert_equal req.cache_key, 'host/path'
|
91
|
-
|
92
|
-
# query
|
93
|
-
req = Sinew::Request.new(sinew, 'get', 'http://host/path', query: { a: 'b' })
|
94
|
-
assert_equal req.cache_key, 'host/path,a=b'
|
95
|
-
|
96
|
-
# post with body
|
97
|
-
req = Sinew::Request.new(sinew, 'post', 'http://host/path', body: { c: 'd' })
|
98
|
-
assert_equal req.cache_key, 'host/post,path,c=d'
|
99
|
-
|
100
|
-
# too long should turn into digest
|
101
|
-
path = 'xyz' * 123
|
102
|
-
req = Sinew::Request.new(sinew, 'get', "http://host/#{path}")
|
103
|
-
assert_equal "host/#{Digest::MD5.hexdigest(path)}", req.cache_key
|
104
|
-
end
|
105
|
-
|
106
|
-
def test_before_generate_cache_key
|
107
|
-
sinew.runtime_options.before_generate_cache_key = method(:redact_cache_key)
|
108
|
-
req = Sinew::Request.new(sinew, 'get', 'http://host', query: { secret: 'xyz' })
|
109
|
-
assert_equal 'host/secret=redacted', req.cache_key
|
110
|
-
end
|
111
|
-
|
112
|
-
def test_urls
|
113
|
-
# simple
|
114
|
-
req = Sinew::Request.new(sinew, 'get', 'https://host')
|
115
|
-
assert_equal 'https://host', req.uri.to_s
|
116
|
-
|
117
|
-
# with query
|
118
|
-
req = Sinew::Request.new(sinew, 'get', 'https://host', query: { a: 1 })
|
119
|
-
assert_equal 'https://host?a=1', req.uri.to_s
|
120
|
-
|
121
|
-
# entity decoding
|
122
|
-
req = Sinew::Request.new(sinew, 'get', 'https://host?a=<5')
|
123
|
-
assert_equal 'https://host?a=%3C5', req.uri.to_s
|
124
|
-
|
125
|
-
# sloppy urls
|
126
|
-
req = Sinew::Request.new(sinew, 'get', 'https://host?a=b c&d=f\'g')
|
127
|
-
assert_equal 'https://host?a=b%20c&d=f%27g', req.uri.to_s
|
128
|
-
end
|
129
|
-
|
130
|
-
def redact_cache_key(key)
|
131
|
-
key[:query].gsub!(/secret=[^&]+/, 'secret=redacted')
|
132
|
-
key
|
133
|
-
end
|
134
|
-
protected :redact_cache_key
|
135
|
-
end
|
data/test/test_utf8.rb
DELETED
@@ -1,39 +0,0 @@
|
|
1
|
-
require_relative 'test_helper'
|
2
|
-
|
3
|
-
class TestRequests < MiniTest::Test
|
4
|
-
def test_get
|
5
|
-
# network (or stub)
|
6
|
-
sinew.dsl.get('http://httpbin.org/get')
|
7
|
-
assert_equal 'UTF-8', sinew.dsl.raw.encoding.name
|
8
|
-
|
9
|
-
# disk
|
10
|
-
sinew.dsl.get('http://httpbin.org/get')
|
11
|
-
assert_equal 'UTF-8', sinew.dsl.raw.encoding.name
|
12
|
-
end
|
13
|
-
|
14
|
-
def test_utf8
|
15
|
-
skip if !test_network?
|
16
|
-
|
17
|
-
# network
|
18
|
-
sinew.dsl.get('http://httpbin.org/encoding/utf8')
|
19
|
-
assert_equal 'UTF-8', sinew.dsl.raw.encoding.name
|
20
|
-
assert_match(/∑/, sinew.dsl.raw)
|
21
|
-
|
22
|
-
# disk
|
23
|
-
sinew.dsl.get('http://httpbin.org/encoding/utf8')
|
24
|
-
assert_equal 'UTF-8', sinew.dsl.raw.encoding.name
|
25
|
-
assert_match(/∑/, sinew.dsl.raw)
|
26
|
-
end
|
27
|
-
|
28
|
-
def test_encode
|
29
|
-
skip if !test_network?
|
30
|
-
|
31
|
-
# network
|
32
|
-
sinew.dsl.get('https://www.google.co.jp')
|
33
|
-
assert_equal 'UTF-8', sinew.dsl.raw.encoding.name
|
34
|
-
|
35
|
-
# disk
|
36
|
-
sinew.dsl.get('https://www.google.co.jp')
|
37
|
-
assert_equal 'UTF-8', sinew.dsl.raw.encoding.name
|
38
|
-
end
|
39
|
-
end
|