sinew 2.0.1 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +26 -0
- data/.rubocop.yml +9 -6
- data/.vscode/settings.json +0 -10
- data/Gemfile +9 -0
- data/LICENSE +1 -1
- data/README.md +77 -58
- data/Rakefile +33 -18
- data/bin/sinew +8 -4
- data/lib/sinew.rb +0 -1
- data/lib/sinew/connection.rb +52 -0
- data/lib/sinew/connection/log_formatter.rb +22 -0
- data/lib/sinew/connection/rate_limit.rb +29 -0
- data/lib/sinew/core_ext.rb +1 -1
- data/lib/sinew/dsl.rb +27 -10
- data/lib/sinew/main.rb +7 -54
- data/lib/sinew/output.rb +26 -19
- data/lib/sinew/request.rb +28 -49
- data/lib/sinew/response.rb +25 -55
- data/lib/sinew/runtime_options.rb +4 -2
- data/lib/sinew/version.rb +1 -1
- data/sample.sinew +2 -2
- data/sinew.gemspec +16 -17
- metadata +41 -81
- data/.travis.yml +0 -4
- data/lib/sinew/cache.rb +0 -79
- data/test/legacy/eu.httpbin.org/head/redirect,3 +0 -51
- data/test/legacy/eu.httpbin.org/head/status,500 +0 -1
- data/test/legacy/eu.httpbin.org/redirect,3 +0 -11
- data/test/legacy/eu.httpbin.org/status,500 +0 -1
- data/test/legacy/legacy.sinew +0 -2
- data/test/test.html +0 -45
- data/test/test_cache.rb +0 -69
- data/test/test_helper.rb +0 -113
- data/test/test_legacy.rb +0 -21
- data/test/test_main.rb +0 -46
- data/test/test_nokogiri_ext.rb +0 -18
- data/test/test_output.rb +0 -73
- data/test/test_requests.rb +0 -135
- data/test/test_utf8.rb +0 -39
data/test/test_nokogiri_ext.rb
DELETED
@@ -1,18 +0,0 @@
|
|
1
|
-
require_relative 'test_helper'
|
2
|
-
|
3
|
-
class TestNokogiriExt < MiniTest::Test
|
4
|
-
def test_inner_text
|
5
|
-
assert_equal('hello world', noko.css('li').inner_text)
|
6
|
-
assert_equal('<li>hello</li> <li>world</li>', noko.css('ul').inner_html.squish)
|
7
|
-
end
|
8
|
-
|
9
|
-
def test_just_me
|
10
|
-
assert_equal('a', noko.css('div').text_just_me.squish)
|
11
|
-
assert_equal('b b', noko.css('p').text_just_me.squish)
|
12
|
-
end
|
13
|
-
|
14
|
-
def noko
|
15
|
-
@noko ||= Nokogiri::HTML(HTML).css('#nokogiri_ext')
|
16
|
-
end
|
17
|
-
protected :noko
|
18
|
-
end
|
data/test/test_output.rb
DELETED
@@ -1,73 +0,0 @@
|
|
1
|
-
require_relative 'test_helper'
|
2
|
-
|
3
|
-
class TestOutput < MiniTest::Test
|
4
|
-
def test_output
|
5
|
-
sinew.dsl.csv_header(:n, :a, :p)
|
6
|
-
sinew.dsl.csv_emit(n: 'n1', a: 'a1')
|
7
|
-
sinew.dsl.csv_emit(n: 'n2', a: 'a2')
|
8
|
-
assert_equal 2, sinew.output.count
|
9
|
-
assert_equal "n,a,p\nn1,a1,\"\"\nn2,a2,\"\"\n", File.read(CSV)
|
10
|
-
end
|
11
|
-
|
12
|
-
def test_implicit_header
|
13
|
-
sinew.dsl.csv_emit(name: 'bob', address: 'main')
|
14
|
-
assert_equal "name,address\nbob,main\n", File.read(CSV)
|
15
|
-
end
|
16
|
-
|
17
|
-
def test_array_header
|
18
|
-
sinew.dsl.csv_header(%i[n a p])
|
19
|
-
sinew.dsl.csv_emit(n: 'n1', a: 'a1')
|
20
|
-
assert_equal "n,a,p\nn1,a1,\"\"\n", File.read(CSV)
|
21
|
-
end
|
22
|
-
|
23
|
-
def test_filenames
|
24
|
-
sinew = Sinew::Main.new(recipe: 'gub.sinew')
|
25
|
-
assert_equal 'gub.csv', sinew.output.filename
|
26
|
-
sinew = Sinew::Main.new(recipe: 'gub')
|
27
|
-
assert_equal 'gub.csv', sinew.output.filename
|
28
|
-
sinew = Sinew::Main.new(recipe: '/somewhere/gub.sinew')
|
29
|
-
assert_equal '/somewhere/gub.csv', sinew.output.filename
|
30
|
-
end
|
31
|
-
|
32
|
-
def test_normalization
|
33
|
-
output = Sinew::Output.new(nil)
|
34
|
-
|
35
|
-
#
|
36
|
-
# simple types
|
37
|
-
#
|
38
|
-
|
39
|
-
assert_equal '', output.send(:normalize, nil)
|
40
|
-
assert_equal '', output.send(:normalize, '')
|
41
|
-
assert_equal 'text', output.send(:normalize, 'text')
|
42
|
-
assert_equal '123', output.send(:normalize, 123)
|
43
|
-
assert_equal('1|2', output.send(:normalize, [ 1, 2 ]))
|
44
|
-
|
45
|
-
#
|
46
|
-
# nokogiri
|
47
|
-
#
|
48
|
-
|
49
|
-
noko = Nokogiri::HTML(HTML)
|
50
|
-
|
51
|
-
# node => text
|
52
|
-
assert_equal('text', output.send(:normalize, noko.css('#element')))
|
53
|
-
# nodes => text joined with space
|
54
|
-
assert_equal('text1 text2', output.send(:normalize, noko.css('.e')))
|
55
|
-
|
56
|
-
#
|
57
|
-
# string cleanups
|
58
|
-
#
|
59
|
-
|
60
|
-
# strip_html_tags
|
61
|
-
assert_equal('gub', output.send(:normalize, '<tag>gub</tag>'))
|
62
|
-
# convert_smart_punctuation
|
63
|
-
assert_equal('"gub"', output.send(:normalize, "\302\223gub\302\224"))
|
64
|
-
# convert_accented_html_entities
|
65
|
-
assert_equal('a', output.send(:normalize, 'á'))
|
66
|
-
# convert_miscellaneous_html_entities
|
67
|
-
assert_equal('<>', output.send(:normalize, '<>'))
|
68
|
-
# to_ascii
|
69
|
-
assert_equal('cafe', output.send(:normalize, "caf\xc3\xa9"))
|
70
|
-
# squish
|
71
|
-
assert_equal('hello world', output.send(:normalize, "\nhello \t \rworld"))
|
72
|
-
end
|
73
|
-
end
|
data/test/test_requests.rb
DELETED
@@ -1,135 +0,0 @@
|
|
1
|
-
require_relative 'test_helper'
|
2
|
-
|
3
|
-
class TestRequests < MiniTest::Test
|
4
|
-
def test_user_agent
|
5
|
-
sinew.dsl.get('http://httpbin.org/get', a: 1, b: 2)
|
6
|
-
assert_match(/sinew/, sinew.dsl.json[:headers][:'User-Agent'])
|
7
|
-
end
|
8
|
-
|
9
|
-
def test_basic_methods
|
10
|
-
sinew.dsl.get('http://httpbin.org/get', a: 1, b: 2)
|
11
|
-
assert_equal({ a: '1', b: '2' }, sinew.dsl.json[:args])
|
12
|
-
|
13
|
-
sinew.dsl.post('http://httpbin.org/post', a: 1, b: 2)
|
14
|
-
assert_equal({ a: '1', b: '2' }, sinew.dsl.json[:form])
|
15
|
-
|
16
|
-
sinew.dsl.post_json('http://httpbin.org/post', a: 1, b: 2)
|
17
|
-
assert_equal({ a: 1, b: 2 }, sinew.dsl.json[:json])
|
18
|
-
end
|
19
|
-
|
20
|
-
def test_custom_headers
|
21
|
-
sinew.dsl.http('get', 'http://httpbin.org/get', headers: { "User-Agent": '007' })
|
22
|
-
assert_match(/007/, sinew.dsl.json[:headers][:'User-Agent'])
|
23
|
-
end
|
24
|
-
|
25
|
-
def test_redirects
|
26
|
-
# absolute redirect
|
27
|
-
sinew.dsl.get('http://httpbin.org/redirect/2')
|
28
|
-
assert_equal 'http://httpbin.org/get', sinew.dsl.url
|
29
|
-
|
30
|
-
# and relative redirect
|
31
|
-
sinew.dsl.get('http://httpbin.org/relative-redirect/2')
|
32
|
-
assert_equal 'http://httpbin.org/get', sinew.dsl.url
|
33
|
-
end
|
34
|
-
|
35
|
-
def test_errors
|
36
|
-
skip if test_network?
|
37
|
-
|
38
|
-
# 500
|
39
|
-
assert_output(/failed with 500/) do
|
40
|
-
sinew.dsl.get('http://httpbin.org/status/500')
|
41
|
-
assert_equal '500', sinew.dsl.raw
|
42
|
-
end
|
43
|
-
|
44
|
-
# timeout
|
45
|
-
assert_output(/failed with 999/) do
|
46
|
-
sinew.dsl.get('http://httpbin.org/delay/1')
|
47
|
-
assert_equal 'timeout', sinew.dsl.raw
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
def test_retry_timeout
|
52
|
-
skip if test_network?
|
53
|
-
|
54
|
-
errors = 2
|
55
|
-
stub_request(:get, %r{http://[^/]+/error}).to_return do
|
56
|
-
if errors > 0
|
57
|
-
errors -= 1
|
58
|
-
raise Timeout::Error
|
59
|
-
end
|
60
|
-
{ body: 'done', status: 200 }
|
61
|
-
end
|
62
|
-
sinew.dsl.get('http://httpbin.org/error')
|
63
|
-
assert_equal 0, errors
|
64
|
-
assert_equal 'done', sinew.dsl.raw
|
65
|
-
end
|
66
|
-
|
67
|
-
def test_retry_500
|
68
|
-
skip if test_network?
|
69
|
-
|
70
|
-
errors = 2
|
71
|
-
stub_request(:get, %r{http://[^/]+/error}).to_return do
|
72
|
-
if errors > 0
|
73
|
-
errors -= 1
|
74
|
-
return { status: 500 }
|
75
|
-
end
|
76
|
-
{ body: 'done', status: 200 }
|
77
|
-
end
|
78
|
-
sinew.dsl.get('http://httpbin.org/error')
|
79
|
-
assert_equal 0, errors
|
80
|
-
assert_equal 'done', sinew.dsl.raw
|
81
|
-
end
|
82
|
-
|
83
|
-
def test_cache_key
|
84
|
-
# empty
|
85
|
-
req = Sinew::Request.new(sinew, 'get', 'http://host')
|
86
|
-
assert_equal req.cache_key, 'host/_root_'
|
87
|
-
|
88
|
-
# path
|
89
|
-
req = Sinew::Request.new(sinew, 'get', 'http://host/path')
|
90
|
-
assert_equal req.cache_key, 'host/path'
|
91
|
-
|
92
|
-
# query
|
93
|
-
req = Sinew::Request.new(sinew, 'get', 'http://host/path', query: { a: 'b' })
|
94
|
-
assert_equal req.cache_key, 'host/path,a=b'
|
95
|
-
|
96
|
-
# post with body
|
97
|
-
req = Sinew::Request.new(sinew, 'post', 'http://host/path', body: { c: 'd' })
|
98
|
-
assert_equal req.cache_key, 'host/post,path,c=d'
|
99
|
-
|
100
|
-
# too long should turn into digest
|
101
|
-
path = 'xyz' * 123
|
102
|
-
req = Sinew::Request.new(sinew, 'get', "http://host/#{path}")
|
103
|
-
assert_equal "host/#{Digest::MD5.hexdigest(path)}", req.cache_key
|
104
|
-
end
|
105
|
-
|
106
|
-
def test_before_generate_cache_key
|
107
|
-
sinew.runtime_options.before_generate_cache_key = method(:redact_cache_key)
|
108
|
-
req = Sinew::Request.new(sinew, 'get', 'http://host', query: { secret: 'xyz' })
|
109
|
-
assert_equal 'host/secret=redacted', req.cache_key
|
110
|
-
end
|
111
|
-
|
112
|
-
def test_urls
|
113
|
-
# simple
|
114
|
-
req = Sinew::Request.new(sinew, 'get', 'https://host')
|
115
|
-
assert_equal 'https://host', req.uri.to_s
|
116
|
-
|
117
|
-
# with query
|
118
|
-
req = Sinew::Request.new(sinew, 'get', 'https://host', query: { a: 1 })
|
119
|
-
assert_equal 'https://host?a=1', req.uri.to_s
|
120
|
-
|
121
|
-
# entity decoding
|
122
|
-
req = Sinew::Request.new(sinew, 'get', 'https://host?a=<5')
|
123
|
-
assert_equal 'https://host?a=%3C5', req.uri.to_s
|
124
|
-
|
125
|
-
# sloppy urls
|
126
|
-
req = Sinew::Request.new(sinew, 'get', 'https://host?a=b c&d=f\'g')
|
127
|
-
assert_equal 'https://host?a=b%20c&d=f%27g', req.uri.to_s
|
128
|
-
end
|
129
|
-
|
130
|
-
def redact_cache_key(key)
|
131
|
-
key[:query].gsub!(/secret=[^&]+/, 'secret=redacted')
|
132
|
-
key
|
133
|
-
end
|
134
|
-
protected :redact_cache_key
|
135
|
-
end
|
data/test/test_utf8.rb
DELETED
@@ -1,39 +0,0 @@
|
|
1
|
-
require_relative 'test_helper'
|
2
|
-
|
3
|
-
class TestRequests < MiniTest::Test
|
4
|
-
def test_get
|
5
|
-
# network (or stub)
|
6
|
-
sinew.dsl.get('http://httpbin.org/get')
|
7
|
-
assert_equal 'UTF-8', sinew.dsl.raw.encoding.name
|
8
|
-
|
9
|
-
# disk
|
10
|
-
sinew.dsl.get('http://httpbin.org/get')
|
11
|
-
assert_equal 'UTF-8', sinew.dsl.raw.encoding.name
|
12
|
-
end
|
13
|
-
|
14
|
-
def test_utf8
|
15
|
-
skip if !test_network?
|
16
|
-
|
17
|
-
# network
|
18
|
-
sinew.dsl.get('http://httpbin.org/encoding/utf8')
|
19
|
-
assert_equal 'UTF-8', sinew.dsl.raw.encoding.name
|
20
|
-
assert_match(/∑/, sinew.dsl.raw)
|
21
|
-
|
22
|
-
# disk
|
23
|
-
sinew.dsl.get('http://httpbin.org/encoding/utf8')
|
24
|
-
assert_equal 'UTF-8', sinew.dsl.raw.encoding.name
|
25
|
-
assert_match(/∑/, sinew.dsl.raw)
|
26
|
-
end
|
27
|
-
|
28
|
-
def test_encode
|
29
|
-
skip if !test_network?
|
30
|
-
|
31
|
-
# network
|
32
|
-
sinew.dsl.get('https://www.google.co.jp')
|
33
|
-
assert_equal 'UTF-8', sinew.dsl.raw.encoding.name
|
34
|
-
|
35
|
-
# disk
|
36
|
-
sinew.dsl.get('https://www.google.co.jp')
|
37
|
-
assert_equal 'UTF-8', sinew.dsl.raw.encoding.name
|
38
|
-
end
|
39
|
-
end
|