raindrops 0.4.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. data/.document +2 -1
  2. data/.gitignore +4 -0
  3. data/.wrongdoc.yml +4 -0
  4. data/GIT-VERSION-GEN +1 -1
  5. data/GNUmakefile +2 -196
  6. data/Gemfile +7 -0
  7. data/LICENSE +1 -1
  8. data/README +17 -47
  9. data/Rakefile +0 -104
  10. data/examples/linux-listener-stats.rb +123 -0
  11. data/examples/{config.ru → middleware.ru} +1 -1
  12. data/examples/watcher.ru +4 -0
  13. data/examples/watcher_demo.ru +13 -0
  14. data/examples/zbatery.conf.rb +13 -0
  15. data/ext/raindrops/extconf.rb +5 -0
  16. data/ext/raindrops/linux_inet_diag.c +449 -151
  17. data/ext/raindrops/linux_tcp_info.c +170 -0
  18. data/ext/raindrops/my_fileno.h +36 -0
  19. data/ext/raindrops/raindrops.c +232 -20
  20. data/lib/raindrops.rb +20 -7
  21. data/lib/raindrops/aggregate.rb +8 -0
  22. data/lib/raindrops/aggregate/last_data_recv.rb +86 -0
  23. data/lib/raindrops/aggregate/pmq.rb +239 -0
  24. data/lib/raindrops/last_data_recv.rb +100 -0
  25. data/lib/raindrops/linux.rb +26 -16
  26. data/lib/raindrops/middleware.rb +112 -41
  27. data/lib/raindrops/middleware/proxy.rb +34 -0
  28. data/lib/raindrops/struct.rb +15 -0
  29. data/lib/raindrops/watcher.rb +362 -0
  30. data/pkg.mk +171 -0
  31. data/raindrops.gemspec +10 -20
  32. data/test/ipv6_enabled.rb +10 -0
  33. data/test/rack_unicorn.rb +12 -0
  34. data/test/test_aggregate_pmq.rb +65 -0
  35. data/test/test_inet_diag_socket.rb +13 -0
  36. data/test/test_last_data_recv_unicorn.rb +69 -0
  37. data/test/test_linux.rb +55 -57
  38. data/test/test_linux_all_tcp_listen_stats.rb +66 -0
  39. data/test/test_linux_all_tcp_listen_stats_leak.rb +43 -0
  40. data/test/test_linux_ipv6.rb +158 -0
  41. data/test/test_linux_tcp_info.rb +61 -0
  42. data/test/test_middleware.rb +15 -2
  43. data/test/test_middleware_unicorn.rb +37 -0
  44. data/test/test_middleware_unicorn_ipv6.rb +37 -0
  45. data/test/test_raindrops.rb +65 -1
  46. data/test/test_raindrops_gc.rb +23 -1
  47. data/test/test_watcher.rb +85 -0
  48. metadata +69 -22
  49. data/examples/linux-tcp-listener-stats.rb +0 -44
@@ -0,0 +1,66 @@
1
+ # -*- encoding: binary -*-
2
+ require 'test/unit'
3
+ require 'socket'
4
+ require 'raindrops'
5
+ require 'pp'
6
+ $stderr.sync = $stdout.sync = true
7
+
8
+ class TestLinuxAllTcpListenStats < Test::Unit::TestCase
9
+ include Raindrops::Linux
10
+ TEST_ADDR = ENV['UNICORN_TEST_ADDR'] || '127.0.0.1'
11
+
12
+ def test_print_all
13
+ puts "EVERYTHING"
14
+ pp Raindrops::Linux.tcp_listener_stats
15
+ puts("-" * 72)
16
+ end if $stdout.tty?
17
+
18
+ def setup
19
+ @socks = []
20
+ end
21
+
22
+ def teardown
23
+ @socks.each { |io| io.closed? or io.close }
24
+ end
25
+
26
+ def new_server
27
+ s = TCPServer.new TEST_ADDR, 0
28
+ @socks << s
29
+ [ s, s.addr[1] ]
30
+ end
31
+
32
+ def new_client(port)
33
+ s = TCPSocket.new("127.0.0.1", port)
34
+ @socks << s
35
+ s
36
+ end
37
+
38
+ def new_accept(srv)
39
+ c = srv.accept
40
+ @socks << c
41
+ c
42
+ end
43
+
44
+ def test_all_ports
45
+ srv, port = new_server
46
+ addr = "#{TEST_ADDR}:#{port}"
47
+ all = Raindrops::Linux.tcp_listener_stats
48
+ assert_equal [0,0], all[addr].to_a
49
+
50
+ new_client(port)
51
+ all = Raindrops::Linux.tcp_listener_stats
52
+ assert_equal [0,1], all[addr].to_a
53
+
54
+ new_client(port)
55
+ all = Raindrops::Linux.tcp_listener_stats
56
+ assert_equal [0,2], all[addr].to_a
57
+
58
+ new_accept(srv)
59
+ all = Raindrops::Linux.tcp_listener_stats
60
+ assert_equal [1,1], all[addr].to_a
61
+
62
+ new_accept(srv)
63
+ all = Raindrops::Linux.tcp_listener_stats
64
+ assert_equal [2,0], all[addr].to_a
65
+ end
66
+ end if RUBY_PLATFORM =~ /linux/
@@ -0,0 +1,43 @@
1
+ # -*- encoding: binary -*-
2
+ require 'test/unit'
3
+ require 'raindrops'
4
+ require 'socket'
5
+ require 'benchmark'
6
+ $stderr.sync = $stdout.sync = true
7
+
8
+ class TestLinuxAllTcpListenStatsLeak < Test::Unit::TestCase
9
+
10
+ TEST_ADDR = ENV['UNICORN_TEST_ADDR'] || '127.0.0.1'
11
+
12
+
13
+ def rss_kb
14
+ File.readlines("/proc/#$$/status").grep(/VmRSS:/)[0].split(/\s+/)[1].to_i
15
+ end
16
+ def test_leak
17
+ s = TCPServer.new(TEST_ADDR, 0)
18
+ start_kb = rss_kb
19
+ p [ :start_kb, start_kb ]
20
+ assert_nothing_raised do
21
+ p(Benchmark.measure {
22
+ 1000.times { Raindrops::Linux.all_tcp_listener_stats }
23
+ })
24
+ end
25
+ cur_kb = rss_kb
26
+ p [ :cur_kb, cur_kb ]
27
+ now = Time.now.to_i
28
+ fin = now + 60
29
+ assert_nothing_raised do
30
+ 1000000000.times { |i|
31
+ if (i % 1024) == 0
32
+ now = Time.now.to_i
33
+ break if now > fin
34
+ end
35
+ Raindrops::Linux.all_tcp_listener_stats
36
+ }
37
+ end
38
+ cur_kb = rss_kb
39
+ p [ :cur_kb, cur_kb ]
40
+ ensure
41
+ s.close
42
+ end
43
+ end if ENV["STRESS"].to_i != 0
@@ -0,0 +1,158 @@
1
+ # -*- encoding: binary -*-
2
+ require 'test/unit'
3
+ require 'tempfile'
4
+ require 'raindrops'
5
+ require 'socket'
6
+ require 'pp'
7
+ require "./test/ipv6_enabled"
8
+ $stderr.sync = $stdout.sync = true
9
+
10
+ class TestLinuxIPv6 < Test::Unit::TestCase
11
+ include Raindrops::Linux
12
+
13
+ TEST_ADDR = ENV["TEST_HOST6"] || "::1"
14
+
15
+ def test_tcp
16
+ s = TCPServer.new(TEST_ADDR, 0)
17
+ port = s.addr[1]
18
+ addr = "[#{TEST_ADDR}]:#{port}"
19
+ addrs = [ addr ]
20
+ stats = tcp_listener_stats(addrs)
21
+ assert_equal 1, stats.size
22
+ assert_equal 0, stats[addr].queued
23
+ assert_equal 0, stats[addr].active
24
+
25
+ c = TCPSocket.new(TEST_ADDR, port)
26
+ stats = tcp_listener_stats(addrs)
27
+ assert_equal 1, stats.size
28
+ assert_equal 1, stats[addr].queued
29
+ assert_equal 0, stats[addr].active
30
+
31
+ sc = s.accept
32
+ stats = tcp_listener_stats(addrs)
33
+ assert_equal 1, stats.size
34
+ assert_equal 0, stats[addr].queued
35
+ assert_equal 1, stats[addr].active
36
+ end
37
+
38
+ def test_tcp_multi
39
+ s1 = TCPServer.new(TEST_ADDR, 0)
40
+ s2 = TCPServer.new(TEST_ADDR, 0)
41
+ port1, port2 = s1.addr[1], s2.addr[1]
42
+ addr1, addr2 = "[#{TEST_ADDR}]:#{port1}", "[#{TEST_ADDR}]:#{port2}"
43
+ addrs = [ addr1, addr2 ]
44
+ stats = tcp_listener_stats(addrs)
45
+ assert_equal 2, stats.size
46
+ assert_equal 0, stats[addr1].queued
47
+ assert_equal 0, stats[addr1].active
48
+ assert_equal 0, stats[addr2].queued
49
+ assert_equal 0, stats[addr2].active
50
+
51
+ c1 = TCPSocket.new(TEST_ADDR, port1)
52
+ stats = tcp_listener_stats(addrs)
53
+ assert_equal 2, stats.size
54
+ assert_equal 1, stats[addr1].queued
55
+ assert_equal 0, stats[addr1].active
56
+ assert_equal 0, stats[addr2].queued
57
+ assert_equal 0, stats[addr2].active
58
+
59
+ sc1 = s1.accept
60
+ stats = tcp_listener_stats(addrs)
61
+ assert_equal 2, stats.size
62
+ assert_equal 0, stats[addr1].queued
63
+ assert_equal 1, stats[addr1].active
64
+ assert_equal 0, stats[addr2].queued
65
+ assert_equal 0, stats[addr2].active
66
+
67
+ c2 = TCPSocket.new(TEST_ADDR, port2)
68
+ stats = tcp_listener_stats(addrs)
69
+ assert_equal 2, stats.size
70
+ assert_equal 0, stats[addr1].queued
71
+ assert_equal 1, stats[addr1].active
72
+ assert_equal 1, stats[addr2].queued
73
+ assert_equal 0, stats[addr2].active
74
+
75
+ c3 = TCPSocket.new(TEST_ADDR, port2)
76
+ stats = tcp_listener_stats(addrs)
77
+ assert_equal 2, stats.size
78
+ assert_equal 0, stats[addr1].queued
79
+ assert_equal 1, stats[addr1].active
80
+ assert_equal 2, stats[addr2].queued
81
+ assert_equal 0, stats[addr2].active
82
+
83
+ sc2 = s2.accept
84
+ stats = tcp_listener_stats(addrs)
85
+ assert_equal 2, stats.size
86
+ assert_equal 0, stats[addr1].queued
87
+ assert_equal 1, stats[addr1].active
88
+ assert_equal 1, stats[addr2].queued
89
+ assert_equal 1, stats[addr2].active
90
+
91
+ sc1.close
92
+ stats = tcp_listener_stats(addrs)
93
+ assert_equal 0, stats[addr1].queued
94
+ assert_equal 0, stats[addr1].active
95
+ assert_equal 1, stats[addr2].queued
96
+ assert_equal 1, stats[addr2].active
97
+ end
98
+
99
+ def test_invalid_addresses
100
+ assert_raises(ArgumentError) { tcp_listener_stats(%w([1:::5)) }
101
+ assert_raises(ArgumentError) { tcp_listener_stats(%w([1:::]5)) }
102
+ end
103
+
104
+ # tries to overflow buffers
105
+ def test_tcp_stress_test
106
+ nr_proc = 32
107
+ nr_sock = 500
108
+ s = TCPServer.new(TEST_ADDR, 0)
109
+ port = s.addr[1]
110
+ addr = "[#{TEST_ADDR}]:#{port}"
111
+ addrs = [ addr ]
112
+ rda, wra = IO.pipe
113
+ rdb, wrb = IO.pipe
114
+
115
+ nr_proc.times do
116
+ fork do
117
+ rda.close
118
+ wrb.close
119
+ socks = (1..nr_sock).map { s.accept }
120
+ wra.syswrite('.')
121
+ wra.close
122
+ rdb.sysread(1) # wait for parent to nuke us
123
+ end
124
+ end
125
+
126
+ nr_proc.times do
127
+ fork do
128
+ rda.close
129
+ wrb.close
130
+ socks = (1..nr_sock).map { TCPSocket.new(TEST_ADDR, port) }
131
+ wra.syswrite('.')
132
+ wra.close
133
+ rdb.sysread(1) # wait for parent to nuke us
134
+ end
135
+ end
136
+
137
+ assert_equal('.' * (nr_proc * 2), rda.read(nr_proc * 2))
138
+
139
+ rda.close
140
+ stats = tcp_listener_stats(addrs)
141
+ expect = { addr => Raindrops::ListenStats[nr_sock * nr_proc, 0] }
142
+ assert_equal expect, stats
143
+
144
+ uno_mas = TCPSocket.new(TEST_ADDR, port)
145
+ stats = tcp_listener_stats(addrs)
146
+ expect = { addr => Raindrops::ListenStats[nr_sock * nr_proc, 1] }
147
+ assert_equal expect, stats
148
+
149
+ if ENV["BENCHMARK"].to_i != 0
150
+ require 'benchmark'
151
+ puts(Benchmark.measure{1000.times { tcp_listener_stats(addrs) }})
152
+ end
153
+
154
+ wrb.syswrite('.' * (nr_proc * 2)) # broadcast a wakeup
155
+ statuses = Process.waitall
156
+ statuses.each { |(pid,status)| assert status.success?, status.inspect }
157
+ end if ENV["STRESS"].to_i != 0
158
+ end if RUBY_PLATFORM =~ /linux/ && ipv6_enabled?
@@ -0,0 +1,61 @@
1
+ # -*- encoding: binary -*-
2
+ require 'test/unit'
3
+ require 'tempfile'
4
+ require 'raindrops'
5
+ require 'socket'
6
+ require 'pp'
7
+ $stderr.sync = $stdout.sync = true
8
+ class TestLinuxTCP_Info < Test::Unit::TestCase
9
+
10
+ TEST_ADDR = ENV['UNICORN_TEST_ADDR'] || '127.0.0.1'
11
+
12
+ # Linux kernel commit 5ee3afba88f5a79d0bff07ddd87af45919259f91
13
+ TCP_INFO_useful_listenq = `uname -r`.strip >= '2.6.24'
14
+
15
+ def test_tcp_server
16
+ s = TCPServer.new(TEST_ADDR, 0)
17
+ rv = Raindrops::TCP_Info.new s
18
+ c = TCPSocket.new TEST_ADDR, s.addr[1]
19
+ tmp = Raindrops::TCP_Info.new s
20
+ TCP_INFO_useful_listenq and assert_equal 1, tmp.unacked
21
+
22
+ assert_equal 0, rv.unacked
23
+ a = s.accept
24
+ tmp = Raindrops::TCP_Info.new s
25
+ assert_equal 0, tmp.unacked
26
+ ensure
27
+ c.close if c
28
+ a.close if a
29
+ s.close
30
+ end
31
+
32
+ def test_accessors
33
+ s = TCPServer.new TEST_ADDR, 0
34
+ tmp = Raindrops::TCP_Info.new s
35
+ tcp_info_methods = tmp.methods - Object.new.methods
36
+ assert tcp_info_methods.size >= 32
37
+ tcp_info_methods.each do |m|
38
+ val = tmp.__send__ m
39
+ assert_kind_of Integer, val
40
+ assert val >= 0
41
+ end
42
+ ensure
43
+ s.close
44
+ end
45
+
46
+ def test_tcp_server_delayed
47
+ delay = 0.010
48
+ delay_ms = (delay * 1000).to_i
49
+ s = TCPServer.new(TEST_ADDR, 0)
50
+ c = TCPSocket.new TEST_ADDR, s.addr[1]
51
+ c.syswrite "."
52
+ sleep delay
53
+ a = s.accept
54
+ i = Raindrops::TCP_Info.new(a)
55
+ assert i.last_data_recv >= delay_ms, "#{i.last_data_recv} < #{delay_ms}"
56
+ ensure
57
+ c.close if c
58
+ a.close if a
59
+ s.close
60
+ end
61
+ end
@@ -14,7 +14,7 @@ class TestMiddleware < Test::Unit::TestCase
14
14
  app = Raindrops::Middleware.new(@app)
15
15
  response = app.call({})
16
16
  assert_equal @response[0,2], response[0,2]
17
- assert response.last.kind_of?(Raindrops::Middleware)
17
+ assert response.last.kind_of?(Raindrops::Middleware::Proxy)
18
18
  assert response.last.object_id != app.object_id
19
19
  tmp = []
20
20
  response.last.each { |y| tmp << y }
@@ -35,7 +35,7 @@ class TestMiddleware < Test::Unit::TestCase
35
35
  assert_equal 0, stats.calling
36
36
  assert_equal 1, stats.writing
37
37
  assert_equal 200, response[0]
38
- assert response.last.kind_of?(Raindrops::Middleware)
38
+ assert response.last.kind_of?(Raindrops::Middleware::Proxy)
39
39
  tmp = []
40
40
  response.last.each do |y|
41
41
  assert_equal 1, stats.writing
@@ -108,4 +108,17 @@ class TestMiddleware < Test::Unit::TestCase
108
108
  assert_equal expect, response
109
109
  end
110
110
 
111
+ def test_middleware_proxy_to_path_missing
112
+ app = Raindrops::Middleware.new(@app)
113
+ response = app.call({})
114
+ body = response[2]
115
+ assert_kind_of Raindrops::Middleware::Proxy, body
116
+ assert ! body.respond_to?(:to_path)
117
+ assert body.respond_to?(:close)
118
+ orig_body = @response[2]
119
+
120
+ def orig_body.to_path; "/dev/null"; end
121
+ assert body.respond_to?(:to_path)
122
+ assert_equal "/dev/null", body.to_path
123
+ end
111
124
  end
@@ -0,0 +1,37 @@
1
+ # -*- encoding: binary -*-
2
+ require "./test/rack_unicorn"
3
+ $stderr.sync = $stdout.sync = true
4
+
5
+ class TestMiddlewareUnicorn < Test::Unit::TestCase
6
+
7
+ def setup
8
+ @host = ENV["UNICORN_TEST_ADDR"] || "127.0.0.1"
9
+ @sock = TCPServer.new @host, 0
10
+ @port = @sock.addr[1]
11
+ ENV["UNICORN_FD"] = @sock.fileno.to_s
12
+ @host_with_port = "#@host:#@port"
13
+ @opts = { :listeners => [ @host_with_port ] }
14
+ @addr_regexp = Regexp.escape @host_with_port
15
+ end
16
+
17
+ def test_auto_listener
18
+ @app = Rack::Builder.new do
19
+ use Raindrops::Middleware
20
+ run Rack::Lobster.new
21
+ end
22
+ @srv = fork { Unicorn.run(@app, @opts) }
23
+
24
+ s = TCPSocket.new @host, @port
25
+ s.write "GET /_raindrops HTTP/1.0\r\n\r\n"
26
+ resp = s.read
27
+ head, body = resp.split /\r\n\r\n/, 2
28
+ assert_match %r{^#@addr_regexp active: 1$}, body
29
+ assert_match %r{^#@addr_regexp queued: 0$}, body
30
+ end
31
+
32
+ def teardown
33
+ Process.kill :QUIT, @srv
34
+ _, status = Process.waitpid2 @srv
35
+ assert status.success?
36
+ end
37
+ end if defined?(Unicorn) && RUBY_PLATFORM =~ /linux/
@@ -0,0 +1,37 @@
1
+ # -*- encoding: binary -*-
2
+ require "./test/rack_unicorn"
3
+ require "./test/ipv6_enabled"
4
+ $stderr.sync = $stdout.sync = true
5
+
6
+ class TestMiddlewareUnicornIPv6 < Test::Unit::TestCase
7
+
8
+ def setup
9
+ @host = ENV["TEST_HOST6"] || "::1"
10
+ sock = TCPServer.new @host, 0
11
+ @port = sock.addr[1]
12
+ ENV["UNICORN_FD"] = sock.fileno.to_s
13
+ @host_with_port = "[#@host]:#@port"
14
+ @opts = { :listeners => [ @host_with_port ] }
15
+ @addr_regexp = Regexp.escape @host_with_port
16
+ end
17
+
18
+ def test_auto_listener
19
+ @app = Rack::Builder.new do
20
+ use Raindrops::Middleware
21
+ run Rack::Lobster.new
22
+ end
23
+ @srv = fork { Unicorn.run(@app, @opts) }
24
+ s = TCPSocket.new @host, @port
25
+ s.write "GET /_raindrops HTTP/1.0\r\n\r\n"
26
+ resp = s.read
27
+ head, body = resp.split /\r\n\r\n/, 2
28
+ assert_match %r{^#@addr_regexp active: 1$}, body
29
+ assert_match %r{^#@addr_regexp queued: 0$}, body
30
+ end
31
+
32
+ def teardown
33
+ Process.kill :QUIT, @srv
34
+ _, status = Process.waitpid2 @srv
35
+ assert status.success?
36
+ end
37
+ end if defined?(Unicorn) && RUBY_PLATFORM =~ /linux/ && ipv6_enabled?
@@ -4,15 +4,27 @@ require 'raindrops'
4
4
 
5
5
  class TestRaindrops < Test::Unit::TestCase
6
6
 
7
+ def test_raindrop_counter_max
8
+ assert_kind_of Integer, Raindrops::MAX
9
+ assert Raindrops::MAX > 0
10
+ printf "Raindrops::MAX = 0x%x\n", Raindrops::MAX
11
+ end
12
+
7
13
  def test_raindrop_size
8
14
  assert_kind_of Integer, Raindrops::SIZE
9
15
  assert Raindrops::SIZE > 0
10
16
  puts "Raindrops::SIZE = #{Raindrops::SIZE}"
11
17
  end
12
18
 
13
- def test_size
19
+ def test_page_size
20
+ assert_kind_of Integer, Raindrops::PAGE_SIZE
21
+ assert Raindrops::PAGE_SIZE > Raindrops::SIZE
22
+ end
23
+
24
+ def test_size_and_capa
14
25
  rd = Raindrops.new(4)
15
26
  assert_equal 4, rd.size
27
+ assert rd.capa >= rd.size
16
28
  end
17
29
 
18
30
  def test_ary
@@ -98,4 +110,56 @@ class TestRaindrops < Test::Unit::TestCase
98
110
  assert_equal expect, rd.to_ary
99
111
  end
100
112
 
113
+ def test_resize
114
+ rd = Raindrops.new(4)
115
+ assert_equal 4, rd.size
116
+ assert_equal rd.capa, rd.size = rd.capa
117
+ assert_equal rd.capa, rd.to_ary.size
118
+ assert_equal 0, rd[rd.capa - 1]
119
+ assert_equal 1, rd.incr(rd.capa - 1)
120
+ assert_raises(ArgumentError) { rd[rd.capa] }
121
+ end
122
+
123
+ def test_resize_mremap
124
+ rd = Raindrops.new(4)
125
+ assert_equal 4, rd.size
126
+ old_capa = rd.capa
127
+ rd.size = rd.capa + 1
128
+ assert_equal old_capa * 2, rd.capa
129
+
130
+ # mremap() is currently broken with MAP_SHARED
131
+ # https://bugzilla.kernel.org/show_bug.cgi?id=8691
132
+ assert_equal 0, rd[old_capa]
133
+ assert_equal rd.capa, rd.to_ary.size
134
+ assert_equal 0, rd[rd.capa - 1]
135
+ assert_equal 1, rd.incr(rd.capa - 1)
136
+ assert_raises(ArgumentError) { rd[rd.capa] }
137
+ rescue RangeError
138
+ end # if RUBY_PLATFORM =~ /linux/
139
+
140
+ def test_evaporate
141
+ rd = Raindrops.new 1
142
+ assert_nil rd.evaporate!
143
+ assert_raises(StandardError) { rd.evaporate! }
144
+ end
145
+
146
+ def test_evaporate_with_fork
147
+ tmp = Raindrops.new 2
148
+ pid = fork do
149
+ tmp.incr 0
150
+ exit(tmp.evaporate! == nil)
151
+ end
152
+ _, status = Process.waitpid2(pid)
153
+ assert status.success?
154
+ assert_equal [ 1, 0 ], tmp.to_ary
155
+ tmp.incr 1
156
+ assert_equal [ 1, 1 ], tmp.to_ary
157
+ pid = fork do
158
+ tmp.incr 1
159
+ exit([ 1, 2 ] == tmp.to_ary)
160
+ end
161
+ _, status = Process.waitpid2(pid)
162
+ assert status.success?
163
+ assert_equal [ 1, 2 ], tmp.to_ary
164
+ end
101
165
  end