conveyor 0.1.4 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -19,24 +19,28 @@ class TestConveyorChannel < Test::Unit::TestCase
19
19
  assert_equal 1, w.instance_variable_get(:@last_id)
20
20
  end
21
21
  end
22
-
22
+
23
23
  def test_parse_headers
24
+ FileUtils.rm_r '/tmp/foo' rescue nil
25
+
24
26
  i = Channel.new '/tmp/foo'
25
27
  [
26
- ["1 2008-01-08T13:04:40-08:00 0 3 acbd18db4cc2f85cedef654fccc4a4d8\n",
27
- {:id => 1, :time => "2008-01-08T13:04:40-08:00", :offset => 0, :length => 3, :hash => "acbd18db4cc2f85cedef654fccc4a4d8"}
28
+ ["1 jucfvs 0 3 acbd18db4cc2f85cedef654fccc4a4d8 0\n",
29
+ {:id => 1, :time => 1199826280, :offset => 0, :length => 3, :hash => "acbd18db4cc2f85cedef654fccc4a4d8", :file => nil, :flags => 0}
28
30
  ]
29
31
  ].each do |(str, ret)|
30
32
  assert_equal ret, i.parse_headers(str)
31
33
  end
32
34
 
33
35
  [
34
- ["2 2008-01-08T13:04:40-08:00 0 3 acbd18db4cc2f85cedef654fccc4a4d8 1\n",
35
- {:id => 2, :time => "2008-01-08T13:04:40-08:00", :offset => 0, :length => 3, :hash => "acbd18db4cc2f85cedef654fccc4a4d8", :file => 1}
36
+ ["2 jucfvs 0 3 acbd18db4cc2f85cedef654fccc4a4d8 0 1\n",
37
+ {:id => 2, :time => 1199826280, :offset => 0, :length => 3, :hash => "acbd18db4cc2f85cedef654fccc4a4d8", :file => 1, :flags => 0}
36
38
  ]
37
39
  ].each do |(str, ret)|
38
40
  assert_equal ret, i.parse_headers(str, true)
39
41
  end
42
+
43
+ FileUtils.rm_r '/tmp/foo'
40
44
  end
41
45
 
42
46
  def test_init_existing
@@ -133,10 +137,10 @@ class TestConveyorChannel < Test::Unit::TestCase
133
137
  c.post 'bam'
134
138
 
135
139
  status = {
136
- :directory => '/tmp/bar',
140
+ :directory => '/tmp/bar',
137
141
  :index => {:size => 3},
138
142
  :data_files => [
139
- {:path => '/tmp/bar/0', :bytes => 210}
143
+ {:path => '/tmp/bar/0', :bytes => 158}
140
144
  ],
141
145
  :iterator => {:position => 1},
142
146
  :iterator_groups => {}
@@ -190,7 +194,6 @@ class TestConveyorChannel < Test::Unit::TestCase
190
194
  r = c.get_next_n 10
191
195
  r.each_with_index do |f, i|
192
196
  assert_equal Digest::MD5.hexdigest((j*10 + i).to_s), f[0][:hash]
193
- assert_equal((j*10 + i).to_s.length, f[0][:length])
194
197
  assert_equal((j*10 + i)+1, f[0][:id])
195
198
  assert_equal((j*10 + i).to_s, f[1])
196
199
  end
@@ -202,7 +205,6 @@ class TestConveyorChannel < Test::Unit::TestCase
202
205
  r = c.get_next_n 10
203
206
  r.each_with_index do |f, i|
204
207
  assert_equal Digest::MD5.hexdigest((j*10 + i).to_s), f[0][:hash]
205
- assert_equal((j*10 + i).to_s.length, f[0][:length])
206
208
  assert_equal((100 + j*10 + i)+1, f[0][:id])
207
209
  assert_equal((j*10 + i).to_s, f[1])
208
210
  end
@@ -218,7 +220,6 @@ class TestConveyorChannel < Test::Unit::TestCase
218
220
  r = c.get_next_n_by_group 10, 'foo'
219
221
  r.each_with_index do |f, i|
220
222
  assert_equal Digest::MD5.hexdigest((j*10 + i).to_s), f[0][:hash]
221
- assert_equal((j*10 + i).to_s.length, f[0][:length])
222
223
  assert_equal((j*10 + i)+1, f[0][:id])
223
224
  assert_equal((j*10 + i).to_s, f[1])
224
225
  end
@@ -230,7 +231,6 @@ class TestConveyorChannel < Test::Unit::TestCase
230
231
  r = c.get_next_n_by_group 10, 'bar'
231
232
  r.each_with_index do |f, i|
232
233
  assert_equal Digest::MD5.hexdigest((j*10 + i).to_s), f[0][:hash]
233
- assert_equal((j*10 + i).to_s.length, f[0][:length])
234
234
  assert_equal((j*10 + i)+1, f[0][:id])
235
235
  assert_equal((j*10 + i).to_s, f[1])
236
236
  end
@@ -238,4 +238,17 @@ class TestConveyorChannel < Test::Unit::TestCase
238
238
  assert_equal [], c.get_next_n_by_group(10, 'bar')
239
239
  end
240
240
 
241
+ def test_delete
242
+ chan = 'test_delete'
243
+ FileUtils.rm_r "/tmp/#{chan}" rescue nil
244
+ c = Channel.new("/tmp/#{chan}")
245
+ 10.times {|i| c.post i.to_s}
246
+ 10.times {|i| assert_equal(i.to_s, c.get(i+1)[1])}
247
+ c.delete!
248
+ 10.times {|i| assert_equal(nil, c.get(i+1))}
249
+
250
+ d = Channel.new("/tmp/#{chan}")
251
+ assert_equal nil, d.get(1)
252
+ end
253
+
241
254
  end
@@ -22,6 +22,8 @@ class TestReplicatedChannel < Test::Unit::TestCase
22
22
  c1.commit_thread.run
23
23
  c2.commit_thread.run
24
24
 
25
+ sleep 1
26
+
25
27
  assert_equal 'foo', c1.get(1)[1]
26
28
  assert_equal 'foo', c2.get(1)[1]
27
29
 
@@ -30,6 +32,9 @@ class TestReplicatedChannel < Test::Unit::TestCase
30
32
 
31
33
  assert_equal c1.get(1), c2.get(1)
32
34
  assert_equal c1.get(2), c2.get(2)
35
+
36
+ FileUtils.rm_r '/tmp/foo' rescue nil
37
+ FileUtils.rm_r '/tmp/bar' rescue nil
33
38
  end
34
39
 
35
40
  def test_more
@@ -49,12 +54,16 @@ class TestReplicatedChannel < Test::Unit::TestCase
49
54
  channels[i % 2].post(d, Time.now)
50
55
  end
51
56
 
57
+ c1.ping_thread.run
58
+ c2.ping_thread.run
52
59
  c1.commit_thread.run
53
60
  c2.commit_thread.run
54
61
 
55
- c1d = (1..10).collect{|i| c1.get(i)}
56
- c2d = (1..10).collect{|i| c2.get(i)}
62
+ sleep 1
57
63
 
58
- assert_equal c1d, c2d
64
+ c1d = (1..10).collect{|i| c1.get(i)[1]}
65
+ c2d = (1..10).collect{|i| c2.get(i)[1]}
66
+ assert_equal data, c1d
67
+ assert_equal data, c2d
59
68
  end
60
69
  end
@@ -2,40 +2,44 @@ require "test/unit"
2
2
  require "conveyor/server"
3
3
  require 'net/http'
4
4
  require 'conveyor/client'
5
+ require 'thin'
5
6
 
6
7
  class TestConveyorServer < Test::Unit::TestCase
7
8
  include Conveyor
8
- def setup
9
- FileUtils.rm_r('/tmp/asdf') rescue nil
10
- FileUtils.mkdir('/tmp/asdf')
11
- @server = Conveyor::Server.new("127.0.0.1", 8011, '/tmp/asdf')
12
- @server.run
13
- end
14
-
15
- def teardown
16
- @server.stop
9
+
10
+ FileUtils.rm_r('/tmp/asdf') rescue nil
11
+ FileUtils.mkdir('/tmp/asdf')
12
+
13
+ Thread.start do
14
+ Thin::Server.start('0.0.0.0', 8011) do
15
+ map '/channels' do
16
+ run Conveyor::App.new('/tmp/asdf', :unsafe_mode => true)
17
+ end
18
+ end
17
19
  end
18
-
20
+
19
21
  def test_channels
20
22
  Net::HTTP.start("localhost", 8011) do |h|
21
23
  req = h.get('/channels')
22
24
  assert_equal Net::HTTPOK, req.class
23
25
  end
24
26
  end
25
-
27
+
26
28
  def test_create_channel
29
+ chan = 'test_create_channel'
27
30
  Net::HTTP.start('localhost', 8011) do |h|
28
- req = h.put('/channels/foo', '', {'Content-Type' => 'application/octet-stream'})
31
+ req = h.put("/channels/#{chan}", '', {'Content-Type' => 'application/octet-stream'})
29
32
  assert_equal Net::HTTPCreated, req.class
30
33
 
31
- req = h.post('/channels/foo', 'foo', {'Content-Type' => 'application/octet-stream', 'Date' => Time.now.to_s})
34
+ req = h.post("/channels/#{chan}", 'foo', {'Content-Type' => 'application/octet-stream', 'Date' => Time.now.to_s})
32
35
  assert_equal Net::HTTPAccepted, req.class
33
36
  end
34
37
  end
35
-
38
+
36
39
  def test_post
40
+ chan = 'test_post'
37
41
  Net::HTTP.start('localhost', 8011) do |h|
38
- req = h.put('/channels/bar', '', {'Content-Type' => 'application/octet-stream'})
42
+ req = h.put("/channels/#{chan}", '', {'Content-Type' => 'application/octet-stream'})
39
43
  assert_equal Net::HTTPCreated, req.class
40
44
 
41
45
  data =
@@ -48,12 +52,12 @@ class TestConveyorServer < Test::Unit::TestCase
48
52
  "yY3vhjeq","2IDeF0ccG8tRZIZSekz6fUii29"]
49
53
 
50
54
  data.each do |d|
51
- req = h.post('/channels/bar', d, {'Content-Type' => 'application/octet-stream', 'Date' => Time.now.to_s})
55
+ req = h.post("/channels/#{chan}", d, {'Content-Type' => 'application/octet-stream', 'Date' => Time.now.to_s})
52
56
  assert_equal Net::HTTPAccepted, req.class
53
57
  end
54
58
 
55
59
  data.each_with_index do |d, i|
56
- req = h.get("/channels/bar/#{i+1}")
60
+ req = h.get("/channels/#{chan}/#{i+1}")
57
61
  assert_equal d, req.body
58
62
  end
59
63
  end
@@ -94,25 +98,25 @@ class TestConveyorServer < Test::Unit::TestCase
94
98
  end
95
99
 
96
100
  def test_status
101
+ chan = 'test_status'
97
102
  Net::HTTP.start('localhost', 8011) do |h|
98
- req = h.put('/channels/bar', '', {'Content-Type' => 'application/octet-stream'})
103
+ req = h.put("/channels/#{chan}", '', {'Content-Type' => 'application/octet-stream'})
99
104
  assert_equal Net::HTTPCreated, req.class
100
105
 
101
- data =
102
- ["ZqZyDN2SouQCYEHYS0LuM1XeqsF0MKIbFEBE6xQ972VqEcjs21wJSosvZMWEH1lq5ukTq4Ze"]
103
-
106
+ data = ["ZqZyDN2SouQCYEHYS0LuM1XeqsF0MKIbFEBE6xQ972VqEcjs21wJSosvZMWEH1lq5ukTq4Ze"]
107
+
104
108
  data.each do |d|
105
- req = h.post('/channels/bar', d, {'Content-Type' => 'application/octet-stream', 'Date' => Time.now.to_s})
109
+ req = h.post("/channels/#{chan}", d, {'Content-Type' => 'application/octet-stream', 'Date' => Time.now.to_s})
106
110
  assert_equal Net::HTTPAccepted, req.class
107
111
  end
108
112
 
109
- req = h.get("/channels/bar")
113
+ req = h.get("/channels/#{chan}")
110
114
  assert_kind_of Net::HTTPOK, req
111
115
  json = {
112
116
  "iterator_groups" => {},
113
117
  "index"=>{"size"=>1},
114
- "directory"=>"/tmp/asdf/bar",
115
- "data_files"=>[{"path"=>"/tmp/asdf/bar/0","bytes"=>139}],
118
+ "directory"=>"/tmp/asdf/#{chan}",
119
+ "data_files"=>[{"path"=>"/tmp/asdf/#{chan}/0","bytes"=>122}],
116
120
  "iterator"=>{"position"=>1}
117
121
  }
118
122
  assert_equal json, JSON::parse(req.body)
@@ -121,83 +125,83 @@ class TestConveyorServer < Test::Unit::TestCase
121
125
  end
122
126
 
123
127
  def test_rewinding
128
+ chan = 'test_rewinding'
124
129
  Net::HTTP.start('localhost', 8011) do |h|
125
- req = h.put('/channels/bar', '', {'Content-Type' => 'application/octet-stream'})
130
+ req = h.put("/channels/#{chan}", '', {'Content-Type' => 'application/octet-stream'})
126
131
  assert_equal Net::HTTPCreated, req.class
127
132
 
128
133
  data =
129
134
  ["ZqZyDN2SouQCYEHYS0LuM1XeqsF0MKIbFEBE6xQ972VqEcjs21wJSosvZMWEH1lq5ukTq4Ze"]
130
135
 
131
136
  data.each do |d|
132
- req = h.post('/channels/bar', d, {'Content-Type' => 'application/octet-stream', 'Date' => Time.now.to_s})
137
+ req = h.post("/channels/#{chan}", d, {'Content-Type' => 'application/octet-stream', 'Date' => Time.now.to_s})
133
138
  assert_equal Net::HTTPAccepted, req.class
134
139
  end
135
140
 
136
- req = h.get('/channels/bar?next')
141
+ req = h.get("/channels/#{chan}?next")
137
142
 
138
143
  assert_kind_of Net::HTTPOK, req
139
144
  assert_equal data[0], req.body
140
-
141
- req = h.get('/channels/bar?next')
142
-
145
+
146
+ req = h.get("/channels/#{chan}?next")
147
+
143
148
  assert_kind_of Net::HTTPNotFound, req
144
-
145
- req = h.post('/channels/bar?rewind_id=1', nil)
149
+
150
+ req = h.post("/channels/#{chan}?rewind_id=1", nil)
146
151
  assert_kind_of Net::HTTPOK, req
147
-
148
- req = h.get('/channels/bar?next')
152
+
153
+ req = h.get("/channels/#{chan}?next")
149
154
 
150
155
  assert_kind_of Net::HTTPOK, req
151
156
  assert_equal data[0], req.body
152
157
 
153
- req = h.get('/channels/bar?next')
158
+ req = h.get("/channels/#{chan}?next")
154
159
  assert_kind_of Net::HTTPNotFound, req
155
160
  end
156
161
  end
157
162
 
158
163
  def test_group_rewind
159
164
  chan = 'test_group_rewind'
160
- c = Client.new 'localhost'
161
- c.create_channel chan
162
- c.post chan, 'foo'
165
+ c = Client.new 'localhost', chan
166
+ c.create_channel
167
+ c.post 'foo'
163
168
 
164
- assert_equal 'foo', c.get_next(chan, 'bar')
165
- c.rewind(chan, 1, 'bar')
166
- assert_equal 'foo', c.get_next(chan, 'bar')
167
- c.rewind(chan, 1, 'bar')
169
+ assert_equal 'foo', c.get_next('bar')
170
+ c.rewind(1, 'bar')
171
+ assert_equal 'foo', c.get_next('bar')
172
+ c.rewind(1, 'bar')
168
173
  end
169
174
 
170
-
171
175
  def test_get_next_by_group
172
- c = Conveyor::Client.new 'localhost'
173
- chan = 'asdf'
174
- c.create_channel chan
175
- c.post chan, 'foo'
176
- c.post chan, 'bar'
177
- c.post chan, 'bam'
176
+ chan = 'test_get_next_by_group'
177
+ c = Conveyor::Client.new 'localhost', chan
178
+ c.create_channel
179
+ c.post 'foo'
180
+ c.post 'bar'
181
+ c.post 'bam'
178
182
 
179
183
  group = 'bam'
180
184
 
181
- assert_equal 'foo', c.get_next(chan, group)
182
- assert_equal 'bar', c.get_next(chan, group)
183
- assert_equal 'bam', c.get_next(chan, group)
184
- assert_equal '', c.get_next(chan, group)
185
+ assert_equal 'foo', c.get_next(group)
186
+ assert_equal 'bar', c.get_next(group)
187
+ assert_equal 'bam', c.get_next(group)
188
+ assert_equal '', c.get_next(group)
185
189
 
186
190
  group = 'bar'
187
- assert_equal 'foo', c.get_next(chan, group)
188
- assert_equal 'bar', c.get_next(chan, group)
189
- assert_equal 'bam', c.get_next(chan, group)
190
- assert_equal '', c.get_next(chan, group)
191
+ assert_equal 'foo', c.get_next(group)
192
+ assert_equal 'bar', c.get_next(group)
193
+ assert_equal 'bam', c.get_next(group)
194
+ assert_equal '', c.get_next(group)
191
195
  end
192
196
 
193
197
  def test_get_next_n
194
198
  chan = 'test_get_next_n'
195
- c = Client.new 'localhost'
196
- c.create_channel chan
197
- 100.times {|i| c.post chan, i.to_s}
199
+ c = Client.new 'localhost', chan
200
+ c.create_channel
201
+ 100.times {|i| c.post i.to_s}
198
202
 
199
203
  10.times do |j|
200
- r = c.get_next_n chan, 10
204
+ r = c.get_next_n 10
201
205
  r.each_with_index do |f, i|
202
206
  assert_equal((j*10 + i)+1, f["id"])
203
207
  assert_equal(Digest::MD5.hexdigest((j*10 + i).to_s), f["hash"])
@@ -205,10 +209,10 @@ class TestConveyorServer < Test::Unit::TestCase
205
209
  end
206
210
  end
207
211
 
208
- 100.times {|i| c.post chan, i.to_s}
212
+ 100.times {|i| c.post i.to_s}
209
213
 
210
214
  10.times do |j|
211
- r = c.get_next_n chan, 10
215
+ r = c.get_next_n 10
212
216
  r.each_with_index do |f, i|
213
217
  assert_equal(Digest::MD5.hexdigest((j*10 + i).to_s), f["hash"])
214
218
  assert_equal((100 + j*10 + i)+1, f["id"])
@@ -219,31 +223,57 @@ class TestConveyorServer < Test::Unit::TestCase
219
223
 
220
224
  def test_get_next_n_by_group
221
225
  chan = 'test_get_next_n_by_group'
222
- c = Client.new 'localhost'
223
- c.create_channel chan
224
- 100.times {|i| c.post chan, i.to_s}
226
+ c = Client.new 'localhost', chan
227
+ c.create_channel
228
+ 100.times {|i| c.post i.to_s}
225
229
 
226
230
  10.times do |j|
227
- r = c.get_next_n chan, 10, 'foo'
231
+ r = c.get_next_n 10, 'foo'
228
232
  r.each_with_index do |f, i|
229
233
  assert_equal(Digest::MD5.hexdigest((j*10 + i).to_s), f[0]["hash"])
230
- assert_equal((j*10 + i).to_s.length, f[0]["length"])
231
234
  assert_equal((j*10 + i)+1, f[0]["id"])
232
235
  assert_equal((j*10 + i).to_s, f[1])
233
236
  end
234
237
  end
235
238
 
236
- assert_equal [], c.get_next_n(chan, 10, 'foo')
239
+ assert_equal [], c.get_next_n(10, 'foo')
237
240
 
238
241
  10.times do |j|
239
- r = c.get_next_n chan, 10, 'bar'
242
+ r = c.get_next_n 10, 'bar'
240
243
  r.each_with_index do |f, i|
241
244
  assert_equal Digest::MD5.hexdigest((j*10 + i).to_s), f[0]["hash"]
242
- assert_equal((j*10 + i).to_s.length, f[0]["length"])
243
245
  assert_equal((j*10 + i)+1, f[0]["id"])
244
246
  assert_equal((j*10 + i).to_s, f[1])
245
247
  end
246
248
  end
247
- assert_equal [], c.get_next_n(chan, 10, 'bar')
249
+ assert_equal [], c.get_next_n(10, 'bar')
250
+ end
251
+
252
+ def test_delete
253
+ chan = "test_delete"
254
+ Net::HTTP.start("localhost", 8011) do |h|
255
+ r = h.put("/channels/#{chan}", "")
256
+ assert_kind_of Net::HTTPCreated, r
257
+ 10.times {|i| h.post("/channels/#{chan}", i.to_s, {'Content-Type' => 'application/octet-stream', 'Date' => Time.now.gmtime.to_s})}
258
+ 10.times {|i| assert_equal(i.to_s, h.get("/channels/#{chan}?next").body)}
259
+ 10.times {|i| assert_equal(i.to_s, h.get("/channels/#{chan}/#{i+1}").body)}
260
+ r = h.delete("/channels/#{chan}")
261
+ assert_kind_of Net::HTTPOK, r
262
+
263
+ r = h.put("/channels/#{chan}", "")
264
+ assert_kind_of Net::HTTPCreated, r
265
+ 10.times {|i| h.post("/channels/#{chan}", i.to_s, {'Content-Type' => 'application/octet-stream', 'Date' => Time.now.gmtime.to_s})}
266
+ 10.times {|i| assert_equal(i.to_s, h.get("/channels/#{chan}?next").body)}
267
+ 10.times {|i| assert_equal(i.to_s, h.get("/channels/#{chan}/#{i+1}").body)}
268
+ end
269
+ end
270
+
271
+ def test_autocreate_channel
272
+ chan = "test_autocreate_channel"
273
+ c = Client.new 'localhost', chan
274
+ c.post 'foo'
275
+ assert_equal 'foo', c.get_next
248
276
  end
249
- end
277
+
278
+ end
279
+
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: conveyor
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.4
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ryan King
@@ -30,11 +30,11 @@ cert_chain:
30
30
  Zls3y84CmyAEGg==
31
31
  -----END CERTIFICATE-----
32
32
 
33
- date: 2008-02-19 00:00:00 -08:00
33
+ date: 2008-02-26 00:00:00 -08:00
34
34
  default_executable:
35
35
  dependencies:
36
36
  - !ruby/object:Gem::Dependency
37
- name: mongrel
37
+ name: thin
38
38
  version_requirement:
39
39
  version_requirements: !ruby/object:Gem::Requirement
40
40
  requirements:
@@ -73,6 +73,7 @@ description: "* Like TiVo for your data * A distributed rewindable multi-queue"
73
73
  email: ryan@theryanking.com
74
74
  executables:
75
75
  - conveyor
76
+ - conveyor-upgrade
76
77
  extensions: []
77
78
 
78
79
  extra_rdoc_files:
@@ -86,6 +87,7 @@ files:
86
87
  - README.txt
87
88
  - Rakefile
88
89
  - bin/conveyor
90
+ - bin/conveyor-upgrade
89
91
  - docs/file-formats.mkd
90
92
  - docs/protocol.mkd
91
93
  - lib/conveyor.rb
@@ -93,12 +95,12 @@ files:
93
95
  - lib/conveyor/channel.rb
94
96
  - lib/conveyor/client.rb
95
97
  - lib/conveyor/server.rb
98
+ - lib/conveyor/upgrader.rb
96
99
  - lib/priority_queue.rb
97
100
  - test/rand.rb
98
101
  - test/test_channel.rb
99
102
  - test/test_feeder-ng.rb
100
103
  - test/test_priority_queue.rb
101
- - test/test_replicated_channel.rb
102
104
  - test/test_server.rb
103
105
  has_rdoc: true
104
106
  homepage: by Ryan King (http://theryanking.com)