conveyor 0.1.4 → 0.2.0

data/test/test_channel.rb

@@ -19,24 +19,28 @@ class TestConveyorChannel < Test::Unit::TestCase
       assert_equal 1, w.instance_variable_get(:@last_id)
     end
   end
-  
+
   def test_parse_headers
+    FileUtils.rm_r '/tmp/foo' rescue nil
+
     i = Channel.new '/tmp/foo'
     [
-      ["1 2008-01-08T13:04:40-08:00 0 3 acbd18db4cc2f85cedef654fccc4a4d8\n", 
-       {:id => 1, :time => "2008-01-08T13:04:40-08:00", :offset => 0, :length => 3, :hash => "acbd18db4cc2f85cedef654fccc4a4d8"}
+      ["1 jucfvs 0 3 acbd18db4cc2f85cedef654fccc4a4d8 0\n", 
+       {:id => 1, :time => 1199826280, :offset => 0, :length => 3, :hash => "acbd18db4cc2f85cedef654fccc4a4d8", :file => nil, :flags => 0}
       ]
     ].each do |(str, ret)|
       assert_equal ret, i.parse_headers(str)
     end
     
     [
-      ["2 2008-01-08T13:04:40-08:00 0 3 acbd18db4cc2f85cedef654fccc4a4d8 1\n", 
-       {:id => 2, :time => "2008-01-08T13:04:40-08:00", :offset => 0, :length => 3, :hash => "acbd18db4cc2f85cedef654fccc4a4d8", :file => 1}
+      ["2 jucfvs 0 3 acbd18db4cc2f85cedef654fccc4a4d8 0 1\n", 
+       {:id => 2, :time => 1199826280, :offset => 0, :length => 3, :hash => "acbd18db4cc2f85cedef654fccc4a4d8", :file => 1, :flags => 0}
       ]
     ].each do |(str, ret)|
       assert_equal ret, i.parse_headers(str, true)
     end
+    
+    FileUtils.rm_r '/tmp/foo'
   end
 
   def test_init_existing
@@ -133,10 +137,10 @@ class TestConveyorChannel < Test::Unit::TestCase
     c.post 'bam'
     
     status = {
-      :directory => '/tmp/bar', 
+      :directory => '/tmp/bar',
       :index => {:size => 3},
       :data_files => [
-        {:path => '/tmp/bar/0', :bytes => 210}
+        {:path => '/tmp/bar/0', :bytes => 158}
         ],
       :iterator => {:position => 1},
       :iterator_groups => {}
@@ -190,7 +194,6 @@ class TestConveyorChannel < Test::Unit::TestCase
       r = c.get_next_n 10
       r.each_with_index do |f, i|
         assert_equal Digest::MD5.hexdigest((j*10 + i).to_s), f[0][:hash]
-        assert_equal((j*10 + i).to_s.length,                 f[0][:length])
         assert_equal((j*10 + i)+1,                           f[0][:id])
         assert_equal((j*10 + i).to_s,                        f[1])
       end
@@ -202,7 +205,6 @@ class TestConveyorChannel < Test::Unit::TestCase
       r = c.get_next_n 10
       r.each_with_index do |f, i|
         assert_equal Digest::MD5.hexdigest((j*10 + i).to_s), f[0][:hash]
-        assert_equal((j*10 + i).to_s.length,                 f[0][:length])
         assert_equal((100 + j*10 + i)+1,                     f[0][:id])
         assert_equal((j*10 + i).to_s,                        f[1])
       end
@@ -218,7 +220,6 @@ class TestConveyorChannel < Test::Unit::TestCase
       r = c.get_next_n_by_group 10, 'foo'
       r.each_with_index do |f, i|
         assert_equal Digest::MD5.hexdigest((j*10 + i).to_s), f[0][:hash]
-        assert_equal((j*10 + i).to_s.length,                 f[0][:length])
         assert_equal((j*10 + i)+1,                           f[0][:id])
         assert_equal((j*10 + i).to_s,                        f[1])
       end
@@ -230,7 +231,6 @@ class TestConveyorChannel < Test::Unit::TestCase
       r = c.get_next_n_by_group 10, 'bar'
       r.each_with_index do |f, i|
         assert_equal Digest::MD5.hexdigest((j*10 + i).to_s), f[0][:hash]
-        assert_equal((j*10 + i).to_s.length,                 f[0][:length])
         assert_equal((j*10 + i)+1,                           f[0][:id])
         assert_equal((j*10 + i).to_s,                        f[1])
       end
@@ -238,4 +238,17 @@ class TestConveyorChannel < Test::Unit::TestCase
     assert_equal [], c.get_next_n_by_group(10, 'bar')
   end
   
+  def test_delete
+    chan = 'test_delete'
+    FileUtils.rm_r "/tmp/#{chan}" rescue nil
+    c = Channel.new("/tmp/#{chan}")
+    10.times {|i| c.post i.to_s}
+    10.times {|i| assert_equal(i.to_s, c.get(i+1)[1])}
+    c.delete!
+    10.times {|i| assert_equal(nil, c.get(i+1))}
+
+    d = Channel.new("/tmp/#{chan}")
+    assert_equal nil, d.get(1)
+  end
+  
 end

data/test/test_replicated_channel.rb

@@ -22,6 +22,8 @@ class TestReplicatedChannel < Test::Unit::TestCase
     c1.commit_thread.run
     c2.commit_thread.run
 
+    sleep 1
+
     assert_equal 'foo', c1.get(1)[1]
     assert_equal 'foo', c2.get(1)[1]
 
@@ -30,6 +32,9 @@ class TestReplicatedChannel < Test::Unit::TestCase
     
     assert_equal c1.get(1), c2.get(1)
     assert_equal c1.get(2), c2.get(2)
+
+    FileUtils.rm_r '/tmp/foo' rescue nil
+    FileUtils.rm_r '/tmp/bar' rescue nil
   end
   
   def test_more
@@ -49,12 +54,16 @@ class TestReplicatedChannel < Test::Unit::TestCase
       channels[i % 2].post(d, Time.now)
     end
 
+    c1.ping_thread.run
+    c2.ping_thread.run
     c1.commit_thread.run
     c2.commit_thread.run
 
-    c1d = (1..10).collect{|i| c1.get(i)}
-    c2d = (1..10).collect{|i| c2.get(i)}
+    sleep 1
 
-    assert_equal c1d, c2d
+    c1d = (1..10).collect{|i| c1.get(i)[1]}
+    c2d = (1..10).collect{|i| c2.get(i)[1]}
+    assert_equal data, c1d
+    assert_equal data, c2d
   end
 end

data/test/test_server.rb

@@ -2,40 +2,44 @@ require "test/unit"
 require "conveyor/server"
 require 'net/http'
 require 'conveyor/client'
+require 'thin'
 
 class TestConveyorServer < Test::Unit::TestCase
   include Conveyor
-  def setup
-    FileUtils.rm_r('/tmp/asdf') rescue nil
-    FileUtils.mkdir('/tmp/asdf')
-    @server = Conveyor::Server.new("127.0.0.1", 8011, '/tmp/asdf')
-    @server.run
-  end
-  
-  def teardown
-    @server.stop
+
+  FileUtils.rm_r('/tmp/asdf') rescue nil
+  FileUtils.mkdir('/tmp/asdf')
+
+  Thread.start do
+    Thin::Server.start('0.0.0.0', 8011) do
+      map '/channels' do
+        run Conveyor::App.new('/tmp/asdf',  :unsafe_mode => true)
+      end
+    end
   end
-  
+
   def test_channels
     Net::HTTP.start("localhost", 8011) do |h|
       req = h.get('/channels')
       assert_equal Net::HTTPOK, req.class
     end
   end
-  
+
   def test_create_channel
+    chan = 'test_create_channel'
     Net::HTTP.start('localhost', 8011) do |h|
-      req = h.put('/channels/foo', '', {'Content-Type' => 'application/octet-stream'})
+      req = h.put("/channels/#{chan}", '', {'Content-Type' => 'application/octet-stream'})
       assert_equal Net::HTTPCreated, req.class
 
-      req = h.post('/channels/foo', 'foo', {'Content-Type' => 'application/octet-stream', 'Date' => Time.now.to_s})
+      req = h.post("/channels/#{chan}", 'foo', {'Content-Type' => 'application/octet-stream', 'Date' => Time.now.to_s})
       assert_equal Net::HTTPAccepted, req.class
     end
   end
-  
+
   def test_post
+    chan = 'test_post'
     Net::HTTP.start('localhost', 8011) do |h|
-      req = h.put('/channels/bar', '', {'Content-Type' => 'application/octet-stream'})
+      req = h.put("/channels/#{chan}", '', {'Content-Type' => 'application/octet-stream'})
       assert_equal Net::HTTPCreated, req.class
 
       data =
@@ -48,12 +52,12 @@ class TestConveyorServer < Test::Unit::TestCase
         "yY3vhjeq","2IDeF0ccG8tRZIZSekz6fUii29"]
         
       data.each do |d|
-        req = h.post('/channels/bar', d, {'Content-Type' => 'application/octet-stream', 'Date' => Time.now.to_s})
+        req = h.post("/channels/#{chan}", d, {'Content-Type' => 'application/octet-stream', 'Date' => Time.now.to_s})
         assert_equal Net::HTTPAccepted, req.class
       end
 
       data.each_with_index do |d, i|
-        req = h.get("/channels/bar/#{i+1}")
+        req = h.get("/channels/#{chan}/#{i+1}")
         assert_equal d, req.body
       end
     end
@@ -94,25 +98,25 @@ class TestConveyorServer < Test::Unit::TestCase
   end
 
   def test_status
+    chan = 'test_status'
     Net::HTTP.start('localhost', 8011) do |h|
-      req = h.put('/channels/bar', '', {'Content-Type' => 'application/octet-stream'})
+      req = h.put("/channels/#{chan}", '', {'Content-Type' => 'application/octet-stream'})
       assert_equal Net::HTTPCreated, req.class
 
-      data =
-      ["ZqZyDN2SouQCYEHYS0LuM1XeqsF0MKIbFEBE6xQ972VqEcjs21wJSosvZMWEH1lq5ukTq4Ze"]
-        
+      data = ["ZqZyDN2SouQCYEHYS0LuM1XeqsF0MKIbFEBE6xQ972VqEcjs21wJSosvZMWEH1lq5ukTq4Ze"]
+
       data.each do |d|
-        req = h.post('/channels/bar', d, {'Content-Type' => 'application/octet-stream', 'Date' => Time.now.to_s})
+        req = h.post("/channels/#{chan}", d, {'Content-Type' => 'application/octet-stream', 'Date' => Time.now.to_s})
         assert_equal Net::HTTPAccepted, req.class
       end
 
-      req = h.get("/channels/bar")
+      req = h.get("/channels/#{chan}")
       assert_kind_of Net::HTTPOK, req
       json = {
         "iterator_groups" => {},
         "index"=>{"size"=>1},
-        "directory"=>"/tmp/asdf/bar",
-        "data_files"=>[{"path"=>"/tmp/asdf/bar/0","bytes"=>139}],
+        "directory"=>"/tmp/asdf/#{chan}",
+        "data_files"=>[{"path"=>"/tmp/asdf/#{chan}/0","bytes"=>122}],
         "iterator"=>{"position"=>1}
       }
       assert_equal json, JSON::parse(req.body)
@@ -121,83 +125,83 @@ class TestConveyorServer < Test::Unit::TestCase
   end
 
   def test_rewinding
+    chan = 'test_rewinding'
     Net::HTTP.start('localhost', 8011) do |h|
-      req = h.put('/channels/bar', '', {'Content-Type' => 'application/octet-stream'})
+      req = h.put("/channels/#{chan}", '', {'Content-Type' => 'application/octet-stream'})
       assert_equal Net::HTTPCreated, req.class
 
       data =
       ["ZqZyDN2SouQCYEHYS0LuM1XeqsF0MKIbFEBE6xQ972VqEcjs21wJSosvZMWEH1lq5ukTq4Ze"]
         
       data.each do |d|
-        req = h.post('/channels/bar', d, {'Content-Type' => 'application/octet-stream', 'Date' => Time.now.to_s})
+        req = h.post("/channels/#{chan}", d, {'Content-Type' => 'application/octet-stream', 'Date' => Time.now.to_s})
         assert_equal Net::HTTPAccepted, req.class
       end
 
-      req = h.get('/channels/bar?next')
+      req = h.get("/channels/#{chan}?next")
 
       assert_kind_of Net::HTTPOK, req
       assert_equal data[0], req.body
-
-      req = h.get('/channels/bar?next')
-
+      
+      req = h.get("/channels/#{chan}?next")
+      
       assert_kind_of Net::HTTPNotFound, req
-
-      req = h.post('/channels/bar?rewind_id=1', nil)
+      
+      req = h.post("/channels/#{chan}?rewind_id=1", nil)
       assert_kind_of Net::HTTPOK, req
-
-      req = h.get('/channels/bar?next')
+      
+      req = h.get("/channels/#{chan}?next")
       
       assert_kind_of Net::HTTPOK, req
       assert_equal data[0], req.body
       
-      req = h.get('/channels/bar?next')
+      req = h.get("/channels/#{chan}?next")
       assert_kind_of Net::HTTPNotFound, req
     end
   end
 
   def test_group_rewind
     chan = 'test_group_rewind'
-    c = Client.new 'localhost'
-    c.create_channel chan
-    c.post chan, 'foo'
+    c = Client.new 'localhost', chan
+    c.create_channel
+    c.post 'foo'
     
-    assert_equal 'foo', c.get_next(chan, 'bar')
-    c.rewind(chan, 1, 'bar')
-    assert_equal 'foo', c.get_next(chan, 'bar')
-    c.rewind(chan, 1, 'bar')
+    assert_equal 'foo', c.get_next('bar')
+    c.rewind(1, 'bar')
+    assert_equal 'foo', c.get_next('bar')
+    c.rewind(1, 'bar')
   end
 
-
   def test_get_next_by_group
-    c = Conveyor::Client.new 'localhost'
-    chan = 'asdf'
-    c.create_channel chan
-    c.post chan, 'foo'
-    c.post chan, 'bar'
-    c.post chan, 'bam'
+    chan = 'test_get_next_by_group'
+    c = Conveyor::Client.new 'localhost', chan
+    c.create_channel
+    c.post 'foo'
+    c.post 'bar'
+    c.post 'bam'
 
     group = 'bam'
 
-    assert_equal 'foo', c.get_next(chan, group)
-    assert_equal 'bar', c.get_next(chan, group)
-    assert_equal 'bam', c.get_next(chan, group)
-    assert_equal '',   c.get_next(chan, group)
+    assert_equal 'foo', c.get_next(group)
+    assert_equal 'bar', c.get_next(group)
+    assert_equal 'bam', c.get_next(group)
+    assert_equal '',   c.get_next(group)
 
     group = 'bar'
-    assert_equal 'foo', c.get_next(chan, group)
-    assert_equal 'bar', c.get_next(chan, group)
-    assert_equal 'bam', c.get_next(chan, group)
-    assert_equal '',    c.get_next(chan, group)
+    assert_equal 'foo', c.get_next(group)
+    assert_equal 'bar', c.get_next(group)
+    assert_equal 'bam', c.get_next(group)
+    assert_equal '',    c.get_next(group)
   end
 
   def test_get_next_n
     chan = 'test_get_next_n'
-    c = Client.new 'localhost'
-    c.create_channel chan
-    100.times {|i| c.post chan, i.to_s}
+    c = Client.new 'localhost', chan
+    c.create_channel
+    100.times {|i| c.post i.to_s}
 
     10.times do |j|
-      r = c.get_next_n chan, 10
+      r = c.get_next_n 10
       r.each_with_index do |f, i|
         assert_equal((j*10 + i)+1,                           f["id"])
         assert_equal(Digest::MD5.hexdigest((j*10 + i).to_s), f["hash"])
@@ -205,10 +209,10 @@ class TestConveyorServer < Test::Unit::TestCase
       end
     end
     
-    100.times {|i| c.post chan, i.to_s}
+    100.times {|i| c.post i.to_s}
 
     10.times do |j|
-      r = c.get_next_n chan, 10
+      r = c.get_next_n 10
       r.each_with_index do |f, i|
         assert_equal(Digest::MD5.hexdigest((j*10 + i).to_s), f["hash"])
         assert_equal((100 + j*10 + i)+1,                     f["id"])
@@ -219,31 +223,57 @@ class TestConveyorServer < Test::Unit::TestCase
 
   def test_get_next_n_by_group
     chan = 'test_get_next_n_by_group'
-    c = Client.new 'localhost'
-    c.create_channel chan
-    100.times {|i| c.post chan, i.to_s}
+    c = Client.new 'localhost', chan
+    c.create_channel
+    100.times {|i| c.post i.to_s}
 
     10.times do |j|
-      r = c.get_next_n chan, 10, 'foo'
+      r = c.get_next_n 10, 'foo'
       r.each_with_index do |f, i|
         assert_equal(Digest::MD5.hexdigest((j*10 + i).to_s), f[0]["hash"])
-        assert_equal((j*10 + i).to_s.length,                 f[0]["length"])
         assert_equal((j*10 + i)+1,                           f[0]["id"])
         assert_equal((j*10 + i).to_s,                        f[1])
       end
     end
 
-    assert_equal [], c.get_next_n(chan, 10, 'foo')
+    assert_equal [], c.get_next_n(10, 'foo')
 
     10.times do |j|
-      r = c.get_next_n chan, 10, 'bar'
+      r = c.get_next_n 10, 'bar'
       r.each_with_index do |f, i|
         assert_equal Digest::MD5.hexdigest((j*10 + i).to_s), f[0]["hash"]
-        assert_equal((j*10 + i).to_s.length,                 f[0]["length"])
         assert_equal((j*10 + i)+1,                           f[0]["id"])
         assert_equal((j*10 + i).to_s,                        f[1])
       end
     end
-    assert_equal [], c.get_next_n(chan, 10, 'bar')
+    assert_equal [], c.get_next_n(10, 'bar')
+  end
+
+  def test_delete
+    chan = "test_delete"
+    Net::HTTP.start("localhost", 8011) do |h|
+      r = h.put("/channels/#{chan}", "")
+      assert_kind_of Net::HTTPCreated, r
+      10.times {|i| h.post("/channels/#{chan}", i.to_s, {'Content-Type' => 'application/octet-stream', 'Date' => Time.now.gmtime.to_s})}
+      10.times {|i| assert_equal(i.to_s, h.get("/channels/#{chan}?next").body)}
+      10.times {|i| assert_equal(i.to_s, h.get("/channels/#{chan}/#{i+1}").body)}
+      r = h.delete("/channels/#{chan}")
+      assert_kind_of Net::HTTPOK, r
+      
+      r = h.put("/channels/#{chan}", "")
+      assert_kind_of Net::HTTPCreated, r
+      10.times {|i| h.post("/channels/#{chan}", i.to_s, {'Content-Type' => 'application/octet-stream', 'Date' => Time.now.gmtime.to_s})}
+      10.times {|i| assert_equal(i.to_s, h.get("/channels/#{chan}?next").body)}
+      10.times {|i| assert_equal(i.to_s, h.get("/channels/#{chan}/#{i+1}").body)}
+    end
+  end
+
+  def test_autocreate_channel
+    chan = "test_autocreate_channel"
+    c = Client.new 'localhost', chan
+    c.post 'foo'
+    assert_equal 'foo', c.get_next
   end
-end
+
+end
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: conveyor
 version: !ruby/object:Gem::Version 
-  version: 0.1.4
+  version: 0.2.0
 platform: ruby
 authors: 
 - Ryan King
@@ -30,11 +30,11 @@ cert_chain:
   Zls3y84CmyAEGg==
   -----END CERTIFICATE-----
 
-date: 2008-02-19 00:00:00 -08:00
+date: 2008-02-26 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: mongrel
+  name: thin
   version_requirement: 
   version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
@@ -73,6 +73,7 @@ description: "* Like TiVo for your data * A distributed rewindable multi-queue"
 email: ryan@theryanking.com
 executables: 
 - conveyor
+- conveyor-upgrade
 extensions: []
 
 extra_rdoc_files: 
@@ -86,6 +87,7 @@ files:
 - README.txt
 - Rakefile
 - bin/conveyor
+- bin/conveyor-upgrade
 - docs/file-formats.mkd
 - docs/protocol.mkd
 - lib/conveyor.rb
@@ -93,12 +95,12 @@ files:
 - lib/conveyor/channel.rb
 - lib/conveyor/client.rb
 - lib/conveyor/server.rb
+- lib/conveyor/upgrader.rb
 - lib/priority_queue.rb
 - test/rand.rb
 - test/test_channel.rb
 - test/test_feeder-ng.rb
 - test/test_priority_queue.rb
-- test/test_replicated_channel.rb
 - test/test_server.rb
 has_rdoc: true
 homepage: by Ryan King (http://theryanking.com)