conveyor 0.1.3 → 0.1.4

data/History.txt

@@ -1,3 +1,7 @@
+== 0.1.4 / 2008-02-19
+* support for getting multiple items at once
+* rewinding implemented for group iterators
+
 == 0.1.2 / 2008-02-05
 * various bugfixes
 

data/docs/protocol.mkd

@@ -68,4 +68,22 @@ Request
 : POST /channels/{channel name}?rewind_id={id}
 
 Response
-:success: 200
+:success: 200
+
+### Get next n ###
+
+Request
+: GET /channels/{channel name}?next&n={n}
+
+Response
+: JSON array of objects, which have the keys 'id', 'hash' and 'data'
+
+
+### Get next n for group ###
+
+Request
+: GET /channels/{channel name}?next&n={n}&group={group}
+
+Response
+: JSON array of objects, which have the keys 'id', 'hash' and 'data'
+

data/lib/conveyor.rb

@@ -1,4 +1,4 @@
 module Conveyor
-  VERSION = '0.1.3'
+  VERSION = '0.1.4'
   QUALITY = 'alpha'
 end

data/lib/conveyor/channel.rb

@@ -62,10 +62,42 @@ module Conveyor
       r = nil
       Thread.exclusive do
         @group_iterators[group] = 1 unless @group_iterators.key?(group)
-        r = get(@group_iterators[group])
-        @group_iterators[group] += 1
-        group_iterators_file(group) do |f|
-          f.write("#{@group_iterators[group]}\n")
+        if @iterator <= @last_id
+          r = get(@group_iterators[group])
+          @group_iterators[group] += 1
+          group_iterators_file(group) do |f|
+            f.write("#{@group_iterators[group]}\n")
+          end
+        else
+          nil
+        end
+      end
+      r
+    end
+
+    def get_next_n n
+      r = []
+      Thread.exclusive do
+        while r.length < n && @iterator <= @last_id
+          r << get(@iterator)
+          @iterator += 1
+          @iterator_file.write("#{@iterator}\n")
+          r
+        end
+      end
+      r
+    end
+    
+    def get_next_n_by_group n, group
+      r = []
+      Thread.exclusive do
+        @group_iterators[group] = 1 unless @group_iterators.key?(group)
+        while r.length < n && @group_iterators[group] < @last_id
+          r << get(@group_iterators[group])
+          @group_iterators[group] += 1
+          group_iterators_file(group) do |f|
+            f.write("#{@group_iterators[group]}\n")
+          end
         end
       end
       r
@@ -79,7 +111,7 @@ module Conveyor
         },
         :data_files => @data_files.collect{|f| {:path => f.path, :bytes => File.size(f.path)}},
         :iterator => {:position => @iterator},
-        :iterator_groups => @group_iterators.inject({}){|k,v, m| m[k] = v; m}
+        :iterator_groups => @group_iterators.inject({}){|m,(k,v)| m[k] = v; m}
       }
     end
 

data/lib/conveyor/client.rb

@@ -31,5 +31,25 @@ module Conveyor
         @conn.get("/channels/#{channel_name}?next").body
       end
     end
+
+    def channel_status channel_name
+      JSON::parse(@conn.get("/channels/#{channel_name}").body)
+    end
+
+    def get_next_n channel_name, n = 10, group = nil
+      if group
+        JSON.parse(@conn.get("/channels/#{channel_name}?next&n=#{n}&group=#{group}").body)
+      else
+        JSON.parse(@conn.get("/channels/#{channel_name}?next&n=#{n}").body)
+      end
+    end
+    
+    def rewind channel_name, id, group=nil
+      if group
+        @conn.post("/channels/#{channel_name}?rewind_id=#{id}&group=#{group}", nil)
+      else
+        @conn.post("/channels/#{channel_name}?rewind_id=#{id}", nil)
+      end
+    end
   end
 end

data/lib/conveyor/server.rb

@@ -68,7 +68,7 @@ module Conveyor
               response.start(202) do |head, out|
                 out.write("channel already exists. didn't do anything")
               end
-              i "#{request.params["REMOTE_ADDR"]} PUT #{request.params["REQUEST_PATH"]} 202"
+              i "#{request.params["REMOTE_ADDR"]} PUT #{request.params["REQUEST_PATH"]}   "
             end
           else
             response.start(406) do |head, out|
@@ -79,10 +79,17 @@ module Conveyor
         elsif request.post? && m = request.path_match(%r{/channels/(.*)})
           if @channels.key?(m.captures[0])
             params = Mongrel::HttpRequest.query_parse(request.params['QUERY_STRING'])
-            if params.include?('rewind_id')
-              @channels[m.captures[0]].rewind(:id => params['rewind_id']).to_i # TODO make sure this is an integer
-              response.start(200) do |head, out|
-                out.write "iterator rewound to #{params['rewind_id']}"
+            if params.key?('rewind_id')
+              if params['group']
+                @channels[m.captures[0]].rewind(:id => params['rewind_id'], :group => params['group']).to_i # TODO make sure this is an integer
+                response.start(200) do |head, out|
+                  out.write "iterator rewound to #{params['rewind_id']}"
+                end
+              else
+                @channels[m.captures[0]].rewind(:id => params['rewind_id']).to_i # TODO make sure this is an integer
+                response.start(200) do |head, out|
+                  out.write "iterator rewound to #{params['rewind_id']}"
+                end
               end
             else
               if request.params.include?('HTTP_DATE') && d = Time.parse(request.params['HTTP_DATE'])
@@ -90,12 +97,12 @@ module Conveyor
                 response.start(202) do |head, out|
                   head["Location"] = "/channels/#{m.captures[0]}/#{id}"
                 end
-                i "#{request.params["REMOTE_ADDR"]} GET #{request.params["REQUEST_PATH"]} 202"
+                i "#{request.params["REMOTE_ADDR"]} POST #{request.params["REQUEST_PATH"]} 202"
               else
                 response.start(400) do |head, out|
                   out.write "A valid Date header is required for all POSTs."
                 end
-                i "#{request.params["REMOTE_ADDR"]} GET #{request.params["REQUEST_PATH"]} 400"
+                i "#{request.params["REMOTE_ADDR"]} POST #{request.params["REQUEST_PATH"]} 400"
               end
             end
           end
@@ -111,9 +118,20 @@ module Conveyor
               params = Mongrel::HttpRequest.query_parse(request.params['QUERY_STRING'])
               if params.key? 'next'
                 if params.key? 'group'
-                  headers, content = @channels[m.captures[0]].get_next_by_group(params['group'])
+                  if params.key? 'n'
+                    list = @channels[m.captures[0]].get_next_n_by_group(params['n'].to_i, params['group'])
+                  else
+                    headers, content = @channels[m.captures[0]].get_next_by_group(params['group'])
+                  end
                 else
-                  headers, content = @channels[m.captures[0]].get_next
+                  if params.key? 'n'
+                    list = @channels[m.captures[0]].get_next_n(params['n'].to_i)
+                    list = list.map do |i|
+                      {:data => i[1], :hash => i[0][:hash], :id => i[0][:id]}
+                    end
+                  else
+                    headers, content = @channels[m.captures[0]].get_next
+                  end
                 end
               else
                 response.start(200) do |head, out|
@@ -136,6 +154,10 @@ module Conveyor
               out.write content
             end
             i "#{request.params["REMOTE_ADDR"]} GET #{request.params["REQUEST_PATH"]} 200 #{headers[:id]} #{headers[:length]} #{headers[:hash]}"
+          elsif list
+            response.start(200) do |head, out|
+              out.write list.to_json
+            end
           end
           
         end

data/test/test_channel.rb

@@ -180,4 +180,62 @@ class TestConveyorChannel < Test::Unit::TestCase
     assert BaseChannel.valid_channel_name?('-')
     assert BaseChannel.valid_channel_name?('_')
   end
+
+  def test_get_next_n
+    FileUtils.rm_r '/tmp/asdfasdf' rescue nil
+    c = Conveyor::Channel.new '/tmp/asdfasdf'
+    100.times {|i| c.post i.to_s}
+
+    12.times do |j|
+      r = c.get_next_n 10
+      r.each_with_index do |f, i|
+        assert_equal Digest::MD5.hexdigest((j*10 + i).to_s), f[0][:hash]
+        assert_equal((j*10 + i).to_s.length,                 f[0][:length])
+        assert_equal((j*10 + i)+1,                           f[0][:id])
+        assert_equal((j*10 + i).to_s,                        f[1])
+      end
+    end
+    
+    100.times {|i| c.post i.to_s}
+
+    12.times do |j|
+      r = c.get_next_n 10
+      r.each_with_index do |f, i|
+        assert_equal Digest::MD5.hexdigest((j*10 + i).to_s), f[0][:hash]
+        assert_equal((j*10 + i).to_s.length,                 f[0][:length])
+        assert_equal((100 + j*10 + i)+1,                     f[0][:id])
+        assert_equal((j*10 + i).to_s,                        f[1])
+      end
+    end
+  end
+
+  def test_get_next_n_by_group
+    FileUtils.rm_r '/tmp/asdfasdf'
+    c = Conveyor::Channel.new '/tmp/asdfasdf'
+    100.times {|i| c.post i.to_s}
+
+    10.times do |j|
+      r = c.get_next_n_by_group 10, 'foo'
+      r.each_with_index do |f, i|
+        assert_equal Digest::MD5.hexdigest((j*10 + i).to_s), f[0][:hash]
+        assert_equal((j*10 + i).to_s.length,                 f[0][:length])
+        assert_equal((j*10 + i)+1,                           f[0][:id])
+        assert_equal((j*10 + i).to_s,                        f[1])
+      end
+    end
+
+    assert_equal [], c.get_next_n_by_group(10, 'foo')
+
+    10.times do |j|
+      r = c.get_next_n_by_group 10, 'bar'
+      r.each_with_index do |f, i|
+        assert_equal Digest::MD5.hexdigest((j*10 + i).to_s), f[0][:hash]
+        assert_equal((j*10 + i).to_s.length,                 f[0][:length])
+        assert_equal((j*10 + i)+1,                           f[0][:id])
+        assert_equal((j*10 + i).to_s,                        f[1])
+      end
+    end
+    assert_equal [], c.get_next_n_by_group(10, 'bar')
+  end
+  
 end

data/test/test_priority_queue.rb

@@ -9,7 +9,7 @@ class TestPriorityQueue < Test::Unit::TestCase
 
     pq << 1 << 2 << -1
 
-    assert_equal -1, pq.pop
+    assert_equal(-1, pq.pop)
     assert_equal 1, pq.pop
     assert_equal 2, pq.pop
   end

data/test/test_server.rb

@@ -1,8 +1,10 @@
 require "test/unit"
 require "conveyor/server"
 require 'net/http'
+require 'conveyor/client'
 
 class TestConveyorServer < Test::Unit::TestCase
+  include Conveyor
   def setup
     FileUtils.rm_r('/tmp/asdf') rescue nil
     FileUtils.mkdir('/tmp/asdf')
@@ -152,4 +154,96 @@ class TestConveyorServer < Test::Unit::TestCase
       assert_kind_of Net::HTTPNotFound, req
     end
   end
+
+  def test_group_rewind
+    chan = 'test_group_rewind'
+    c = Client.new 'localhost'
+    c.create_channel chan
+    c.post chan, 'foo'
+    
+    assert_equal 'foo', c.get_next(chan, 'bar')
+    c.rewind(chan, 1, 'bar')
+    assert_equal 'foo', c.get_next(chan, 'bar')
+    c.rewind(chan, 1, 'bar')
+  end
+
+
+  def test_get_next_by_group
+    c = Conveyor::Client.new 'localhost'
+    chan = 'asdf'
+    c.create_channel chan
+    c.post chan, 'foo'
+    c.post chan, 'bar'
+    c.post chan, 'bam'
+
+    group = 'bam'
+
+    assert_equal 'foo', c.get_next(chan, group)
+    assert_equal 'bar', c.get_next(chan, group)
+    assert_equal 'bam', c.get_next(chan, group)
+    assert_equal '',   c.get_next(chan, group)
+
+    group = 'bar'
+    assert_equal 'foo', c.get_next(chan, group)
+    assert_equal 'bar', c.get_next(chan, group)
+    assert_equal 'bam', c.get_next(chan, group)
+    assert_equal '',    c.get_next(chan, group)
+  end
+
+  def test_get_next_n
+    chan = 'test_get_next_n'
+    c = Client.new 'localhost'
+    c.create_channel chan
+    100.times {|i| c.post chan, i.to_s}
+
+    10.times do |j|
+      r = c.get_next_n chan, 10
+      r.each_with_index do |f, i|
+        assert_equal((j*10 + i)+1,                           f["id"])
+        assert_equal(Digest::MD5.hexdigest((j*10 + i).to_s), f["hash"])
+        assert_equal((j*10 + i).to_s,                        f["data"])
+      end
+    end
+    
+    100.times {|i| c.post chan, i.to_s}
+
+    10.times do |j|
+      r = c.get_next_n chan, 10
+      r.each_with_index do |f, i|
+        assert_equal(Digest::MD5.hexdigest((j*10 + i).to_s), f["hash"])
+        assert_equal((100 + j*10 + i)+1,                     f["id"])
+        assert_equal((j*10 + i).to_s,                        f["data"])
+      end
+    end
+  end
+
+  def test_get_next_n_by_group
+    chan = 'test_get_next_n_by_group'
+    c = Client.new 'localhost'
+    c.create_channel chan
+    100.times {|i| c.post chan, i.to_s}
+
+    10.times do |j|
+      r = c.get_next_n chan, 10, 'foo'
+      r.each_with_index do |f, i|
+        assert_equal(Digest::MD5.hexdigest((j*10 + i).to_s), f[0]["hash"])
+        assert_equal((j*10 + i).to_s.length,                 f[0]["length"])
+        assert_equal((j*10 + i)+1,                           f[0]["id"])
+        assert_equal((j*10 + i).to_s,                        f[1])
+      end
+    end
+
+    assert_equal [], c.get_next_n(chan, 10, 'foo')
+
+    10.times do |j|
+      r = c.get_next_n chan, 10, 'bar'
+      r.each_with_index do |f, i|
+        assert_equal Digest::MD5.hexdigest((j*10 + i).to_s), f[0]["hash"]
+        assert_equal((j*10 + i).to_s.length,                 f[0]["length"])
+        assert_equal((j*10 + i)+1,                           f[0]["id"])
+        assert_equal((j*10 + i).to_s,                        f[1])
+      end
+    end
+    assert_equal [], c.get_next_n(chan, 10, 'bar')
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: conveyor
 version: !ruby/object:Gem::Version 
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors: 
 - Ryan King
@@ -30,7 +30,7 @@ cert_chain:
   Zls3y84CmyAEGg==
   -----END CERTIFICATE-----
 
-date: 2008-02-15 00:00:00 -08:00
+date: 2008-02-19 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency