conveyor 0.1.1 → 0.1.2

data/History.txt

@@ -1,3 +1,6 @@
+== 0.1.2 / 2008-02-05
+* various bugfixes
+
 == 0.1.1 / 2008-01-31
 
 * Fix TCP binding.

data/Manifest.txt

@@ -9,6 +9,7 @@ docs/protocol.mkd
 lib/conveyor.rb
 lib/conveyor/base_channel.rb
 lib/conveyor/channel.rb
+lib/conveyor/client.rb
 lib/conveyor/server.rb
 lib/priority_queue.rb
 test/rand.rb

data/Rakefile

@@ -8,8 +8,8 @@ Hoe.new('conveyor', Conveyor::VERSION) do |p|
   p.rubyforge_name = 'conveyor'
   p.author = 'Ryan King'
   p.email = 'ryan@theryanking.com'
+  p.remote_rdoc_dir = ''
   p.extra_deps << ['mongrel']
-  p.extra_deps << ['activesupport']
   p.extra_deps << ['json']
 end
 

data/bin/conveyor

@@ -1,11 +1,42 @@
 #!/usr/bin/env ruby -KU
 
-unless ARGV.length == 2
-  puts "usage: #{$0} <port> <data directory>"
+require 'optparse'
+require 'rubygems'
+require 'daemons/daemonize'
+
+options = {:port => 8011, :daemonize => false}
+opts = OptionParser.new do |opts|
+  opts.summary_indent = " "
+  opts.banner = "usage: #{$0} [options] <data directory>"
+
+  opts.separator ""
+  opts.separator "Options:"
+  opts.on("-p PORT", Integer, "HTTP port number. Default: 8011") do |port|
+    options[:port] = port
+  end
+  
+  opts.on("-l LOGDIRECTORY", "Directory for log files.") do |log_dir|
+    options[:log_directory] = log_dir
+  end
+  
+  opts.on("-d", "Daemonize.") do |d|
+    options[:daemonize] = d
+  end
+end
+
+opts.parse!
+p options
+unless ARGV.length == 1
+  puts opts.help
   exit
 end
 
 $: << 'lib'
 
 require 'conveyor/server'
-Conveyor::Server.new('0.0.0.0', ARGV[0].to_i, ARGV[1]).run.join
+
+if options[:daemonize]
+  Daemonize.daemonize
+end
+
+Conveyor::Server.new('0.0.0.0', options[:port], ARGV[0], options[:log_directory]).run.join

data/docs/file-formats.mkd

@@ -27,3 +27,13 @@ notes:
 * 1 is the filename
 * assuming a lucene-style directory of datafiles + ToC/index
 * given that the files are written sequentially we can avoid writing every entry to the index file (as long as you write the first and last entry to the index). At most this means you have to read n entries, where n is the gap between index entries. Given that most clients will have persistent connections and be reading sequentially, we can do some clever things on the server side to make this really efficient (basically meaning you'll only have to pay that penalty on the first item you read).
+
+== LOG FILES
+
+When given -l LOGDIR, conveyor will write a log file with this format:
+
+  IP HTTP_VERB REQUEST_URI STATUS ID LENGTH HASH
+  
+(In addition to the formatting that Logger adds.)
+
+Not every operation has every bit of that data, but optional bits are on the right.

data/docs/protocol.mkd

@@ -55,9 +55,17 @@ If this is called for the first time, it will return the first item in the chann
 ### Get Next by Group (Multi-consumer queue) ###
 
 Request
-: GET /channels/{channel name}?next&group=foo
+: GET /channels/{channel name}?next&group={group name}
 
 Response
 : success: 200
 
-If this is called for the first time, it will return the first item in the channel. Otherwise it will return the next item.
+If this is called for the first time, it will return the first item in the channel. Otherwise it will return the next item.
+
+### Rewinding to id ###
+
+Request
+: POST /channels/{channel name}?rewind_id={id}
+
+Response
+:success: 200

data/lib/conveyor.rb

@@ -1,4 +1,4 @@
 module Conveyor
-  VERSION = '0.1.1'
+  VERSION = '0.1.2'
   QUALITY = 'alpha'
 end

data/lib/conveyor/base_channel.rb

@@ -1,6 +1,6 @@
 require 'digest/md5'
 require 'rubygems'
-require 'active_support/core_ext/date/conversions'
+require 'time'
 require 'priority_queue'
 
 module Conveyor
@@ -16,7 +16,7 @@ module Conveyor
       @directory               = directory
       @data_files              = []
       @index = []
-      @iterator = 0
+      @iterator = 1
 
       if File.exists?(@directory)
         if !File.directory?(@directory)
@@ -45,7 +45,7 @@ module Conveyor
     end
     
     def inspect
-      "<#{self.class} dir:'#{@directory.to_s}' last_id:#{@last_id}>"
+      "<#{self.class} dir:'#{@directory.to_s}' last_id:#{@last_id} iterator:#{@iterator}>"
     end
     
     def pick_bucket i

data/lib/conveyor/channel.rb

@@ -29,7 +29,7 @@ module Conveyor
       Dir.glob(File.join(@directory, 'iterator-*')) do |i|
         g = i.split(%r{/}).last.match(%r{iterator-(.*)}).captures[0]
         @group_iterators_files[g] = File.open(i, 'r+')
-        @group_iterators[g] = 0
+        @group_iterators[g] = 1
         @group_iterators_files[g].each_line do |line|
           @group_iterators[g] = line.to_i
         end
@@ -46,20 +46,24 @@ module Conveyor
     def get_next
       r = nil
       Thread.exclusive do
-        @iterator += 1 # TODO make sure this is lower than @last_id
-        r = get(@iterator)
-        @iterator_file.write("#{@iterator}\n")
+        if @iterator <= @last_id
+          r = get(@iterator)
+          @iterator += 1
+          @iterator_file.write("#{@iterator}\n")
+          r
+        else
+          nil
+        end
       end
-      r
     end
 
     # Returns the next item for +group+. If +group+ hasn't been seen before, the first item is returned.
     def get_next_by_group group
       r = nil
       Thread.exclusive do
-        @group_iterators[group] = 0 unless @group_iterators.key?(group)
-        @group_iterators[group] += 1
+        @group_iterators[group] = 1 unless @group_iterators.key?(group)
         r = get(@group_iterators[group])
+        @group_iterators[group] += 1
         group_iterators_file(group) do |f|
           f.write("#{@group_iterators[group]}\n")
         end
@@ -79,6 +83,25 @@ module Conveyor
       }
     end
 
+    def rewind *opts
+      opts = opts.first
+      if opts.key?(:id)
+        if opts.key?(:group)
+          Thread.exclusive do
+            @group_iterators[opts[:group]] = opts[:id].to_i
+            group_iterators_file(opts[:group]) do |f|
+              f.write("#{@group_iterators[opts[:group]]}\n")
+            end
+          end
+        else
+          Thread.exclusive do
+            @iterator = opts[:id].to_i
+            @iterator_file.write("#{@iterator}\n")
+          end
+        end
+      end
+    end
+    
     private
     
     def group_iterators_file group

data/lib/conveyor/client.rb

@@ -0,0 +1,35 @@
+require 'net/http'
+
+module Conveyor
+  class Client
+    def initialize host, port = 8011
+      @host = host
+      @port = port
+      connect!
+    end
+    
+    def connect!
+      @conn = Net::HTTP.start(@host, @port)
+    end
+
+    def create_channel channel_name
+      @conn.put("/channels/#{channel_name}", nil, {'Content-Type' => 'application/octet-stream'})
+    end
+
+    def post channel_name, content
+      @conn.post("/channels/#{channel_name}", content, {'Content-Type' => 'application/octet-stream', 'Date' => Time.now.to_s})
+    end
+    
+    def get channel_name, id
+      @conn.get("/channels/#{channel_name}/#{id}").body
+    end
+
+    def get_next channel_name, group=nil
+      if group
+        @conn.get("/channels/#{channel_name}?next&group=#{group}").body
+      else
+        @conn.get("/channels/#{channel_name}?next").body
+      end
+    end
+  end
+end

data/lib/conveyor/server.rb

@@ -3,6 +3,7 @@ require 'mongrel'
 require 'conveyor/channel'
 require 'fileutils'
 require 'json'
+require 'logger'
 
 class Mongrel::HttpRequest
   def put?
@@ -30,8 +31,14 @@ module Conveyor
     # A Mongrel handler for multiple Conveyor Channels.
     class ChannelsHandler < Mongrel::HttpHandler
 
-      def initialize data_directory
+      def initialize data_directory, log_directory=nil
         @data_directory = data_directory
+        if log_directory
+          @logger = Logger.new File.join(log_directory, 'conveyor.log')
+        else
+          @logger = Logger.new '/dev/null'
+        end
+
         @channels = {}
         Dir.entries(@data_directory).each do |e|
           if !['.', '..'].include?(e) && File.directory?(File.join(@data_directory, e))
@@ -44,28 +51,51 @@ module Conveyor
         @channels[channel_name] = Conveyor::Channel.new(File.join(@data_directory, channel_name))
       end
 
+      def i str
+        @logger.info str
+      end
+
       def process request, response
         if request.put? && m = request.path_match(%r{/channels/(.*)})
           if Channel.valid_channel_name?(m.captures[0])
-            create_new_channel m.captures[0]
-            response.start(201) do |head, out|
-              out.write("created channel #{m.captures[0]}")
+            if !@channels.key?(m.captures[0])
+              create_new_channel m.captures[0]
+              response.start(201) do |head, out|
+                out.write("created channel #{m.captures[0]}")
+              end
+              i "#{request.params["REMOTE_ADDR"]} PUT #{request.params["REQUEST_PATH"]} 201"
+            else
+              response.start(202) do |head, out|
+                out.write("channel already exists. didn't do anything")
+              end
+              i "#{request.params["REMOTE_ADDR"]} PUT #{request.params["REQUEST_PATH"]} 202"
             end
           else
             response.start(406) do |head, out|
               out.write("invalid channel name. must match #{Channel::NAME_PATTERN}")
+              i "#{request.params["REMOTE_ADDR"]} GET #{request.params["REQUEST_PATH"]} 406"
             end
           end
         elsif request.post? && m = request.path_match(%r{/channels/(.*)})
           if @channels.key?(m.captures[0])
-            if request.params.include?('HTTP_DATE') && d = Time.parse(request.params['HTTP_DATE'])
-              id = @channels[m.captures[0]].post(request.body.read)
-              response.start(202) do |head, out|
-                head["Location"] = "/channels/#{m.captures[0]}/#{id}"
+            params = Mongrel::HttpRequest.query_parse(request.params['QUERY_STRING'])
+            if params.include?('rewind_id')
+              @channels[m.captures[0]].rewind(:id => params['rewind_id']).to_i # TODO make sure this is an integer
+              response.start(200) do |head, out|
+                out.write "iterator rewound to #{params['rewind_id']}"
               end
             else
-              response.start(400) do |head, out|
-                out.write "A valid Date header is required for all POSTs."
+              if request.params.include?('HTTP_DATE') && d = Time.parse(request.params['HTTP_DATE'])
+                id = @channels[m.captures[0]].post(request.body.read)
+                response.start(202) do |head, out|
+                  head["Location"] = "/channels/#{m.captures[0]}/#{id}"
+                end
+                i "#{request.params["REMOTE_ADDR"]} GET #{request.params["REQUEST_PATH"]} 202"
+              else
+                response.start(400) do |head, out|
+                  out.write "A valid Date header is required for all POSTs."
+                end
+                i "#{request.params["REMOTE_ADDR"]} GET #{request.params["REQUEST_PATH"]} 400"
               end
             end
           end
@@ -105,6 +135,7 @@ module Conveyor
               head['Last-Modified']    = Time.parse(headers[:time]).gmtime.to_s
               out.write content
             end
+            i "#{request.params["REMOTE_ADDR"]} GET #{request.params["REQUEST_PATH"]} 200 #{headers[:id]} #{headers[:length]} #{headers[:hash]}"
           end
           
         end
@@ -113,9 +144,9 @@ module Conveyor
 
     # +host+ and +port+ are passed along to Mongrel::HttpServer for TCP binding. +data_directory+ is used to store
     # all channel data and should be created before intializing a Server.
-    def initialize(host, port, data_directory)
+    def initialize(host, port, data_directory, log_directory = nil)
       super(host, port)
-      ch = ChannelsHandler.new(data_directory)
+      ch = ChannelsHandler.new(data_directory, log_directory)
       register("/channels", ch)
     end
   end

data/test/test_channel.rb

@@ -66,6 +66,7 @@ class TestConveyorChannel < Test::Unit::TestCase
     assert_equal 'bar', c.get_next[1]
     assert_equal 'bam', c.get_next[1]
     assert_equal nil, c.get_next
+    assert_equal 4, c.status[:iterator][:position]
   end
 
   def test_get_next_interupted
@@ -137,10 +138,40 @@ class TestConveyorChannel < Test::Unit::TestCase
       :data_files => [
         {:path => '/tmp/bar/0', :bytes => 210}
         ],
-      :iterator => {:position => 0},
+      :iterator => {:position => 1},
       :iterator_groups => {}
     }
     
     assert_equal(status, c.status)
   end
+
+  def test_rewind
+    FileUtils.rm_r('/tmp/bar') rescue nil
+    c = Channel.new('/tmp/bar')
+    c.post 'foo'
+
+    assert_equal 'foo', c.get_next[1]
+    c.rewind(:id => 1)
+    assert_equal 'foo', c.get_next[1]
+    c.rewind(:id => 1)
+
+    d = Channel.new('/tmp/bar')
+    assert_equal 'foo', d.get_next[1]
+  end
+  
+  def test_group_rewind
+    FileUtils.rm_r('/tmp/bar') rescue nil
+    c = Channel.new('/tmp/bar')
+    c.post 'foo'
+    
+    assert_equal 'foo', c.get_next_by_group('bar')[1]
+    c.rewind(:id => 1, :group => 'bar')
+    assert_equal 'foo', c.get_next_by_group('bar')[1]
+    c.rewind(:id => 1, :group => 'bar')
+
+    d = Channel.new('/tmp/bar')
+    assert_equal 'foo', d.get_next_by_group('bar')[1]
+  end
+  
+  
 end

data/test/test_server.rb

@@ -6,7 +6,7 @@ class TestConveyorServer < Test::Unit::TestCase
   def setup
     FileUtils.rm_r('/tmp/asdf') rescue nil
     FileUtils.mkdir('/tmp/asdf')
-    @server = Conveyor::Server.new("127.0.0.1", 8888, '/tmp/asdf')
+    @server = Conveyor::Server.new("127.0.0.1", 8011, '/tmp/asdf')
     @server.run
   end
   
@@ -15,14 +15,14 @@ class TestConveyorServer < Test::Unit::TestCase
   end
   
   def test_channels
-    Net::HTTP.start("localhost", 8888) do |h|
+    Net::HTTP.start("localhost", 8011) do |h|
       req = h.get('/channels')
       assert_equal Net::HTTPOK, req.class
     end
   end
   
   def test_create_channel
-    Net::HTTP.start('localhost', 8888) do |h|
+    Net::HTTP.start('localhost', 8011) do |h|
       req = h.put('/channels/foo', '', {'Content-Type' => 'application/octet-stream'})
       assert_equal Net::HTTPCreated, req.class
 
@@ -32,7 +32,7 @@ class TestConveyorServer < Test::Unit::TestCase
   end
   
   def test_post
-    Net::HTTP.start('localhost', 8888) do |h|
+    Net::HTTP.start('localhost', 8011) do |h|
       req = h.put('/channels/bar', '', {'Content-Type' => 'application/octet-stream'})
       assert_equal Net::HTTPCreated, req.class
 
@@ -58,7 +58,7 @@ class TestConveyorServer < Test::Unit::TestCase
   end
   
   def test_invalid_channel
-    Net::HTTP.start('localhost', 8888) do |h|
+    Net::HTTP.start('localhost', 8011) do |h|
       req = h.put('/channels/|', '', {'Content-Type' => 'application/octet-stream'})
       assert_equal Net::HTTPNotAcceptable, req.class
     end
@@ -66,7 +66,7 @@ class TestConveyorServer < Test::Unit::TestCase
   end
 
   def test_get_next
-    Net::HTTP.start('localhost', 8888) do |h|
+    Net::HTTP.start('localhost', 8011) do |h|
       req = h.put('/channels/bar', '', {'Content-Type' => 'application/octet-stream'})
       assert_equal Net::HTTPCreated, req.class
 
@@ -92,7 +92,7 @@ class TestConveyorServer < Test::Unit::TestCase
   end
 
   def test_status
-    Net::HTTP.start('localhost', 8888) do |h|
+    Net::HTTP.start('localhost', 8011) do |h|
       req = h.put('/channels/bar', '', {'Content-Type' => 'application/octet-stream'})
       assert_equal Net::HTTPCreated, req.class
 
@@ -111,12 +111,45 @@ class TestConveyorServer < Test::Unit::TestCase
         "index"=>{"size"=>1},
         "directory"=>"/tmp/asdf/bar",
         "data_files"=>[{"path"=>"/tmp/asdf/bar/0","bytes"=>139}],
-        "iterator"=>{"position"=>0}
+        "iterator"=>{"position"=>1}
       }
       assert_equal json, JSON::parse(req.body)
       
     end
+  end
+
+  def test_rewinding
+    Net::HTTP.start('localhost', 8011) do |h|
+      req = h.put('/channels/bar', '', {'Content-Type' => 'application/octet-stream'})
+      assert_equal Net::HTTPCreated, req.class
+
+      data =
+      ["ZqZyDN2SouQCYEHYS0LuM1XeqsF0MKIbFEBE6xQ972VqEcjs21wJSosvZMWEH1lq5ukTq4Ze"]
+        
+      data.each do |d|
+        req = h.post('/channels/bar', d, {'Content-Type' => 'application/octet-stream', 'Date' => Time.now.to_s})
+        assert_equal Net::HTTPAccepted, req.class
+      end
+
+      req = h.get('/channels/bar?next')
+
+      assert_kind_of Net::HTTPOK, req
+      assert_equal data[0], req.body
+
+      req = h.get('/channels/bar?next')
 
+      assert_kind_of Net::HTTPNotFound, req
 
+      req = h.post('/channels/bar?rewind_id=1', nil)
+      assert_kind_of Net::HTTPOK, req
+
+      req = h.get('/channels/bar?next')
+      
+      assert_kind_of Net::HTTPOK, req
+      assert_equal data[0], req.body
+      
+      req = h.get('/channels/bar?next')
+      assert_kind_of Net::HTTPNotFound, req
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: conveyor
 version: !ruby/object:Gem::Version 
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors: 
 - Ryan King
@@ -30,7 +30,7 @@ cert_chain:
   Zls3y84CmyAEGg==
   -----END CERTIFICATE-----
 
-date: 2008-01-31 00:00:00 -08:00
+date: 2008-02-05 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -42,15 +42,6 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: "0"
     version: 
-- !ruby/object:Gem::Dependency 
-  name: activesupport
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
-    version: 
 - !ruby/object:Gem::Dependency 
   name: json
   version_requirement: 
@@ -91,6 +82,7 @@ files:
 - lib/conveyor.rb
 - lib/conveyor/base_channel.rb
 - lib/conveyor/channel.rb
+- lib/conveyor/client.rb
 - lib/conveyor/server.rb
 - lib/priority_queue.rb
 - test/rand.rb