conveyor 0.1.0

data/History.txt

@@ -0,0 +1,5 @@
+== 0.1.0 / 2008-01-29
+
+* Basic Channel implementation
+* Basic HTTP interface
+

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2008 Ryan King
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/Manifest.txt

@@ -0,0 +1,19 @@
+History.txt
+LICENSE
+Manifest.txt
+README.txt
+Rakefile
+bin/conveyor
+docs/file-formats.mkd
+docs/protocol.mkd
+lib/conveyor.rb
+lib/conveyor/base_channel.rb
+lib/conveyor/channel.rb
+lib/conveyor/server.rb
+lib/priority_queue.rb
+test/rand.rb
+test/test_channel.rb
+test/test_feeder-ng.rb
+test/test_priority_queue.rb
+test/test_replicated_channel.rb
+test/test_server.rb

data/README.txt

@@ -0,0 +1,61 @@
+= Conveyor
+
+by Ryan King (http://theryanking.com)
+
+== Description
+
+* Like TiVo for your data
+* A distributed rewindable multi-queue
+
+== Overview
+
+A Conveyor server provides an HTTP interface that allows for POSTing and GETing items in streams called Channels.
+
+POSTing is simple: you add an item to the channel and it gets persisted and assigned a sequence number.
+
+Consuming items from a Channel is more flexible, you can:
+
+* consume by id number ("GET /channels/foo/1337")
+* consume by from a global queue ("GET /channels/foo?next")
+* consume from a queue group ("GET /channels/foo?next&group=bar")
+  * this allows multiple groups of consumers to each have what appears to them to be a queue.
+
+The payload for all of these is a stream of bytes. Conveyor will stream it back exactly as it was given.
+
+Ok, actually rewinding functionality is still TODO, but should be done soon.
+
+== REQUIREMENTS:
+
+* Ruby
+* Mongrel
+* active_support
+
+== INSTALL:
+
+* gem install conveyor
+
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2008 Ryan King
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,19 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+require './lib/conveyor'
+
+Hoe.new('conveyor', Conveyor::VERSION) do |p|
+  p.rubyforge_name = 'conveyor'
+  p.author = 'Ryan King'
+  p.email = 'ryan@theryanking.com'
+  p.summary = 'Like TiVo for your data.'
+  p.description = p.paragraphs_of('README.txt', 2..5).join("\n\n")
+  p.url = p.paragraphs_of('README.txt', 0).first.split(/\n/)[1..-1]
+  p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
+  p.extra_deps << ['mongrel']
+  p.extra_deps << ['activesupport']
+end
+
+# vim: syntax=Ruby

data/bin/conveyor

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby -KU
+
+unless ARGV.length == 2
+  puts "usage: #{$0} <port> <data directory>"
+  exit
+end
+
+$: << 'lib'
+
+require 'conveyor/server'
+Conveyor::Server.new('localhost', ARGV[0], ARGV[1]).run.join

data/docs/file-formats.mkd

@@ -0,0 +1,29 @@
+= FILE FORMATS
+==  DATA FILES
+
+id time offset length hash
+content
+...
+
+contrived example:
+
+1213124 2008-01-05T13:35:32 1234 11 asdfasdfasdfasdfasdfasdfasdfa
+foo bar bam
+
+* space separated line of metadata followed by content
+* delimiter might be useful for sanity checking, but the hash could probably suffice for ensuring that the offset was calculated and persisted properly. We should look at what ARC does here.
+* offset is to beginning of metadata line
+* length doesn't include a trailing \n that separates the content from the next bit of metadata (this might not be necessary)
+
+=== INDEX FILES
+
+id time offset length hash file
+
+contrived example:
+
+1213124 2008-01-05T13:35:32 1234 11 asdfasdfasdfasdfasdfasdfasdfa 1
+
+notes:
+* 1 is the filename
+* assuming a lucene-style directory of datafiles + ToC/index
+* given that the files are written sequentially we can avoid writing every entry to the index file (as long as you write the first and last entry to the index). At most this means you have to read n entries, where n is the gap between index entries. Given that most clients will have persistent connections and be reading sequentially, we can do some clever things on the server side to make this really efficient (basically meaning you'll only have to pay that penalty on the first item you read).

data/docs/protocol.mkd

@@ -0,0 +1,63 @@
+# Conveyor protocol #
+
+## Create a channel ##
+
+Request
+: PUT /channels/{channel name}
+
+Response
+: success: 201, failure: ? 
+
+The simple explanation is that to create a channel you do a PUT operation on the url you want for the channel, which must conform to /channels/\A[a-zA-Z0-9\-]+\Z. In other words, the channel name may only have letters numbers and dashes.
+
+## Post to a channel ##
+Request
+: POST /channels/{channel name}
+: body is the message
+
+Response
+: success: 202, failure: ?
+
+A post to a channel URL with the message in the body.
+
+## Get from channel ##
+
+### Get by id ###
+
+Request
+: GET /channel/{channel name}/{id}
+
+Response
+: success: 200, failure: 404
+
+### Get by datetime ###
+
+NOT IMPLEMENTED YET!
+
+Request
+: GET /channels/{channel name}?at={ISO datetime like 2008-01-11T17:53:59}
+
+Response
+: success: 200
+
+Will return the first entry *after* that datetime.
+
+### Get Next (Queue-like semantics) ###
+
+Request
+: GET /channels/{channel name}?next
+
+Response
+: success: 200
+
+If this is called for the first time, it will return the first item in the channel. Otherwise it will return the next item.
+
+### Get Next by Group (Multi-consumer queue) ###
+
+Request
+: GET /channels/{channel name}?next&group=foo
+
+Response
+: success: 200
+
+If this is called for the first time, it will return the first item in the channel. Otherwise it will return the next item.

data/lib/conveyor.rb

@@ -0,0 +1,4 @@
+module Conveyor
+  VERSION = '0.1.0'
+  QUALITY = 'alpha'
+end

data/lib/conveyor/base_channel.rb

@@ -0,0 +1,118 @@
+require 'digest/md5'
+require 'rubygems'
+require 'active_support/core_ext/date/conversions'
+require 'priority_queue'
+
+module Conveyor
+  # BaseChannel
+  #
+  # Base implementation for channels. Not useful to instantiate directly.
+  class BaseChannel
+
+    NAME_PATTERN = %r{\A[a-zA-Z\-0-9]+\Z}
+    BUCKET_SIZE = 100_000
+
+    def initialize directory
+      @directory               = directory
+      @data_files              = []
+      @index = []
+      @iterator = 0
+
+      if File.exists?(@directory)
+        if !File.directory?(@directory)
+          raise "#{@directory} is not a directory"
+        end
+      else
+        Dir.mkdir(@directory)
+      end
+
+      index_path = File.join(@directory, 'index')
+
+      if File.exists?(index_path) && File.size(index_path) > 0
+        @index_file = File.open(index_path, 'r+')
+
+        @index_file.each_line do |line|
+          @index << parse_headers(line.strip, true)
+          @last_id = @index.last[:id]
+        end
+        @index_file.seek(0, IO::SEEK_END)
+      else
+        @index_file = File.open(index_path, 'a')
+        @last_id = 0
+      end
+      @index_file.sync = true
+
+    end
+    
+    def inspect
+      "<#{self.class} dir:'#{@directory.to_s}' last_id:#{@last_id}>"
+    end
+    
+    def pick_bucket i
+      (i / BUCKET_SIZE).to_i
+    end
+
+    def bucket_file i
+      unless @data_files[i]
+        @data_files[i] = File.open(File.join(@directory, i.to_s), 'a+')
+        @data_files[i].sync = true
+      end
+      yield @data_files[i]
+    end
+
+    def commit data, time = nil
+      Thread.exclusive do
+        i = @last_id + 1
+        t = time || Time.now
+        l = data.length
+        h = Digest::MD5.hexdigest(data)
+        b = pick_bucket(i)
+        header, o = nil
+        bucket_file(b) do |f|
+          f.seek(0, IO::SEEK_END)
+          o = f.pos
+          header = "#{i} #{t.xmlschema} #{o} #{l} #{h}"
+          f.write("#{header}\n" + data + "\n")
+        end
+
+        @last_id = i
+        @index_file.write "#{header} #{b}\n"
+        @index << {:id => i, :time => t, :offset => o, :length => l, :hash => h, :file => b}
+        i
+      end
+    end
+
+    def get id
+      return nil unless id <= @last_id
+      i = @index.find{|e| e[:id] == id}
+      header, content = nil
+      Thread.exclusive do
+        bucket_file(i[:file]) do |f|
+          f.seek i[:offset]
+          header  = f.readline.strip
+          content = f.read(i[:length])
+        end
+      end
+      [parse_headers(header), content]
+    end
+
+    def parse_headers str, index_file=false
+      pattern =  '\A(\d+) (\d{4}\-\d{2}\-\d{2}T\d{2}\:\d{2}\:\d{2}[+\-]\d{2}\:\d{2}) (\d+) (\d+) ([a-f0-9]+)'
+      pattern += ' (\d+)' if index_file
+      pattern += '\Z'
+      m = str.match(Regexp.new(pattern))
+      {
+        :id => m.captures[0].to_i,
+        :time => m.captures[1],
+        :offset => m.captures[2].to_i,
+        :length => m.captures[3].to_i,
+        :hash => m.captures[4], 
+        :file => (index_file ? m.captures[5].to_i : nil)
+      }.reject {|k,v| v == nil}
+    end
+
+    def self.valid_channel_name? name
+      !!name.match(NAME_PATTERN)
+    end
+  end
+end

data/lib/conveyor/channel.rb

@@ -0,0 +1,81 @@
+require 'conveyor/base_channel'
+
+module Conveyor
+  # Channel
+  #
+  # A basic channel.
+  class Channel < BaseChannel
+
+    # If +directory+ doesn't already exist, it will be created during initialization.
+    def initialize directory 
+      @group_iterators         = {}
+      @group_iterators_files   = {}
+      
+      super(directory)
+      
+      iterator_path = File.join(@directory, 'iterator')
+
+      if File.exists?(iterator_path) && File.size(iterator_path) > 0
+        @iterator_file = File.open(iterator_path, 'r+')
+        @iterator_file.each_line do |line|
+          @iterator = line.to_i
+        end
+        @iterator_file.seek(0, IO::SEEK_END)
+      else
+        @iterator_file = File.open(iterator_path, 'a')
+      end
+      @iterator_file.sync = true
+
+      Dir.glob(File.join(@directory, 'iterator-*')) do |i|
+        g = i.split(%r{/}).last.match(%r{iterator-(.*)}).captures[0]
+        @group_iterators_files[g] = File.open(i, 'r+')
+        @group_iterators[g] = 0
+        @group_iterators_files[g].each_line do |line|
+          @group_iterators[g] = line.to_i
+        end
+        @group_iterators_files[g].seek(0, IO::SEEK_END)
+      end
+    end
+
+    # Add data to the channel.
+    def post data
+      commit data
+    end
+    
+    # Returns the next item from the global (non-group) iterator.
+    def get_next
+      r = nil
+      Thread.exclusive do
+        @iterator += 1 # TODO make sure this is lower than @last_id
+        r = get(@iterator)
+        @iterator_file.write("#{@iterator}\n")
+      end
+      r
+    end
+
+    # Returns the next item for +group+. If +group+ hasn't been seen before, the first item is returned.
+    def get_next_by_group group
+      r = nil
+      Thread.exclusive do
+        @group_iterators[group] = 0 unless @group_iterators.key?(group)
+        @group_iterators[group] += 1
+        r = get(@group_iterators[group])
+        group_iterators_file(group) do |f|
+          f.write("#{@group_iterators[group]}\n")
+        end
+      end
+      r
+    end
+
+    private
+    
+    def group_iterators_file group
+      unless @group_iterators_files[group]
+        @group_iterators_files[group] = File.open(File.join(@directory, 'iterator-' + group), 'a+')
+        @group_iterators_files[group].sync = true
+      end
+      yield @group_iterators_files[group]
+    end
+    
+  end
+end

data/lib/conveyor/server.rb

@@ -0,0 +1,117 @@
+require 'rubygems'
+require 'mongrel'
+require 'conveyor/channel'
+require 'fileutils'
+
+class Mongrel::HttpRequest
+  def put?
+    params["REQUEST_METHOD"] == "PUT"
+  end
+  
+  def post?
+    params["REQUEST_METHOD"] == "POST"
+  end
+  
+  def get?
+    params["REQUEST_METHOD"] == "GET"
+  end
+
+  def path_match pattern
+    params["REQUEST_PATH"].match(pattern)
+  end
+end
+
+module Conveyor
+  
+  # An HTTP server for Conveyor.
+  class Server < Mongrel::HttpServer
+
+    # A Mongrel handler for multiple Conveyor Channels.
+    class ChannelsHandler < Mongrel::HttpHandler
+
+      def initialize data_directory
+        @data_directory = data_directory
+        @channels = {}
+        Dir.entries(@data_directory).each do |e|
+          if !['.', '..'].include?(e) && File.directory?(File.join(@data_directory, e))
+            @channels[e] = Channel.new(File.join(@data_directory, e))
+          end
+        end
+      end
+
+      def create_new_channel channel_name
+        @channels[channel_name] = Conveyor::Channel.new(File.join(@data_directory, channel_name))
+      end
+
+      def process request, response
+        if request.put? && m = request.path_match(%r{/channels/(.*)})
+          if Channel.valid_channel_name?(m.captures[0])
+            create_new_channel m.captures[0]
+            response.start(201) do |head, out|
+              out.write("created channel #{m.captures[0]}")
+            end
+          else
+            response.start(406) do |head, out|
+              out.write("invalid channel name. must match #{Channel::NAME_PATTERN}")
+            end
+          end
+        elsif request.post? && m = request.path_match(%r{/channels/(.*)})
+          if @channels.key?(m.captures[0])
+            if request.params.include?('HTTP_DATE') && d = Time.parse(request.params['HTTP_DATE'])
+              id = @channels[m.captures[0]].post(request.body.read)
+              response.start(202) do |head, out|
+                head["Location"] = "/channels/#{m.captures[0]}/#{id}"
+              end
+            else
+              response.start(400) do |head, out|
+                out.write "A valid Date header is required for all POSTs."
+              end
+            end
+          end
+
+        elsif request.get?
+          headers = content = nil
+          if m = request.path_match(%r{/channels/(.*)/(\d+)})
+            if @channels.key?(m.captures[0])
+              headers, content = @channels[m.captures[0]].get(m.captures[1].to_i)
+            end
+          elsif m = request.path_match(%r{/channels/(.*)})
+            if @channels.key?(m.captures[0])
+              params = Mongrel::HttpRequest.query_parse(request.params['QUERY_STRING'])
+              if params.key? 'next'
+                if params.key? 'group'
+                  headers, content = @channels[m.captures[0]].get_next_by_group(params['group'])
+                else
+                  headers, content = @channels[m.captures[0]].get_next
+                end
+              end
+            end
+          else
+            response.start(200) do |head, out|
+              out.write("fake!")
+            end
+          end
+
+          if headers && content
+            response.start(200) do |head, out|
+              head['Content-Location'] = "/channels/#{m.captures[0]}/#{headers[:id]}"
+              head['Content-MD5']      = headers[:hash]
+              head['Content-Type']     = 'application/octet-stream'
+              head['Last-Modified']    = Time.parse(headers[:time]).gmtime.to_s
+              out.write content
+            end
+          end
+          
+        end
+      end
+    end
+
+    # +host+ and +port+ are passed along to Mongrel::HttpServer for TCP binding. +data_directory+ is used to store
+    # all channel data and should be created before intializing a Server.
+    def initialize(host, port, data_directory)
+      super(host, port)
+      ch = ChannelsHandler.new(data_directory)
+      register("/channels", ch)
+    end
+  end
+end

data/lib/priority_queue.rb

@@ -0,0 +1,33 @@
+# a naive (performance-wise) priority queue implementation
+#
+# modified from http://www.rubyquiz.com/quiz98.html
+class PriorityQueue #:nodoc: this isn't ready for primetime yet
+  def initialize sort_proc = nil
+    @list = []
+    @sort_proc = sort_proc || proc{|x,y| x <=> y}
+  end
+
+  def add(item)
+    @list << item
+    @list.sort!(&@sort_proc)
+    self
+  end
+
+  alias << add
+
+  def front
+    @list.first
+  end
+  
+  def pop
+    @list.shift
+  end
+
+  def empty?
+    @list.empty?
+  end
+
+  def length
+    @list.length
+  end
+end

data/test/rand.rb

@@ -0,0 +1,34 @@
+$: << 'lib'
+require 'conveyor/channel'
+
+def rand_str(len, domain)
+  (1..len).inject("") { |s, x| s << domain[rand(domain.length)] }
+end
+
+alphanum = [ ('a'..'z').to_a, ('A'..'Z').to_a, ('0'..'9').to_a ].flatten
+data = []
+
+channel = Conveyor::Channel.new '/tmp/bar'
+
+puts "writing random data"
+1000.times{channel.post(d=rand_str(rand(10000), alphanum)); data << d}
+
+puts "reading data back"
+
+1000.times do |i|
+  headers, content = channel.get(i + 1)
+  
+  unless content == data[i]
+    puts "unmatched content"
+    puts "ORIGINAL"
+    puts "----"
+    puts data[i]
+    puts "----"
+    
+    puts "RETURNED"
+    puts "----"
+    puts content
+    puts "----"
+  end
+  
+end

data/test/test_channel.rb

@@ -0,0 +1,126 @@
+require "test/unit"
+require 'fileutils'
+require "conveyor/channel"
+
+class TestConveyorChannel < Test::Unit::TestCase
+  include Conveyor
+  def test_init
+    FileUtils.rm_r('/tmp/bar')
+    assert_nothing_raised do
+      Channel.new '/tmp/bar'
+    end
+  end
+
+  def test_post
+    FileUtils.rm_r('/tmp/bar')
+    assert_nothing_raised do
+      w = Channel.new '/tmp/bar'
+      w.post 'foo'
+      assert_equal 1, w.instance_variable_get(:@last_id)
+    end
+  end
+  
+  def test_parse_headers
+    i = Channel.new '/tmp/foo'
+    [
+      ["1 2008-01-08T13:04:40-08:00 0 3 acbd18db4cc2f85cedef654fccc4a4d8\n", 
+       {:id => 1, :time => "2008-01-08T13:04:40-08:00", :offset => 0, :length => 3, :hash => "acbd18db4cc2f85cedef654fccc4a4d8"}
+      ]
+    ].each do |(str, ret)|
+      assert_equal ret, i.parse_headers(str)
+    end
+    
+    [
+      ["2 2008-01-08T13:04:40-08:00 0 3 acbd18db4cc2f85cedef654fccc4a4d8 1\n", 
+       {:id => 2, :time => "2008-01-08T13:04:40-08:00", :offset => 0, :length => 3, :hash => "acbd18db4cc2f85cedef654fccc4a4d8", :file => 1}
+      ]
+    ].each do |(str, ret)|
+      assert_equal ret, i.parse_headers(str, true)
+    end
+  end
+
+  def test_init_existing
+    FileUtils.rm_r('/tmp/bar') rescue nil
+    c = Channel.new('/tmp/bar')
+    c.post 'foo'
+    c.post 'bar'
+    c = nil
+    GC.start
+    
+    d = Channel.new('/tmp/bar')
+    assert_equal 'foo', d.get(1)[1]
+    assert_equal 'bar', d.get(2)[1]
+    
+    d.post('bam')
+    assert_equal 'bam', d.get(3)[1]
+  end
+
+  def test_get_next
+    FileUtils.rm_r('/tmp/bar') rescue nil
+    c = Channel.new('/tmp/bar')
+    c.post 'foo'
+    c.post 'bar'
+    c.post 'bam'
+
+    assert_equal 'foo', c.get_next[1]
+    assert_equal 'bar', c.get_next[1]
+    assert_equal 'bam', c.get_next[1]
+    assert_equal nil, c.get_next
+  end
+
+  def test_get_next_interupted
+    FileUtils.rm_r('/tmp/bar') rescue nil
+    c = Channel.new('/tmp/bar')
+    c.post 'foo'
+    c.post 'bar'
+    c.post 'bam'
+    
+    assert_equal 'foo', c.get_next[1]
+    assert_equal 'bar', c.get_next[1]
+
+    d = Channel.new('/tmp/bar')
+    assert_not_equal c, d
+    assert_equal 'bam', d.get_next[1]
+    assert_equal nil, d.get_next
+  end
+
+  def test_get_next_by_group
+    FileUtils.rm_r('/tmp/bar') rescue nil
+    c = Channel.new('/tmp/bar')
+    c.post 'foo'
+    c.post 'bar'
+    c.post 'bam'
+    
+    assert_equal 'foo', c.get_next_by_group('foo')[1]
+    assert_equal 'bar', c.get_next_by_group('foo')[1]
+    assert_equal 'bam', c.get_next_by_group('foo')[1]
+    assert_equal nil, c.get_next_by_group('foo')
+    
+    assert_equal 'foo', c.get_next_by_group('bar')[1]
+    assert_equal 'bar', c.get_next_by_group('bar')[1]
+    assert_equal 'bam', c.get_next_by_group('bar')[1]
+    assert_equal nil, c.get_next_by_group('bar')
+  end
+
+  def test_get_next_by_group_interupted
+    FileUtils.rm_r('/tmp/bar') rescue nil
+    c = Channel.new('/tmp/bar')
+    c.post 'foo'
+    c.post 'bar'
+    c.post 'bam'
+    
+    assert_equal 'foo', c.get_next_by_group('foo')[1]
+    assert_equal 'bar', c.get_next_by_group('foo')[1]
+    assert_equal 'foo', c.get_next_by_group('bar')[1]
+    assert_equal 'bar', c.get_next_by_group('bar')[1]
+
+    c = nil
+    GC.start
+    c = Channel.new('/tmp/bar')
+
+    assert_equal 'bam', c.get_next_by_group('foo')[1]
+    assert_equal nil, c.get_next_by_group('foo')    
+    assert_equal 'bam', c.get_next_by_group('bar')[1]
+    assert_equal nil, c.get_next_by_group('bar')
+  end
+end

data/test/test_priority_queue.rb

@@ -0,0 +1,16 @@
+require "test/unit"
+
+require "priority_queue"
+
+class TestPriorityQueue < Test::Unit::TestCase
+  def test_simple
+    
+    pq = PriorityQueue.new
+
+    pq << 1 << 2 << -1
+
+    assert_equal -1, pq.pop
+    assert_equal 1, pq.pop
+    assert_equal 2, pq.pop
+  end
+end

data/test/test_replicated_channel.rb

@@ -0,0 +1,60 @@
+require "test/unit"
+require 'fileutils'
+require "conveyor/replicated_channel"
+
+class TestReplicatedChannel < Test::Unit::TestCase
+  def test_basic
+    FileUtils.rm_r '/tmp/foo' rescue nil
+    FileUtils.rm_r '/tmp/bar' rescue nil
+    
+    c1 = Conveyor::ReplicatedChannel.new '/tmp/foo'
+    c2 = Conveyor::ReplicatedChannel.new '/tmp/bar'
+    
+    c1.peers << c2
+    c2.peers << c1
+
+    c1.post('foo', Time.now)
+    c2.post('bar', Time.now)
+
+    c1.post('not', Time.now)
+    c2.post('not', Time.now)
+
+    c1.commit_thread.run
+    c2.commit_thread.run
+
+    assert_equal 'foo', c1.get(1)[1]
+    assert_equal 'foo', c2.get(1)[1]
+
+    assert_equal 'bar', c1.get(2)[1]
+    assert_equal 'bar', c2.get(2)[1]
+    
+    assert_equal c1.get(1), c2.get(1)
+    assert_equal c1.get(2), c2.get(2)
+  end
+  
+  def test_more
+    FileUtils.rm_r '/tmp/foo' rescue nil
+    FileUtils.rm_r '/tmp/bar' rescue nil
+    
+    c1 = Conveyor::ReplicatedChannel.new '/tmp/foo'
+    c2 = Conveyor::ReplicatedChannel.new '/tmp/bar'
+    
+    c1.peers << c2
+    c2.peers << c1
+
+    channels = [c1, c2]
+    data = %w[1 2 3 4 5 6 7 8 9 10]
+
+    data.each_with_index do |d, i|
+      channels[i % 2].post(d, Time.now)
+    end
+
+    c1.commit_thread.run
+    c2.commit_thread.run
+
+    c1d = (1..10).collect{|i| c1.get(i)}
+    c2d = (1..10).collect{|i| c2.get(i)}
+
+    assert_equal c1d, c2d
+  end
+end

data/test/test_server.rb

@@ -0,0 +1,93 @@
+require "test/unit"
+require "conveyor/server"
+require 'net/http'
+
+class TestConveyorServer < Test::Unit::TestCase
+  def setup
+    FileUtils.rm_r('/tmp/asdf') rescue nil
+    FileUtils.mkdir('/tmp/asdf')
+    @server = Conveyor::Server.new("127.0.0.1", 8888, '/tmp/asdf')
+    @server.run
+  end
+  
+  def teardown
+    @server.stop
+  end
+  
+  def test_channels
+    Net::HTTP.start("localhost", 8888) do |h|
+      req = h.get('/channels')
+      assert_equal Net::HTTPOK, req.class
+    end
+  end
+  
+  def test_create_channel
+    Net::HTTP.start('localhost', 8888) do |h|
+      req = h.put('/channels/foo', '', {'Content-Type' => 'application/octet-stream'})
+      assert_equal Net::HTTPCreated, req.class
+
+      req = h.post('/channels/foo', 'foo', {'Content-Type' => 'application/octet-stream', 'Date' => Time.now.to_s})
+      assert_equal Net::HTTPAccepted, req.class
+    end
+  end
+  
+  def test_post
+    Net::HTTP.start('localhost', 8888) do |h|
+      req = h.put('/channels/bar', '', {'Content-Type' => 'application/octet-stream'})
+      assert_equal Net::HTTPCreated, req.class
+
+      data =
+      ["ZqZyDN2SouQCYEHYS0LuM1XeqsF0MKIbFEBE6xQ972VqEcjs21wJSosvZMWEH1lq5ukTq4Ze", 
+        "5sgCbNpumntlHC2jl6uXcW3Wz1RGTc5lqeGhpH2ZaCtAOc61TLBmLPUzPWVeJzkfr6qEQqTkuPK5kCE54u6iiTVFArMPSUy3oo", 
+        "6qPDKL09AquFvgj9Zv9CFY2dn0x", "84ReeNklrWJeIu2USbBtwoUnkDwUgU8lNjJ", 
+        "WUSYY2dCBdDdZEiGWtyfC5yGKVMgDhzBhyNLwcefxa49fED1Sf05f8MlgXOBx6n5I6Ae2Wy3Mds", 
+        "uAlUDvngWqDl3PaRVl1i9RcwDIvJlNp6yMy9RQgVsucwNvKaSOQlJMarWItKy8zT2ON08ElKkZ2aQJlb45Z8FwfE0xh8sA", 
+        "NxWmEBmJp0uiNRhyxa26frQjfFaNERmZbConrytNQKnHfilFsZWAo0Qy8eVKgq", "ajq3i5ksiBovQYfvj", 
+        "yY3vhjeq","2IDeF0ccG8tRZIZSekz6fUii29"]
+        
+      data.each do |d|
+        req = h.post('/channels/bar', d, {'Content-Type' => 'application/octet-stream', 'Date' => Time.now.to_s})
+        assert_equal Net::HTTPAccepted, req.class
+      end
+
+      data.each_with_index do |d, i|
+        req = h.get("/channels/bar/#{i+1}")
+        assert_equal d, req.body
+      end
+    end
+  end
+  
+  def test_invalid_channel
+    Net::HTTP.start('localhost', 8888) do |h|
+      req = h.put('/channels/|', '', {'Content-Type' => 'application/octet-stream'})
+      assert_equal Net::HTTPNotAcceptable, req.class
+    end
+    
+  end
+
+  def test_get_next
+    Net::HTTP.start('localhost', 8888) do |h|
+      req = h.put('/channels/bar', '', {'Content-Type' => 'application/octet-stream'})
+      assert_equal Net::HTTPCreated, req.class
+
+      data =
+      ["ZqZyDN2SouQCYEHYS0LuM1XeqsF0MKIbFEBE6xQ972VqEcjs21wJSosvZMWEH1lq5ukTq4Ze", 
+        "5sgCbNpumntlHC2jl6uXcW3Wz1RGTc5lqeGhpH2ZaCtAOc61TLBmLPUzPWVeJzkfr6qEQqTkuPK5kCE54u6iiTVFArMPSUy3oo", 
+        "6qPDKL09AquFvgj9Zv9CFY2dn0x", "84ReeNklrWJeIu2USbBtwoUnkDwUgU8lNjJ", 
+        "WUSYY2dCBdDdZEiGWtyfC5yGKVMgDhzBhyNLwcefxa49fED1Sf05f8MlgXOBx6n5I6Ae2Wy3Mds", 
+        "uAlUDvngWqDl3PaRVl1i9RcwDIvJlNp6yMy9RQgVsucwNvKaSOQlJMarWItKy8zT2ON08ElKkZ2aQJlb45Z8FwfE0xh8sA", 
+        "NxWmEBmJp0uiNRhyxa26frQjfFaNERmZbConrytNQKnHfilFsZWAo0Qy8eVKgq", "ajq3i5ksiBovQYfvj", 
+        "yY3vhjeq","2IDeF0ccG8tRZIZSekz6fUii29"]
+        
+      data.each do |d|
+        req = h.post('/channels/bar', d, {'Content-Type' => 'application/octet-stream', 'Date' => Time.now.to_s})
+        assert_equal Net::HTTPAccepted, req.class
+      end
+
+      data.each_with_index do |d, i|
+        req = h.get("/channels/bar?next")
+        assert_equal d, req.body
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,125 @@
+--- !ruby/object:Gem::Specification 
+name: conveyor
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Ryan King
+autorequire: 
+bindir: bin
+cert_chain: 
+- |
+  -----BEGIN CERTIFICATE-----
+  MIIDNjCCAh6gAwIBAgIBADANBgkqhkiG9w0BAQUFADBBMQ0wCwYDVQQDDARyeWFu
+  MRswGQYKCZImiZPyLGQBGRYLdGhlcnlhbmtpbmcxEzARBgoJkiaJk/IsZAEZFgNj
+  b20wHhcNMDgwMTMwMjM0MTMzWhcNMDkwMTI5MjM0MTMzWjBBMQ0wCwYDVQQDDARy
+  eWFuMRswGQYKCZImiZPyLGQBGRYLdGhlcnlhbmtpbmcxEzARBgoJkiaJk/IsZAEZ
+  FgNjb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDDTUJAZENpuhnu
+  P+R7DZlKe8mOuSkpyL6ZXTfCY2dgtwM7/uKy9Z31oOJshAJLuhscbuJqhHIt47nE
+  Adov3Ewp/q7jE7yodMLirsFod15sYPXij7vvNKOri2qDqvRCB/jQ0L65DqoHXckV
+  WOpZRBLzOp/q/XmJs/cq4KSZ0LFh8la4jIZsrYEG/dJ8BBXzQc14x6VkSxTGhlFz
+  LDz8IUIvefXUAGD5NnY+P8KbmPtf4RiAoWxLRwKIZiUHRQzFXwUdJyrDC+qYJ6CX
+  KlcU3Y5b259KD2eEmJsgpPexWRsaeLUlxG73Qsg2Dffs0GiyoZ4fhg+C0gcN1Yza
+  tLPQH79fAgMBAAGjOTA3MAkGA1UdEwQCMAAwCwYDVR0PBAQDAgSwMB0GA1UdDgQW
+  BBTDywO/cXwzSXuI7wyMnsboiRFTbzANBgkqhkiG9w0BAQUFAAOCAQEAmUm1El7H
+  4c423UFHQ1cYTxEdicBdSCDqCBJcYyVMoOwOSjDxw3/G2iRafr4OpY+WQaTYIx05
+  FpV4tarYrP6u3w+3hynmgdHRBJJLGtoDhB8ln5QwdcsSN7nK+ERoZvkM6LMpH5G3
+  P5PNDhq+Snq26SzFrr8fYTtBqccixD4Njepo3r3KVZaK8VlrbDrDwiliPtTL/s9R
+  MpCn2/2deOie29ZLFh41VWqBzDba05+j9e7ztisxttrzTO6IA525y60eQ5AUylzc
+  1grJrsSQhj0OXe1kLH0fPXHewEVQzaHhvd5JGSTMiGntqbfFKV9raP2xidO9jMMO
+  Zls3y84CmyAEGg==
+  -----END CERTIFICATE-----
+
+date: 2008-01-30 00:00:00 -08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: mongrel
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: activesupport
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.4.0
+    version: 
+description: == Description  * Like TiVo for your data * A distributed rewindable multi-queue  == Overview  A Conveyor server provides an HTTP interface that allows for POSTing and GETing items in streams called Channels.
+email: ryan@theryanking.com
+executables: 
+- conveyor
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- Manifest.txt
+- README.txt
+files: 
+- History.txt
+- LICENSE
+- Manifest.txt
+- README.txt
+- Rakefile
+- bin/conveyor
+- docs/file-formats.mkd
+- docs/protocol.mkd
+- lib/conveyor.rb
+- lib/conveyor/base_channel.rb
+- lib/conveyor/channel.rb
+- lib/conveyor/server.rb
+- lib/priority_queue.rb
+- test/rand.rb
+- test/test_channel.rb
+- test/test_feeder-ng.rb
+- test/test_priority_queue.rb
+- test/test_replicated_channel.rb
+- test/test_server.rb
+has_rdoc: true
+homepage: 
+post_install_message: 
+rdoc_options: 
+- --main
+- README.txt
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: conveyor
+rubygems_version: 1.0.1
+signing_key: 
+specification_version: 2
+summary: Like TiVo for your data.
+test_files: 
+- test/test_channel.rb
+- test/test_feeder-ng.rb
+- test/test_priority_queue.rb
+- test/test_replicated_channel.rb
+- test/test_server.rb

metadata.gz.sig

@@ -0,0 +1,2 @@
+���dp0���^o��a1�f%��d��JM�;���W��|X�G���?K�	��/.+a��>"J"u��oy�+�N(���#؝WNZ+�i�j�l���8�J��`�7�Ɔ��h�N<Vl�)V*7^] /D���#$��P*�W?~�>p,��tn�js�]�օL��iQ5��H�s!���P�鎮y�U�wO����ج��f+.�9��
-p�K �Y��0���v}���U�
+�ۥ�N|��S