conveyor 0.1.4 → 0.2.0

data/History.txt

@@ -1,3 +1,12 @@
+== 0.2.0 / 2008-02-26
+
+* switched from using Mongrel to Thin. this adds a dependency on thin, which depends on Event Machine
+* GZIPed storage. Entries over an arbitrary size (>=256 bytes) are compressed before written down. Since this constitutes a change in file format, the script conveyor-upgrade should be run on all date before running the new version.
+* Channels can now be deleted. This is very unsafe and only available when running with the --unsafe switch.
+* Channels are auto-created the first time you try and post a message to them.
+* Only channels with valid names are opened by the server.
+* The client API was refactored to be more like the Channel API and Channel-specific.
+
 == 0.1.4 / 2008-02-19
 * support for getting multiple items at once
 * rewinding implemented for group iterators

data/Manifest.txt

@@ -4,6 +4,7 @@ Manifest.txt
 README.txt
 Rakefile
 bin/conveyor
+bin/conveyor-upgrade
 docs/file-formats.mkd
 docs/protocol.mkd
 lib/conveyor.rb
@@ -11,10 +12,10 @@ lib/conveyor/base_channel.rb
 lib/conveyor/channel.rb
 lib/conveyor/client.rb
 lib/conveyor/server.rb
+lib/conveyor/upgrader.rb
 lib/priority_queue.rb
 test/rand.rb
 test/test_channel.rb
 test/test_feeder-ng.rb
 test/test_priority_queue.rb
-test/test_replicated_channel.rb
 test/test_server.rb

data/Rakefile

@@ -9,7 +9,7 @@ Hoe.new('conveyor', Conveyor::VERSION) do |p|
   p.author = 'Ryan King'
   p.email = 'ryan@theryanking.com'
   p.remote_rdoc_dir = ''
-  p.extra_deps << ['mongrel']
+  p.extra_deps << ['thin']
   p.extra_deps << ['json']
   p.extra_deps << ['daemons']
 end

data/bin/conveyor

@@ -3,6 +3,7 @@
 require 'optparse'
 require 'rubygems'
 require 'daemons/daemonize'
+require 'thin'
 
 options = {:port => 8011, :daemonize => false}
 opts = OptionParser.new do |opts|
@@ -22,6 +23,10 @@ opts = OptionParser.new do |opts|
   opts.on("-d", "Daemonize.") do |d|
     options[:daemonize] = d
   end
+  
+  opts.on("--unsafe", "Unsafe Mode.") do |d|
+    options[:unsafe] = d
+  end
 end
 
 opts.parse!
@@ -39,10 +44,9 @@ if options[:daemonize]
   Daemonize.daemonize
 end
 
-server = Conveyor::Server.new('0.0.0.0', options[:port], ARGV[0], options[:log_directory])
-
-Signal.trap 'INT' do
-  server.stop
+Thin::Server.start('0.0.0.0', options[:port]) do
+  map '/channels' do
+    run Conveyor::App.new(ARGV[0], :log_directory => options[:log_directory], :unsafe_mode => options[:unsafe])
+  end
 end
 
-server.run.join

data/bin/conveyor-upgrade

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby -KU
+
+$: << 'lib'
+
+require 'conveyor/server'
+require 'conveyor/upgrader'
+
+u = Conveyor::Upgrader.new ARGV.first
+
+u.upgrade

data/lib/conveyor.rb

@@ -1,4 +1,4 @@
 module Conveyor
-  VERSION = '0.1.4'
+  VERSION = '0.2.0'
   QUALITY = 'alpha'
 end

data/lib/conveyor/base_channel.rb

@@ -1,7 +1,9 @@
 require 'digest/md5'
 require 'rubygems'
 require 'time'
-require 'priority_queue'
+require 'stringio'
+require 'zlib'
+require 'json'
 
 module Conveyor
   # BaseChannel
@@ -11,43 +13,37 @@ module Conveyor
 
     NAME_PATTERN = %r{\A[a-zA-Z\-0-9\_]+\Z}
     BUCKET_SIZE = 100_000
+    FORMAT_VERSION = 1
+
+    module Flags
+      GZIP = 1
+    end
 
     def initialize directory
-      @directory               = directory
-      @data_files              = []
-      @index = []
-      @iterator = 1
+      @directory    = directory
+      @data_files   = []
+      @file_mutexes = []
+      @index        = []
+      @iterator     = 1
+      @id_lock      = Mutex.new
 
       if File.exists?(@directory)
         if !File.directory?(@directory)
           raise "#{@directory} is not a directory"
         end
+        load_channel
       else
         Dir.mkdir(@directory)
+        setup_channel
       end
 
-      index_path = File.join(@directory, 'index')
-
-      if File.exists?(index_path) && File.size(index_path) > 0
-        @index_file = File.open(index_path, 'r+')
-
-        @index_file.each_line do |line|
-          @index << parse_headers(line.strip, true)
-          @last_id = @index.last[:id]
-        end
-        @index_file.seek(0, IO::SEEK_END)
-      else
-        @index_file = File.open(index_path, 'a')
-        @last_id = 0
-      end
-      @index_file.sync = true
-
+      @index_file.sync    = true
     end
-    
+
     def inspect
       "<#{self.class} dir:'#{@directory.to_s}' last_id:#{@last_id} iterator:#{@iterator}>"
     end
-    
+
     def pick_bucket i
       (i / BUCKET_SIZE).to_i
     end
@@ -56,63 +52,149 @@ module Conveyor
       unless @data_files[i]
         @data_files[i] = File.open(File.join(@directory, i.to_s), 'a+')
         @data_files[i].sync = true
+        @file_mutexes[i] = Mutex.new
+      end
+      @file_mutexes[i].synchronize do
+        yield @data_files[i]
+      end
+    end
+
+    def id_lock
+      @id_lock.synchronize do
+        yield
       end
-      yield @data_files[i]
     end
 
     def commit data, time = nil
-      Thread.exclusive do
+      l = nil
+      gzip = data.length >= 256
+      if gzip
+        compressed_data = StringIO.new
+        g = Zlib::GzipWriter.new(compressed_data)
+        g << data
+        g.finish
+        compressed_data.rewind
+        compressed_data = compressed_data.read
+        l = compressed_data.length
+      else
+        l = data.length
+      end
+
+      h = Digest::MD5.hexdigest(data)
+
+      id_lock do
         i = @last_id + 1
         t = time || Time.now
-        l = data.length
-        h = Digest::MD5.hexdigest(data)
         b = pick_bucket(i)
+        flags = 0
+        flags = flags | Flags::GZIP if gzip
         header, o = nil
         bucket_file(b) do |f|
           f.seek(0, IO::SEEK_END)
           o = f.pos
-          header = "#{i} #{t.xmlschema} #{o} #{l} #{h}"
-          f.write("#{header}\n" + data + "\n")
+          header = "#{i.to_s(36)} #{t.to_i.to_s(36)} #{o.to_s(36)} #{l.to_s(36)} #{h} #{flags.to_s(36)}"
+          f.write("#{header}\n")
+          f.write((gzip ? compressed_data : data))
+          f.write("\n")
         end
 
         @last_id = i
-        @index_file.write "#{header} #{b}\n"
+        @index_file.write "#{header} #{b.to_s(36)}\n"
         @index << {:id => i, :time => t, :offset => o, :length => l, :hash => h, :file => b}
         i
       end
     end
 
-    def get id
-      return nil unless id <= @last_id
-      i = @index.find{|e| e[:id] == id}
-      header, content = nil
-      Thread.exclusive do
-        bucket_file(i[:file]) do |f|
-          f.seek i[:offset]
-          header  = f.readline.strip
-          content = f.read(i[:length])
-        end
+    def get id, stream = false
+      return nil unless id <= @last_id && id > 0
+      i = @index[id-1]
+      headers, content, compressed_content, g = nil
+      bucket_file(i[:file]) do |f|
+        f.seek i[:offset]
+        headers  = parse_headers(f.readline.strip)
+        compressed_content = f.read(i[:length])
+      end
+      io = StringIO.new(compressed_content)
+      if (headers[:flags] & Flags::GZIP) != 0
+        g  = Zlib::GzipReader.new(io)
+      else
+        g = io
+      end
+      if stream
+        [headers, g]
+      else
+        [headers, g.read]
       end
-      [parse_headers(header), content]
     end
 
-    def parse_headers str, index_file=false
-      pattern =  '\A(\d+) (\d{4}\-\d{2}\-\d{2}T\d{2}\:\d{2}\:\d{2}[+\-]\d{2}\:\d{2}) (\d+) (\d+) ([a-f0-9]+)'
+    def self.parse_headers str, index_file=false
+      pattern =  '\A([a-z\d]+) ([a-z\d]+) ([a-z\d]+) ([a-z\d]+) ([a-f0-9]+) ([a-z\d]+)'
       pattern += ' (\d+)' if index_file
       pattern += '\Z'
       m = str.match(Regexp.new(pattern))
       {
-        :id => m.captures[0].to_i,
-        :time => m.captures[1],
-        :offset => m.captures[2].to_i,
-        :length => m.captures[3].to_i,
-        :hash => m.captures[4], 
-        :file => (index_file ? m.captures[5].to_i : nil)
-      }.reject {|k,v| v == nil}
+        :id     => m.captures[0].to_i(36),
+        :time   => m.captures[1].to_i(36),
+        :offset => m.captures[2].to_i(36),
+        :length => m.captures[3].to_i(36),
+        :hash   => m.captures[4],
+        :flags  => m.captures[5].to_i(36),
+        :file   => (index_file ? m.captures[6].to_i(36) : nil)
+      }
     end
 
+    def parse_headers str, index_file=false
+      self.class.parse_headers str, index_file
+    end
+    
     def self.valid_channel_name? name
       !!name.match(NAME_PATTERN)
     end
+
+    def delete!
+      FileUtils.rm_r(@directory)
+      @index = []
+      @data_files =[]
+      @last_id = 0
+    end
+
+    protected
+
+    def setup_channel
+      @index_file = File.open(index_path, 'a')
+      @last_id = 0
+      @version = FORMAT_VERSION
+      File.open(version_path, 'w+'){|f| f.write(@version.to_s)}
+    end
+
+    def load_channel
+      if File.exists?(version_path) && File.size(version_path) > 0
+        @version = File.open(version_path).read.to_i
+      else
+        @version = 0
+      end
+
+      if @version != FORMAT_VERSION
+        raise "Format versions don't match. Try upgrading."
+      end
+
+      @index_file = File.open(index_path, 'r+')
+
+      @index_file.each_line do |line|
+        @index << parse_headers(line.strip, true)
+        @last_id = @index.last[:id]
+      end
+      @index_file.seek(0, IO::SEEK_END)
+    end
+
+
+    def index_path
+      File.join(@directory, 'index')
+    end
+
+    def version_path
+      File.join(@directory, 'version')
+    end
+
   end
 end

data/lib/conveyor/channel.rb

@@ -8,33 +8,14 @@ module Conveyor
 
     # If +directory+ doesn't already exist, it will be created during initialization.
     def initialize directory
-      @group_iterators         = {}
-      @group_iterators_files   = {}
-      
-      super(directory)
-      
-      iterator_path = File.join(@directory, 'iterator')
+      @group_iterators       = {}
+      @group_iterators_files = {}
+      @iterator_lock         = Mutex.new
+      @group_iterator_locks  = Hash.new{|k,v| Mutex.new }
 
-      if File.exists?(iterator_path) && File.size(iterator_path) > 0
-        @iterator_file = File.open(iterator_path, 'r+')
-        @iterator_file.each_line do |line|
-          @iterator = line.to_i
-        end
-        @iterator_file.seek(0, IO::SEEK_END)
-      else
-        @iterator_file = File.open(iterator_path, 'a')
-      end
-      @iterator_file.sync = true
+      super(directory)
 
-      Dir.glob(File.join(@directory, 'iterator-*')) do |i|
-        g = i.split(%r{/}).last.match(%r{iterator-(.*)}).captures[0]
-        @group_iterators_files[g] = File.open(i, 'r+')
-        @group_iterators[g] = 1
-        @group_iterators_files[g].each_line do |line|
-          @group_iterators[g] = line.to_i
-        end
-        @group_iterators_files[g].seek(0, IO::SEEK_END)
-      end
+      @iterator_file.sync    = true
     end
 
     # Add data to the channel.
@@ -45,11 +26,11 @@ module Conveyor
     # Returns the next item from the global (non-group) iterator.
     def get_next
       r = nil
-      Thread.exclusive do
+      iterator_lock do
         if @iterator <= @last_id
           r = get(@iterator)
           @iterator += 1
-          @iterator_file.write("#{@iterator}\n")
+          @iterator_file.write("#{@iterator.to_s(36)}\n")
           r
         else
           nil
@@ -60,13 +41,13 @@ module Conveyor
     # Returns the next item for +group+. If +group+ hasn't been seen before, the first item is returned.
     def get_next_by_group group
       r = nil
-      Thread.exclusive do
+      group_iterator_lock(group) do
         @group_iterators[group] = 1 unless @group_iterators.key?(group)
         if @iterator <= @last_id
           r = get(@group_iterators[group])
           @group_iterators[group] += 1
           group_iterators_file(group) do |f|
-            f.write("#{@group_iterators[group]}\n")
+            f.write("#{@group_iterators[group].to_s(36)}\n")
           end
         else
           nil
@@ -77,11 +58,11 @@ module Conveyor
 
     def get_next_n n
       r = []
-      Thread.exclusive do
+      iterator_lock do
         while r.length < n && @iterator <= @last_id
           r << get(@iterator)
           @iterator += 1
-          @iterator_file.write("#{@iterator}\n")
+          @iterator_file.write("#{@iterator.to_s(36)}\n")
           r
         end
       end
@@ -90,13 +71,13 @@ module Conveyor
     
     def get_next_n_by_group n, group
       r = []
-      Thread.exclusive do
+      group_iterator_lock(group) do
         @group_iterators[group] = 1 unless @group_iterators.key?(group)
         while r.length < n && @group_iterators[group] < @last_id
           r << get(@group_iterators[group])
           @group_iterators[group] += 1
           group_iterators_file(group) do |f|
-            f.write("#{@group_iterators[group]}\n")
+            f.write("#{@group_iterators[group].to_s(36)}\n")
           end
         end
       end
@@ -119,23 +100,36 @@ module Conveyor
       opts = opts.first
       if opts.key?(:id)
         if opts.key?(:group)
-          Thread.exclusive do
+          group_iterator_lock(opts[:group]) do
             @group_iterators[opts[:group]] = opts[:id].to_i
             group_iterators_file(opts[:group]) do |f|
-              f.write("#{@group_iterators[opts[:group]]}\n")
+              f.write("#{@group_iterators[opts[:group]].to_s(36)}\n")
             end
           end
         else
-          Thread.exclusive do
+          iterator_lock do
             @iterator = opts[:id].to_i
-            @iterator_file.write("#{@iterator}\n")
+            @iterator_file.write("#{@iterator.to_s(36)}\n")
           end
         end
       end
     end
-    
+
+
     private
-    
+
+    def iterator_lock
+      @iterator_lock.synchronize do
+        yield
+      end
+    end
+
+    def group_iterator_lock group
+      @group_iterator_locks[group].synchronize do
+        yield
+      end
+    end
+
     def group_iterators_file group
       unless @group_iterators_files[group]
         @group_iterators_files[group] = File.open(File.join(@directory, 'iterator-' + group), 'a+')
@@ -143,6 +137,37 @@ module Conveyor
       end
       yield @group_iterators_files[group]
     end
-    
+
+    def load_channel
+      super
+      @iterator_file = File.open(iterator_path, 'r+')
+      @iterator_file.each_line do |line|
+        @iterator = line.to_i(36)
+      end
+      @iterator_file.seek(0, IO::SEEK_END)
+
+      Dir.glob(File.join(@directory, 'iterator-*')) do |i|
+        g = i.split(%r{/}).last.match(%r{iterator-(.*)}).captures[0]
+        @group_iterators_files[g] = File.open(i, 'r+')
+        @group_iterators[g] = 1
+        @group_iterators_files[g].each_line do |line|
+          @group_iterators[g] = line.to_i(36)
+        end
+        @group_iterators_files[g].seek(0, IO::SEEK_END)
+      end
+    end
+
+    def setup_channel
+      super
+      @iterator_file = File.open(iterator_path, 'a')
+    end
+
+    def iterator_path
+      File.join(@directory, 'iterator')
+    end
+
+    def version_path
+      File.join(@directory, 'version')
+    end
   end
 end