conveyor 0.2.2 → 0.2.3

data/History.txt

@@ -1,3 +1,18 @@
+== 0.2.3 / 2008-10-21
+* update channel reading code for base-36
+* adding logging to the server startup process
+* new 2-level indexing scheme. only %10 of index items are held in memory 
+   * 10x reduction in memory useage
+   * 1000x speedup in startup time
+* added rack dependency
+* index rebuilding script
+* time the startup
+* some performance improvements for index loading
+* initial block caching strategy for lookups by id
+* cache rejecting + dealing with an edge case on nearest_after
+* refuse to open an index that isn't in sequential id order
+* log http response code when channel already exists
+
 == 0.2.2 / 2008-04-10
 * make sure that an empty channel still has @last_id set
 * switch back to mongrel. too many problems with event machine on solaris

data/Manifest.txt

@@ -4,6 +4,7 @@ Manifest.txt
 README.txt
 Rakefile
 bin/conveyor
+bin/conveyor-rebuild-index
 bin/conveyor-upgrade
 docs/file-formats.rdoc
 docs/protocol.rdoc

data/Rakefile

@@ -12,6 +12,7 @@ Hoe.new('conveyor', Conveyor::VERSION) do |p|
   p.extra_deps << ['mongrel']
   p.extra_deps << ['json']
   p.extra_deps << ['daemons']
+  p.extra_deps << ['rack']
 end
 
 # vim: syntax=Ruby

data/bin/conveyor-rebuild-index

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby -KU
+
+$: << 'lib'
+
+require 'conveyor/server'
+require 'conveyor/upgrader'
+
+if ARGV.first
+  FileUtils.mv(File.join(ARGV.first, 'index'), File.join(ARGV.first, 'index.bak'))
+  FileUtils.touch(File.join(ARGV.first, 'index'))
+  u = Conveyor::Channel.new ARGV.first
+
+  u.rebuild_index
+end

data/lib/conveyor.rb

@@ -1,4 +1,4 @@
 module Conveyor
-  VERSION = '0.2.2'
+  VERSION = '0.2.3'
   QUALITY = 'alpha'
 end

data/lib/conveyor/base_channel.rb

@@ -11,21 +11,42 @@ module Conveyor
   # Base implementation for channels. Not useful to instantiate directly.
   class BaseChannel
 
-    NAME_PATTERN = %r{\A[a-zA-Z\-0-9\_]+\Z}
-    BUCKET_SIZE = 100_000
-    FORMAT_VERSION = 1
+    NAME_PATTERN   = %r{\A[a-zA-Z\-0-9\_]+\Z}
+    BUCKET_SIZE    = 100_000
+    FORMAT_VERSION =       1
+    BLOCK_SIZE     =    1000
+    CACHE_SIZE     =     100
 
     module Flags
       GZIP = 1
     end
 
+    def self.parse_headers str, index_file=false
+      id, time, offset, length, hash, flags, file = str.split ' '
+      {
+        :id     => id.to_i(36),
+        :time   => time.to_i(36),
+        :offset => offset.to_i(36),
+        :length => length.to_i(36),
+        :hash   => hash,
+        :flags  => flags.to_i(36),
+        :file   => (index_file ? file.to_i(36) : nil)
+      }
+    end
+
+    def self.valid_channel_name? name
+      !!name.match(NAME_PATTERN)
+    end
+
     def initialize directory
-      @directory    = directory
-      @data_files   = []
-      @file_mutexes = []
-      @index        = []
-      @iterator     = 1
-      @id_lock      = Mutex.new
+      @directory             = directory
+      @data_files            = []
+      @file_mutexes          = []
+      @iterator              = 1 #TODO: move to Channel.rb
+      @id_lock               = Mutex.new
+      @index_file_lock       = Mutex.new
+      @block_cache           = {}
+      @block_last_used       = {}
 
       if File.exists?(@directory)
         if !File.directory?(@directory)
@@ -44,27 +65,6 @@ module Conveyor
       "<#{self.class} dir:'#{@directory.to_s}' last_id:#{@last_id} iterator:#{@iterator}>"
     end
 
-    def pick_bucket i
-      (i / BUCKET_SIZE).to_i
-    end
-
-    def bucket_file i
-      unless @data_files[i]
-        @data_files[i] = File.open(File.join(@directory, i.to_s), 'a+')
-        @data_files[i].sync = true
-        @file_mutexes[i] = Mutex.new
-      end
-      @file_mutexes[i].synchronize do
-        yield @data_files[i]
-      end
-    end
-
-    def id_lock
-      @id_lock.synchronize do
-        yield
-      end
-    end
-
     def commit data, time=nil
       l = nil
       gzip = data.length >= 256
@@ -84,7 +84,7 @@ module Conveyor
 
       id_lock do
         i = @last_id + 1
-        t = time || Time.now
+        t = time || Time.now.to_i
         b = pick_bucket(i)
         flags = 0
         flags = flags | Flags::GZIP if gzip
@@ -99,20 +99,33 @@ module Conveyor
         end
 
         @last_id = i
-        @index_file.write "#{header} #{b.to_s(36)}\n"
-        @index << {:id => i, :time => t, :offset => o, :length => l, :hash => h, :file => b}
+        index_offset = nil
+        index_file_lock do
+          @index_file.seek(0, IO::SEEK_END)
+          index_offset = @index_file.pos
+          @index_file.write "#{header} #{b.to_s(36)}\n"
+        end
+        block_num = block_num(i)
+        if !@blocks[block_num]
+          @blocks << {:offset => index_offset}
+        end
+        if @block_cache.key?(block_num)
+          @block_cache[block_num] << {:id => i, :time => t, :offset => o, :length => l, :hash => h, :file => b}
+        end
         i
       end
     end
 
     def get id, stream=false
       return nil unless id <= @last_id && id > 0
-      i = @index[id-1]
+
+      index_entry = search_index(id)
+
       headers, content, compressed_content, g = nil
-      bucket_file(i[:file]) do |f|
-        f.seek i[:offset]
-        headers  = parse_headers(f.readline.strip)
-        compressed_content = f.read(i[:length])
+      bucket_file(index_entry[:file]) do |f|
+        f.seek index_entry[:offset]
+        headers  = self.class.parse_headers(f.readline.strip)
+        compressed_content = f.read(index_entry[:length])
       end
       io = StringIO.new(compressed_content)
       if (headers[:flags] & Flags::GZIP) != 0
@@ -128,48 +141,89 @@ module Conveyor
     end
 
     def get_nearest_after_timestamp timestamp, stream=false
-      # i = binary search to find nearest item at or after timestamp
       i = nearest_after(timestamp)
       get(i) if i
     end
 
-    def self.parse_headers str, index_file=false
-      pattern =  '\A([a-z\d]+) ([a-z\d]+) ([a-z\d]+) ([a-z\d]+) ([a-f0-9]+) ([a-z\d]+)'
-      pattern += ' (\d+)' if index_file
-      pattern += '\Z'
-      m = str.match(Regexp.new(pattern))
+    def delete!
+      FileUtils.rm_r(@directory)
+      @data_files =[]
+      @last_id = 0
+      @blocks = []
+    end
+
+    def rebuild_index
+      files = Dir.glob(@directory + '/' + '[0-9]*')
+      files = files.map{|f| [f, f.split('/').last.to_i]}
+      files.sort!{|a,b| a[1] <=> b[1]}
+      files.each do |(f, b)|
+        File.open(f, 'r') do |file|
+          puts "reading #{f}"
+          while line = file.gets
+            headers = self.class.parse_headers(line.strip)
+            content = file.read(headers[:length])
+            file.read(1)
+            index_offset = nil
+            header = "#{headers[:id].to_s(36)} #{headers[:time].to_s(36)} #{headers[:offset].to_s(36)} #{headers[:length].to_s(36)} #{headers[:hash]} #{headers[:flags].to_s(36)}"
+            index_file_lock do
+              @index_file.seek(0, IO::SEEK_END)
+              index_offset = @index_file.pos
+              @index_file.write "#{header} #{b.to_s(36)}\n"
+            end
+          end
+        end
+      end
+    end
+
+    def status
       {
-        :id     => m.captures[0].to_i(36),
-        :time   => m.captures[1].to_i(36),
-        :offset => m.captures[2].to_i(36),
-        :length => m.captures[3].to_i(36),
-        :hash   => m.captures[4],
-        :flags  => m.captures[5].to_i(36),
-        :file   => (index_file ? m.captures[6].to_i(36) : nil)
+        :directory        => @directory,
+        :data_files => @data_files.collect{|f| {:path => f.path, :bytes => File.size(f.path)}},
+        :iterator         => @iterator,
+        :block_cache_keys => @block_cache.keys,
+        :last_id => @last_id,
+        :blocks => @blocks.length,
       }
     end
 
-    def parse_headers str, index_file=false
-      self.class.parse_headers str, index_file
+    protected
+
+    def block_num i
+      ((i-1) / BLOCK_SIZE)
     end
-    
-    def self.valid_channel_name? name
-      !!name.match(NAME_PATTERN)
+
+    def pick_bucket i
+      (i / BUCKET_SIZE).to_i
     end
 
-    def delete!
-      FileUtils.rm_r(@directory)
-      @index = []
-      @data_files =[]
-      @last_id = 0
+    def bucket_file i
+      unless @data_files[i]
+        @data_files[i] = File.open(File.join(@directory, i.to_s), 'a+')
+        @data_files[i].sync = true
+        @file_mutexes[i] = Mutex.new
+      end
+      @file_mutexes[i].synchronize do
+        yield @data_files[i]
+      end
     end
 
-    protected
+    def id_lock
+      @id_lock.synchronize do
+        yield
+      end
+    end
+
+    def index_file_lock
+      @index_file_lock.synchronize do
+        yield
+      end
+    end
 
     def setup_channel
-      @index_file = File.open(index_path, 'a')
+      @index_file = File.open(index_path, 'a+')
       @last_id = 0
       @version = FORMAT_VERSION
+      @blocks = []
       File.open(version_path, 'w+'){|f| f.write(@version.to_s)}
     end
 
@@ -185,16 +239,21 @@ module Conveyor
       end
 
       @index_file = File.open(index_path, 'r+')
-
+      @blocks = []
       @last_id = 0
-      @index_file.each_line do |line|
-        @index << parse_headers(line.strip, true)
-        @last_id = @index.last[:id]
+      index_offset = 0
+      while line = @index_file.gets
+        entry = self.class.parse_headers(line.strip, true)
+        raise "corrupt index. try rebuilding." unless entry[:id] == @last_id + 1
+        if entry[:id] % BLOCK_SIZE == 1
+          @blocks << {:offset => index_offset}
+        end
+        @last_id = entry[:id]
+        index_offset = @index_file.pos
       end
       @index_file.seek(0, IO::SEEK_END)
     end
 
-
     def index_path
       File.join(@directory, 'index')
     end
@@ -203,21 +262,57 @@ module Conveyor
       File.join(@directory, 'version')
     end
 
-    def nearest_after(timestamp)
-      low = 0
-      high = @index.length
-      while low < high
-        mid = (low + high) / 2
-        if (@index[mid][:time].to_i > timestamp)
-          high = mid - 1
-        elsif (@index[mid][:time].to_i < timestamp)
-          low = mid + 1
+    def cache_block block_num
+      if @block_cache.length > CACHE_SIZE
+        reject = @block_last_used.sort{|a,b| a[1] <=> b[1]}.last.first
+        @block_cache.delete(reject)
+        puts "rejected #{reject}"
+      end
+      a = []
+
+      buf = ''
+      block_start = @blocks[block_num][:offset]
+      block_end   = @blocks[block_num + 1] ? @blocks[block_num + 1][:offset] : nil
+      index_file_lock do
+        @index_file.seek(block_start)
+        if block_end
+          buf = @index_file.read(block_end - block_start)
         else
-          return mid
+          buf = @index_file.read
+        end
+        @index_file.seek(0, IO::SEEK_END)
+      end
+      buf.split(/\n/).each do |line|
+        a << self.class.parse_headers(line.strip, true)
+      end
+      @block_cache[block_num] = a
+    end
+
+    def search_index id
+      block_num = block_num(id)
+
+      if !@block_cache.has_key?(block_num)
+        cache_block(block_num)
+      end
+      @block_last_used[block_num] = Time.now.to_i
+      entry = @block_cache[block_num][id - 1 - (block_num * BLOCK_SIZE)]
+    end
+
+    def nearest_after(timestamp)
+      i = 0
+      while (i < @blocks.length - 1) && timestamp < @blocks[i+1][:time]
+        i += 1
+      end
+      cache_block(i) if !@block_cache.has_key?(i)
+      @block_last_used[i] = Time.now.to_i
+      @block_cache[i].each do |entry|
+        if entry[:time] > timestamp
+          return entry[:id]
         end
       end
-      if timestamp <= @index[mid][:time].to_i
-        @index[mid][:id]
+      if @blocks[i+1]
+        cache_block(i+1)
+        @block_cache[i+1].first[:id]
       else
         nil
       end

data/lib/conveyor/channel.rb

@@ -85,15 +85,9 @@ module Conveyor
     end
 
     def status
-      {
-        :directory => @directory,
-        :index     => {
-          :size => @index.length
-        },
-        :data_files => @data_files.collect{|f| {:path => f.path, :bytes => File.size(f.path)}},
-        :iterator => {:position => @iterator},
-        :iterator_groups => @group_iterators.inject({}){|m,(k,v)| m[k] = v; m}
-      }
+      super.merge({
+        :iterator_groups => @group_iterators.inject({}){|m,(k,v)| m[k] = v; m},
+      })
     end
 
     def rewind *opts
@@ -146,7 +140,8 @@ module Conveyor
 
     def load_channel
       super
-      @iterator_file = File.open(iterator_path, 'r+')
+      @iterator_file = File.open(iterator_path, 'a+')
+      @iterator_file.seek(0)
       @iterator_file.each_line do |line|
         @iterator = line.to_i(36)
       end

data/lib/conveyor/server.rb

@@ -21,13 +21,19 @@ module Conveyor
         @logger = Logger.new '/dev/null'
       end
 
+      t0 = Time.now
+      i "reading data"
+
       @channels = {}
       Dir.entries(@data_directory).each do |e|
         if !['.', '..'].include?(e) && File.directory?(File.join(@data_directory, e)) && Channel.valid_channel_name?(e)
+          i "initializing channel '#{e}'"
           @channels[e] = Channel.new(File.join(@data_directory, e))
         end
       end
 
+      i "done reading data (took #{Time.now - t0} sec.)"
+
       @requests = 0
     end
 
@@ -50,7 +56,7 @@ module Conveyor
           i "#{env["REMOTE_ADDR"]} PUT #{env["REQUEST_PATH"]} 201"
           [201, {}, "created channel #{m.captures[0]}"]
         else
-          i "#{env["REMOTE_ADDR"]} PUT #{env["REQUEST_PATH"]}"
+          i "#{env["REMOTE_ADDR"]} PUT #{env["REQUEST_PATH"]} 202"
           [202, {}, "channel already exists. didn't do anything"]
         end
       else

data/test/test_channel.rb

@@ -21,15 +21,12 @@ class TestConveyorChannel < Test::Unit::TestCase
   end
 
   def test_parse_headers
-    FileUtils.rm_r '/tmp/foo' rescue nil
-
-    i = Channel.new '/tmp/foo'
     [
       ["1 jucfvs 0 3 acbd18db4cc2f85cedef654fccc4a4d8 0\n", 
        {:id => 1, :time => 1199826280, :offset => 0, :length => 3, :hash => "acbd18db4cc2f85cedef654fccc4a4d8", :file => nil, :flags => 0}
       ]
     ].each do |(str, ret)|
-      assert_equal ret, i.parse_headers(str)
+      assert_equal ret, Channel.parse_headers(str)
     end
     
     [
@@ -37,10 +34,8 @@ class TestConveyorChannel < Test::Unit::TestCase
        {:id => 2, :time => 1199826280, :offset => 0, :length => 3, :hash => "acbd18db4cc2f85cedef654fccc4a4d8", :file => 1, :flags => 0}
       ]
     ].each do |(str, ret)|
-      assert_equal ret, i.parse_headers(str, true)
+      assert_equal ret, Channel.parse_headers(str, true)
     end
-    
-    FileUtils.rm_r '/tmp/foo'
   end
 
   def test_init_existing
@@ -70,7 +65,7 @@ class TestConveyorChannel < Test::Unit::TestCase
     assert_equal 'bar', c.get_next[1]
     assert_equal 'bam', c.get_next[1]
     assert_equal nil, c.get_next
-    assert_equal 4, c.status[:iterator][:position]
+    assert_equal 4, c.status[:iterator]
   end
 
   def test_get_next_interupted
@@ -138,12 +133,14 @@ class TestConveyorChannel < Test::Unit::TestCase
     
     status = {
       :directory => '/tmp/bar',
-      :index => {:size => 3},
+      :blocks => 1,
       :data_files => [
         {:path => '/tmp/bar/0', :bytes => 158}
         ],
-      :iterator => {:position => 1},
-      :iterator_groups => {}
+      :iterator => 1,
+      :iterator_groups => {},
+      :last_id => 3,
+      :block_cache_keys => []
     }
     
     assert_equal(status, c.status)
@@ -261,8 +258,10 @@ class TestConveyorChannel < Test::Unit::TestCase
     assert_equal nil, c.get_nearest_after_timestamp(2**32)
 
     t0 = Time.now.to_i
+    sleep 1 # we only have second-level granularity on time
+
     10.times{|i| c.post((10 + i).to_s)}
-    assert_equal '9', c.get_nearest_after_timestamp(t0)[1]
+    assert_equal '10', c.get_nearest_after_timestamp(t0)[1]
   end
   
   def test_rewind_to_timestamp

data/test/test_replicated_channel.rb

@@ -5,11 +5,13 @@ require 'priority_queue'
 
 class TestReplicatedChannel < Test::Unit::TestCase
   def test_basic
-    FileUtils.rm_r '/tmp/foo' rescue nil
-    FileUtils.rm_r '/tmp/bar' rescue nil
+    chan1 = '/tmp/test_basic1'
+    chan2 = '/tmp/test_basic2'
+    FileUtils.rm_r chan1 rescue nil
+    FileUtils.rm_r chan2 rescue nil
     
-    c1 = Conveyor::ReplicatedChannel.new '/tmp/foo'
-    c2 = Conveyor::ReplicatedChannel.new '/tmp/bar'
+    c1 = Conveyor::ReplicatedChannel.new chan1
+    c2 = Conveyor::ReplicatedChannel.new chan2
     
     c1.peers << c2
     c2.peers << c1
@@ -34,8 +36,8 @@ class TestReplicatedChannel < Test::Unit::TestCase
     assert_equal c1.get(1), c2.get(1)
     assert_equal c1.get(2), c2.get(2)
 
-    FileUtils.rm_r '/tmp/foo' rescue nil
-    FileUtils.rm_r '/tmp/bar' rescue nil
+    FileUtils.rm_r chan1 rescue nil
+    FileUtils.rm_r chan2 rescue nil
   end
   
   def test_more

data/test/test_server.rb

@@ -19,6 +19,7 @@ class TestConveyorServer < Test::Unit::TestCase
 
     Rack::Handler::Mongrel.run(app, :Host => '0.0.0.0', :Port => 8011)
    end
+   sleep 1
 
   def test_channels
     Net::HTTP.start("localhost", 8011) do |h|
@@ -117,10 +118,12 @@ class TestConveyorServer < Test::Unit::TestCase
       assert_kind_of Net::HTTPOK, req
       json = {
         "iterator_groups" => {},
-        "index"=>{"size"=>1},
+        "blocks" => 1,
         "directory"=>"/tmp/asdf/#{chan}",
         "data_files"=>[{"path"=>"/tmp/asdf/#{chan}/0","bytes"=>122}],
-        "iterator"=>{"position"=>1}
+        "iterator"=>1,
+        "last_id" => 1,
+        "block_cache_keys" => []
       }
       assert_equal json, JSON::parse(req.body)
       
@@ -287,8 +290,9 @@ class TestConveyorServer < Test::Unit::TestCase
     assert_equal '', c.get_nearest_after_timestamp(2**32)
 
     t0 = Time.now.to_i
+    sleep 1
     10.times{|i| c.post((10 + i).to_s)}
-    assert_equal '9', c.get_nearest_after_timestamp(t0)
+    assert_equal '10', c.get_nearest_after_timestamp(t0)
   end
 
   def test_rewind_to_timestamp

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: conveyor
 version: !ruby/object:Gem::Version 
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors: 
 - Ryan King
@@ -30,11 +30,12 @@ cert_chain:
   Zls3y84CmyAEGg==
   -----END CERTIFICATE-----
 
-date: 2008-04-10 00:00:00 -07:00
+date: 2008-10-21 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: mongrel
+  type: :runtime
   version_requirement: 
   version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
@@ -44,6 +45,7 @@ dependencies:
     version: 
 - !ruby/object:Gem::Dependency 
   name: json
+  type: :runtime
   version_requirement: 
   version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
@@ -53,6 +55,17 @@ dependencies:
     version: 
 - !ruby/object:Gem::Dependency 
   name: daemons
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: rack
+  type: :runtime
   version_requirement: 
   version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
@@ -62,17 +75,19 @@ dependencies:
     version: 
 - !ruby/object:Gem::Dependency 
   name: hoe
+  type: :development
   version_requirement: 
   version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 1.5.1
+        version: 1.7.0
     version: 
 description: "* Like TiVo for your data * A distributed rewindable multi-queue"
 email: ryan@theryanking.com
 executables: 
 - conveyor
+- conveyor-rebuild-index
 - conveyor-upgrade
 extensions: []
 
@@ -87,6 +102,7 @@ files:
 - README.txt
 - Rakefile
 - bin/conveyor
+- bin/conveyor-rebuild-index
 - bin/conveyor-upgrade
 - docs/file-formats.rdoc
 - docs/protocol.rdoc
@@ -125,7 +141,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: conveyor
-rubygems_version: 1.1.0
+rubygems_version: 1.2.0
 signing_key: 
 specification_version: 2
 summary: "* Like TiVo for your data * A distributed rewindable multi-queue"