conveyor 0.1.4 → 0.2.0

data/lib/conveyor/client.rb

@@ -2,53 +2,54 @@ require 'net/http'
 
 module Conveyor
   class Client
-    def initialize host, port = 8011
-      @host = host
-      @port = port
+    def initialize host, channel, port = 8011
+      @host    = host
+      @port    = port
+      @channel = channel
       connect!
     end
-    
+
     def connect!
       @conn = Net::HTTP.start(@host, @port)
     end
 
-    def create_channel channel_name
-      @conn.put("/channels/#{channel_name}", nil, {'Content-Type' => 'application/octet-stream'})
+    def create_channel
+      @conn.put("/channels/#{@channel}", nil, {'Content-Type' => 'application/octet-stream'})
     end
 
-    def post channel_name, content
-      @conn.post("/channels/#{channel_name}", content, {'Content-Type' => 'application/octet-stream', 'Date' => Time.now.to_s})
+    def post content
+      @conn.post("/channels/#{@channel}", content, {'Content-Type' => 'application/octet-stream', 'Date' => Time.now.gmtime.to_s})
     end
     
-    def get channel_name, id
-      @conn.get("/channels/#{channel_name}/#{id}").body
+    def get id
+      @conn.get("/channels/#{@channel}/#{id}").body
     end
 
-    def get_next channel_name, group=nil
+    def get_next group=nil
       if group
-        @conn.get("/channels/#{channel_name}?next&group=#{group}").body
+        @conn.get("/channels/#{@channel}?next&group=#{group}").body
       else
-        @conn.get("/channels/#{channel_name}?next").body
+        @conn.get("/channels/#{@channel}?next").body
       end
     end
 
-    def channel_status channel_name
-      JSON::parse(@conn.get("/channels/#{channel_name}").body)
+    def status
+      JSON::parse(@conn.get("/channels/#{@channel}").body)
     end
 
-    def get_next_n channel_name, n = 10, group = nil
+    def get_next_n n = 10, group = nil
       if group
-        JSON.parse(@conn.get("/channels/#{channel_name}?next&n=#{n}&group=#{group}").body)
+        JSON.parse(@conn.get("/channels/#{@channel}?next&n=#{n}&group=#{group}").body)
       else
-        JSON.parse(@conn.get("/channels/#{channel_name}?next&n=#{n}").body)
+        JSON.parse(@conn.get("/channels/#{@channel}?next&n=#{n}").body)
       end
     end
     
-    def rewind channel_name, id, group=nil
+    def rewind id, group=nil
       if group
-        @conn.post("/channels/#{channel_name}?rewind_id=#{id}&group=#{group}", nil)
+        @conn.post("/channels/#{@channel}?rewind_id=#{id}&group=#{group}", nil)
       else
-        @conn.post("/channels/#{channel_name}?rewind_id=#{id}", nil)
+        @conn.post("/channels/#{@channel}?rewind_id=#{id}", nil)
       end
     end
   end

data/lib/conveyor/server.rb

@@ -5,171 +5,160 @@ require 'fileutils'
 require 'json'
 require 'logger'
 
-class Mongrel::HttpRequest
-  def put?
-    params["REQUEST_METHOD"] == "PUT"
-  end
-  
-  def post?
-    params["REQUEST_METHOD"] == "POST"
-  end
-  
-  def get?
-    params["REQUEST_METHOD"] == "GET"
-  end
-
-  def path_match pattern
-    params["REQUEST_PATH"].match(pattern)
-  end
-end
-
 module Conveyor
-  
-  # An HTTP server for Conveyor.
-  class Server < Mongrel::HttpServer
+  class App
+    def initialize(data_directory, *options)
+      options = options.inject(){|(k, v), m| m[k] = v; m}
+      @data_directory = data_directory
+      @log_directory  = options[:log_directory]
+      @unsafe_mode    = options[:unsafe_mode] # allows deleting of channels. REALLY UNSAFE!
 
-    # A Mongrel handler for multiple Conveyor Channels.
-    class ChannelsHandler < Mongrel::HttpHandler
-
-      def initialize data_directory, log_directory=nil
-        @data_directory = data_directory
-        if log_directory
-          @logger = Logger.new File.join(log_directory, 'conveyor.log')
-        else
-          @logger = Logger.new '/dev/null'
-        end
+      if @log_directory
+        @logger = Logger.new File.join(@log_directory, 'conveyor.log')
+      else
+        @logger = Logger.new '/dev/null'
+      end
 
-        @channels = {}
-        Dir.entries(@data_directory).each do |e|
-          if !['.', '..'].include?(e) && File.directory?(File.join(@data_directory, e))
-            @channels[e] = Channel.new(File.join(@data_directory, e))
-          end
+      @channels = {}
+      Dir.entries(@data_directory).each do |e|
+        if !['.', '..'].include?(e) && File.directory?(File.join(@data_directory, e)) && Channel.valid_channel_name?(e)
+          @channels[e] = Channel.new(File.join(@data_directory, e))
         end
       end
 
-      def create_new_channel channel_name
-        @channels[channel_name] = Conveyor::Channel.new(File.join(@data_directory, channel_name))
-      end
+      @requests = 0
+    end
+
+    def path_match env, pattern
+      env["REQUEST_PATH"].match(pattern)
+    end
 
-      def i str
-        @logger.info str
+    def create_new_channel channel_name
+      @channels[channel_name] = Conveyor::Channel.new(File.join(@data_directory, channel_name))
+    end
+
+    def i msg
+      @logger.info msg
+    end
+
+    def put env, m
+      if Channel.valid_channel_name?(m.captures[0])
+        if !@channels.key?(m.captures[0])
+          create_new_channel m.captures[0]
+          i "#{env["REMOTE_ADDR"]} PUT #{env["REQUEST_PATH"]} 201"
+          [201, {}, "created channel #{m.captures[0]}"]
+        else
+          i "#{env["REMOTE_ADDR"]} PUT #{env["REQUEST_PATH"]}"
+          [202, {}, "channel already exists. didn't do anything"]
+        end
+      else
+        i "#{env["REMOTE_ADDR"]} GET #{env["REQUEST_PATH"]} 406"
+        [406, {}, "invalid channel name. must match #{Channel::NAME_PATTERN}"]
       end
+    end
 
-      def process request, response
-        if request.put? && m = request.path_match(%r{/channels/(.*)})
-          if Channel.valid_channel_name?(m.captures[0])
-            if !@channels.key?(m.captures[0])
-              create_new_channel m.captures[0]
-              response.start(201) do |head, out|
-                out.write("created channel #{m.captures[0]}")
-              end
-              i "#{request.params["REMOTE_ADDR"]} PUT #{request.params["REQUEST_PATH"]} 201"
-            else
-              response.start(202) do |head, out|
-                out.write("channel already exists. didn't do anything")
-              end
-              i "#{request.params["REMOTE_ADDR"]} PUT #{request.params["REQUEST_PATH"]}   "
-            end
+    def post env, m
+      if @channels.key?(m.captures[0])
+        params = Mongrel::HttpRequest.query_parse(env['QUERY_STRING'])
+        if params.key?('rewind_id')
+          if params['group']
+            @channels[m.captures[0]].rewind(:id => params['rewind_id'], :group => params['group']).to_i # TODO make sure this is an integer
+            [200, {}, "iterator rewound to #{params['rewind_id']}"]
           else
-            response.start(406) do |head, out|
-              out.write("invalid channel name. must match #{Channel::NAME_PATTERN}")
-              i "#{request.params["REMOTE_ADDR"]} GET #{request.params["REQUEST_PATH"]} 406"
-            end
+            @channels[m.captures[0]].rewind(:id => params['rewind_id']).to_i # TODO make sure this is an integer
+            [200, {}, "iterator rewound to #{params['rewind_id']}"]
           end
-        elsif request.post? && m = request.path_match(%r{/channels/(.*)})
-          if @channels.key?(m.captures[0])
-            params = Mongrel::HttpRequest.query_parse(request.params['QUERY_STRING'])
-            if params.key?('rewind_id')
-              if params['group']
-                @channels[m.captures[0]].rewind(:id => params['rewind_id'], :group => params['group']).to_i # TODO make sure this is an integer
-                response.start(200) do |head, out|
-                  out.write "iterator rewound to #{params['rewind_id']}"
-                end
+        else
+          if env.key?('HTTP_DATE') && d = Time.parse(env['HTTP_DATE'])
+            id = @channels[m.captures[0]].post(env['rack.input'].read)
+            i "#{env["REMOTE_ADDR"]} POST #{env["REQUEST_PATH"]} 202"
+            [202, {"Location" => "/channels/#{m.captures[0]}/#{id}"}, ""]
+          else
+            i "#{env["REMOTE_ADDR"]} POST #{env["REQUEST_PATH"]} 400"
+            [400, {}, "A valid Date header is required for all POSTs."]
+          end
+        end
+      elsif Channel.valid_channel_name?(m.captures[0])
+        create_new_channel(m.captures[0])
+        post(env, m)
+      else
+        [404, {}, '']
+      end
+    end
+
+    def get env
+      headers = content = nil
+      if m = path_match(env, %r{/channels/(.*)/(\d+)})
+        if @channels.key?(m.captures[0])
+          headers, content = @channels[m.captures[0]].get(m.captures[1].to_i)
+        end
+      elsif m = path_match(env, %r{/channels/(.*)})
+        if @channels.key?(m.captures[0])
+          params = Mongrel::HttpRequest.query_parse(env['QUERY_STRING'])
+          if params.key? 'next'
+            if params.key? 'group'
+              if params.key? 'n'
+                list = @channels[m.captures[0]].get_next_n_by_group(params['n'].to_i, params['group'])
               else
-                @channels[m.captures[0]].rewind(:id => params['rewind_id']).to_i # TODO make sure this is an integer
-                response.start(200) do |head, out|
-                  out.write "iterator rewound to #{params['rewind_id']}"
-                end
+                headers, content = @channels[m.captures[0]].get_next_by_group(params['group'])
               end
             else
-              if request.params.include?('HTTP_DATE') && d = Time.parse(request.params['HTTP_DATE'])
-                id = @channels[m.captures[0]].post(request.body.read)
-                response.start(202) do |head, out|
-                  head["Location"] = "/channels/#{m.captures[0]}/#{id}"
+              if params.key? 'n'
+                list = @channels[m.captures[0]].get_next_n(params['n'].to_i).map do |i|
+                  {:data => i[1], :hash => i[0][:hash], :id => i[0][:id]}
                 end
-                i "#{request.params["REMOTE_ADDR"]} POST #{request.params["REQUEST_PATH"]} 202"
               else
-                response.start(400) do |head, out|
-                  out.write "A valid Date header is required for all POSTs."
-                end
-                i "#{request.params["REMOTE_ADDR"]} POST #{request.params["REQUEST_PATH"]} 400"
-              end
-            end
-          end
-
-        elsif request.get?
-          headers = content = nil
-          if m = request.path_match(%r{/channels/(.*)/(\d+)})
-            if @channels.key?(m.captures[0])
-              headers, content = @channels[m.captures[0]].get(m.captures[1].to_i)
-            end
-          elsif m = request.path_match(%r{/channels/(.*)})
-            if @channels.key?(m.captures[0])
-              params = Mongrel::HttpRequest.query_parse(request.params['QUERY_STRING'])
-              if params.key? 'next'
-                if params.key? 'group'
-                  if params.key? 'n'
-                    list = @channels[m.captures[0]].get_next_n_by_group(params['n'].to_i, params['group'])
-                  else
-                    headers, content = @channels[m.captures[0]].get_next_by_group(params['group'])
-                  end
-                else
-                  if params.key? 'n'
-                    list = @channels[m.captures[0]].get_next_n(params['n'].to_i)
-                    list = list.map do |i|
-                      {:data => i[1], :hash => i[0][:hash], :id => i[0][:id]}
-                    end
-                  else
-                    headers, content = @channels[m.captures[0]].get_next
-                  end
-                end
-              else
-                response.start(200) do |head, out|
-                  out.write @channels[m.captures[0]].status.to_json
-                end
+                headers, content = @channels[m.captures[0]].get_next
               end
             end
           else
-            response.start(200) do |head, out|
-              out.write("fake!")
-            end
-          end
-
-          if headers && content
-            response.start(200) do |head, out|
-              head['Content-Location'] = "/channels/#{m.captures[0]}/#{headers[:id]}"
-              head['Content-MD5']      = headers[:hash]
-              head['Content-Type']     = 'application/octet-stream'
-              head['Last-Modified']    = Time.parse(headers[:time]).gmtime.to_s
-              out.write content
-            end
-            i "#{request.params["REMOTE_ADDR"]} GET #{request.params["REQUEST_PATH"]} 200 #{headers[:id]} #{headers[:length]} #{headers[:hash]}"
-          elsif list
-            response.start(200) do |head, out|
-              out.write list.to_json
-            end
+            return [200, {}, @channels[m.captures[0]].status.to_json]
           end
-          
         end
+      else
+        return [200, {}, "fake!"]
       end
+
+      if headers && content
+        i "#{env["REMOTE_ADDR"]} GET #{env["REQUEST_PATH"]} 200 #{headers[:id]} #{headers[:length]} #{headers[:hash]}"
+        return [
+          200,
+          {
+            'Content-Location' => "/channels/#{m.captures[0]}/#{headers[:id]}",
+            'Content-MD5'      => headers[:hash],
+            'Content-Type'     => 'application/octet-stream',
+            'Last-Modified'    => Time.at(headers[:time]).gmtime.to_s,
+          },
+            content
+        ]
+      elsif list
+        return [200, {}, list.to_json]
+      end
+
+      return [404, {}, '']
     end
 
-    # +host+ and +port+ are passed along to Mongrel::HttpServer for TCP binding. +data_directory+ is used to store
-    # all channel data and should be created before intializing a Server.
-    def initialize(host, port, data_directory, log_directory = nil)
-      super(host, port)
-      ch = ChannelsHandler.new(data_directory, log_directory)
-      register("/channels", ch)
+    def delete env, m
+      if @channels.key?(m.captures[0])
+        @channels[m.captures[0]].delete!
+        @channels.delete(m.captures[0])
+        [200, {}, "Channel deleted."]
+      end
+    end
+
+    def call(env)
+      @requests += 1
+      if env['REQUEST_METHOD']    == 'PUT'  && m = path_match(env, %r{/channels/(.*)})
+        put(env, m)
+      elsif env['REQUEST_METHOD'] == 'POST' && m = path_match(env, %r{/channels/(.*)})
+        post(env, m)
+      elsif @unsafe_mode && env['REQUEST_METHOD'] == 'DELETE' && m = path_match(env, %r{/channels/(.*)})
+        delete(env, m)
+      elsif env['REQUEST_METHOD'] == 'GET'
+        get(env)
+      else
+        [404, {}, '']
+      end
     end
   end
 end

data/lib/conveyor/upgrader.rb

@@ -0,0 +1,119 @@
+require 'conveyor/base_channel'
+
+def String.random_alphanumeric(size=16)
+  s = ""
+  size.times { s << (i = Kernel.rand(62); i += ((i < 10) ? 48 : ((i < 36) ? 55 : 61 ))).chr }
+  s
+end
+
+
+module Conveyor
+class Upgrader
+
+  def initialize directory
+    @directory = directory
+  end
+
+  def version_path
+    File.join(@directory, 'version')
+  end
+
+  def source_version
+    if File.exists?(version_path) && File.size(version_path) > 0
+      File.open(version_path).read.to_i
+    else
+      0
+    end
+  end
+
+  def upgrade
+    case source_version
+    when 0
+      from_0
+    end
+  end
+
+  def parse_headers_0 str, index_file = false
+    pattern =  '\A(\d+) (\d{4}\-\d{2}\-\d{2}T\d{2}\:\d{2}\:\d{2}[+\-]\d{2}\:\d{2}) (\d+) (\d+) ([a-f0-9]+)'
+    pattern += ' (\d+)' if index_file
+    pattern += '\Z'
+    m = str.match(Regexp.new(pattern))
+    {
+      :id => m.captures[0].to_i,
+      :time => m.captures[1],
+      :offset => m.captures[2].to_i,
+      :length => m.captures[3].to_i,
+      :hash => m.captures[4], 
+      :file => (index_file ? m.captures[5].to_i : nil)
+    }.reject {|k,v| v == nil}
+  end
+
+  def from_0
+    Dir.glob(@directory + "/*").each do |d|
+
+      if File.directory?(d)
+        # create tmp dir
+        tmp_dir = create_tmp_dir!
+        puts "writing to #{tmp_dir}"
+        chan = Channel.new(tmp_dir)
+
+        Dir.glob("#{d}/[0-9]*").each do |f|
+          puts "upgrading #{f}"
+          size = File.size(f)
+          f    = File.open(f)
+
+          while f.pos < size
+            l       = f.readline.strip
+            header  = parse_headers_0(l)
+            content = f.read(header[:length])
+            f.readline # newline chomp
+
+            chan.commit(content, Time.parse(header[:time]))
+          end
+        end
+
+        puts "upgrading iterator"
+        iterator = File.open(File.join(d, 'iterator')) do |f|
+          l = nil
+          while !f.eof? && l = f.readline
+          end
+          if l
+            chan.rewind :id => l.strip.to_i
+          end
+        end
+
+        Dir.glob(File.join(d, 'iterator-*')) do |i|
+          group = i.split('/').last.split('-').last
+          puts "upgrading group iterator for #{group}"
+          iterator = File.open(i) do |f|
+            l = nil
+            while !f.eof? && l = f.readline
+            end
+            if l
+              chan.rewind :id => l.strip.to_i, :group => group
+            end
+          end
+        end
+
+        puts "backing up #{d} to #{d}.bak"
+        FileUtils.mv d, d + ".bak"
+
+        puts "copying from #{tmp_dir} to #{d}"
+        FileUtils.cp_r tmp_dir, d
+
+        puts "deleting temp data"
+        FileUtils.rm_r tmp_dir
+      end
+    end
+  end
+
+  def create_tmp_dir!
+    loop do
+      tmp_dir = File.join('/tmp', String.random_alphanumeric)
+      if !File.exists?(tmp_dir)
+        return tmp_dir
+      end
+    end
+  end
+end
+end