couchproxy 0.1.0

data/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2010 David Graham
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README

@@ -0,0 +1,36 @@
+== Welcome to CouchProxy
+
+CouchProxy is a simple proxy server that distributes reads and writes to a
+cluster of Apache CouchDB servers so they appear to be a single huge database.
+Documents are stored and retrieved from a particular CouchDB instance, using
+consistent hashing of the document id. Map/reduce views are processed
+concurrently on each CouchDB instance and merged together by the proxy before
+returning the results to the client.
+
+CouchProxy uses the Thin web server and EventMachine for asynchronous IO. There
+are no blocking IO calls in the server, which allows it to handle many clients
+and many backend CouchDB nodes concurrently, using very little processor and
+memory.
+
+== Usage
+
+1. gem install couchproxy
+2. couchproxy config
+3. Configure CouchDB nodes in couchproxy.yml
+4. couchproxy start
+
+== Dependencies
+
+* em-http-request >= 0.2.11
+* json >= 1.4.6
+* json-stream >= 0.1.0
+* thin >= 1.2.7
+* ruby >= 1.9.1
+
+== Contact
+
+Project contact: David Graham <david.malcom.graham@gmail.com>
+
+== License
+
+CouchProxy is released under the MIT license.  Check the LICENSE file for details.

data/Rakefile

@@ -0,0 +1,42 @@
+require 'rake'
+require 'rake/clean'
+require 'rake/gempackagetask'
+require 'rake/testtask'
+require File.expand_path('../lib/couchproxy', __FILE__)
+
+spec = Gem::Specification.new do |s| 
+  s.name = "couchproxy"
+  s.version = CouchProxy::VERSION
+  s.date = Time.now.strftime("%Y-%m-%d")
+  s.summary = "A proxy server for Apache CouchDB clusters."
+  s.description = "CouchProxy is a simple proxy server that distributes reads and writes to a
+cluster of Apache CouchDB servers so they appear to be a single huge database.
+Documents are stored and retrieved from a particular CouchDB instance, using
+consistent hashing of the document id. Map/reduce views are processed
+concurrently on each CouchDB instance and merged together by the proxy before
+returning the results to the client."
+  s.email = "david.malcom.graham@gmail.com"
+  s.homepage = "http://github.com/dgraham/couchproxy"
+  s.authors = ["David Graham"]
+  s.files = FileList['[A-Z]*', '{bin,lib,conf}/**/*']
+  s.test_files = FileList["test/**/*test.rb"]
+  s.executables = %w[couchproxy]
+  s.require_path = "lib"
+  s.has_rdoc = true
+  s.add_dependency('em-http-request', '~> 0.2')
+  s.add_dependency('json', '~> 1.4')
+  s.add_dependency('json-stream', '~> 0.1')
+  s.add_dependency('thin', '~> 1.2')
+  s.required_ruby_version = '>= 1.9.1'
+end
+ 
+Rake::GemPackageTask.new(spec) do |pkg|
+  pkg.need_tar = true 
+end 
+
+Rake::TestTask.new(:test) do |test|
+  test.pattern = 'test/**/*_test.rb'
+  test.warning = true
+end
+
+task :default => [:clobber, :test, :gem]

data/bin/couchproxy

@@ -0,0 +1,88 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+require 'fileutils'
+require 'optparse'
+
+def parse
+  options = {}
+  parser = OptionParser.new do |opts|
+    opts.banner = "Usage: couchproxy [options] start|stop|config"
+
+    options[:config] = default_config
+    opts.on('-C', '--config FILE',
+              "Specify couchproxy.yml file (default: #{default_config})") do |config|
+      options[:config] = config
+    end
+
+    options[:host] = '0.0.0.0'
+    opts.on('-a', '--address HOST',
+              'Bind to HOST address (default: 0.0.0.0)') do |host|
+      options[:host] = host
+    end
+
+    options[:port] = 6984
+    opts.on('-p', '--port PORT', 'Use PORT (default: 6984)') do |port|
+      options[:port] = port
+    end
+
+    options[:log] = 'couchproxy.log'
+    opts.on('-l', '--log FILE',
+              'File to redirect output (default: couchproxy.log)') do |log|
+      options[:log] = log
+    end
+
+    options[:pid] = 'couchproxy.pid'
+    opts.on('-P', '--pid FILE',
+              'File to store PID (default: couchproxy.pid)') do |pid|
+      options[:pid] = pid
+    end
+  end
+
+  begin
+    parser.parse!
+  rescue
+    puts parser
+    exit(1)
+  end
+
+  options.tap do |opts|
+    if ARGV.size != 1 || !%w[start stop config].include?(ARGV.first)
+      puts parser
+      exit(1)
+    end
+    opts[:cmd] = ARGV.first
+  end
+end
+
+def default_config
+  File.join(Dir.pwd, 'couchproxy.yml')
+end
+
+def start(opts)
+  ENV['COUCH_PROXY_CONFIG'] = opts[:config]
+  flags = {
+    '-a' => opts[:host],
+    '-p' => opts[:port],
+    '-l' => opts[:log],
+    '-P' => opts[:pid],
+    '-R' => File.expand_path('../../lib/couchproxy.ru', __FILE__),
+    '--tag' => 'couchproxy'
+  }
+  system("thin -d #{flags.map {|k, v| "#{k} #{v}" }.join(' ')} start")
+end
+
+def stop(opts)
+  system("thin -P #{opts[:pid]} stop")
+end
+
+def config(opts)
+  template = File.expand_path('../../conf/couchproxy.yml', __FILE__)
+  FileUtils.copy(template, default_config)
+  puts "Copied template cluster file to #{default_config}."
+  puts "Update the file and start the server."
+end
+
+opts = parse
+method(opts[:cmd]).call(opts)
+

data/conf/couchproxy.yml

@@ -0,0 +1,21 @@
+# This is the couchproxy configuration file. Restart the couchproxy process
+# after changing this file for the changes to take effect.
+#
+# Parameters:
+# couchjs:  The fully qualified path to the couchjs view server.
+# reducers: The number of couchjs processes to run. We typically want one
+#           view server process per core for best rereduce performance.
+# nodes:    A list of CouchDB servers with the partitions that they host. Once
+#           we start using the proxy with these partitions, we can't change the
+#           total number of partitions without carefully repartitioning the database.
+#           We can move partitions to new servers easily though. The number of
+#           partitions should match the maximum number of servers we think we'll
+#           need for our data set.
+
+couchjs: /usr/bin/couchjs
+reducers: 4
+nodes:
+  - host: 'http://localhost:5984'
+    partitions: !ruby/range 0...32
+  - host: 'http://localhost:5984'
+    partitions: !ruby/range 32...64

data/lib/couchproxy/cluster.rb

@@ -0,0 +1,43 @@
+# encoding: UTF-8
+
+module CouchProxy
+  class Cluster
+    attr_reader :nodes
+
+    def initialize(nodes, couchjs, reducers)
+      @nodes, @partitions = [], 0
+      nodes.each {|n| self << n }
+      @reducers = Array.new(reducers) { Reducer.new(couchjs) }
+    end
+
+    def reducer
+      @reducers[rand(@reducers.size)]
+    end
+
+    def <<(node)
+      @nodes << node
+      @partitions = @nodes.inject(0) do |acc, n|
+        acc + n.partitions.size
+      end
+      self
+    end
+
+    def partition(doc_id)
+      num = Zlib.crc32(doc_id.to_s).abs % @partitions
+      node = @nodes.find {|n| n.hosts?(num) }
+      node.partition(num)
+    end
+
+    def any_node
+      @nodes[rand(@nodes.size)]
+    end
+
+    def any_partition
+      any_node.any_partition
+    end
+
+    def partitions
+      @nodes.map {|n| n.partitions}.flatten
+    end
+  end
+end

data/lib/couchproxy/collator.rb

@@ -0,0 +1,60 @@
+module CouchProxy
+  # Implements the JSON sorting rules defined at
+  # http://wiki.apache.org/couchdb/View_collation.
+  class Collator
+    CLASSES = [NilClass, FalseClass, TrueClass, Numeric, String, Array, Hash]
+
+    def initialize(reverse=false)
+      @reverse = reverse
+    end
+
+    def compare(a, b)
+      klass = compare_class(a, b)
+      val = case klass
+        when 0
+          case a
+            when String then compare_string(a, b)
+            when Array  then compare_array(a, b)
+            when Hash   then compare_array(a.to_a, b.to_a)
+            else a <=> b
+          end
+        else
+          klass
+        end
+      @reverse ? val * -1 : val
+    end
+
+    private
+
+    def compare_class(a, b)
+      aix = CLASSES.find_index {|c| a.is_a?(c) }
+      bix = CLASSES.find_index {|c| b.is_a?(c) }
+      aix == bix ? 0 : aix < bix ? -1 : 1
+    end
+
+    # FIXME Implement UCA sorting with ICU
+    def compare_string(a, b)
+      a <=> b
+    end
+
+    def compare_array(a, b)
+      if a.size == b.size
+        compare_same_size_array(a, b)
+      elsif a.size < b.size
+        val = compare_same_size_array(a, b[0, a.size])
+        val == 0 ? -1 : val
+      else
+        val = compare_same_size_array(a[0, b.size], b)
+        val == 0 ? 1 : val
+      end
+    end
+
+    def compare_same_size_array(a, b)
+      a.each_with_index do |el, ix|
+        val = compare(el, b[ix])
+        return val unless val == 0
+      end
+      0
+    end
+  end
+end

data/lib/couchproxy/deferrable_body.rb

@@ -0,0 +1,15 @@
+# encoding: UTF-8
+
+class DeferrableBody
+  include EventMachine::Deferrable
+
+  def call(body)
+    body.each do |chunk|
+      @body_callback.call(chunk)
+    end
+  end
+
+  def each(&blk)
+    @body_callback = blk
+  end
+end

data/lib/couchproxy/node.rb

@@ -0,0 +1,25 @@
+# encoding: UTF-8
+
+module CouchProxy
+  class Node
+    attr_reader :host, :port, :uri, :partitions
+
+    def initialize(uri, partitions)
+      parsed = URI.parse(uri)
+      @uri, @host, @port = uri, parsed.host, parsed.port
+      @partitions = partitions.map {|num| Partition.new(self, num) }
+    end
+
+    def hosts?(partition)
+      @partitions.any? {|p| p.num == partition }
+    end
+
+    def partition(num)
+      @partitions.find {|p| p.num == num }
+    end
+
+    def any_partition
+      @partitions[rand(@partitions.size)]
+    end
+  end
+end

data/lib/couchproxy/partition.rb

@@ -0,0 +1,15 @@
+# encoding: UTF-8
+
+module CouchProxy
+  class Partition
+    attr_reader :node, :num
+
+    def initialize(node, num)
+      @node, @num = node, num
+    end
+
+    def uri(db)
+      "#{@node.uri.chomp('/')}/#{db}_#{@num}"
+    end
+  end
+end

data/lib/couchproxy/rack/active_tasks.rb

@@ -0,0 +1,9 @@
+# encoding: UTF-8
+
+module CouchProxy
+  module Rack
+    class ActiveTasks < Base
+      # FIXME
+    end
+  end
+end

data/lib/couchproxy/rack/all_databases.rb

@@ -0,0 +1,23 @@
+# encoding: UTF-8
+
+module CouchProxy
+  module Rack
+    class AllDatabases < Base
+      SUFFIX = /_\d+$/
+
+      def get
+        proxy_to_all_nodes do |responses|
+          dbs = responses.map do |res|
+            JSON.parse(res.response).map {|name| name.gsub(SUFFIX, '') }
+          end.flatten.uniq.sort
+          send_response(responses.first.response_header.status,
+            response_headers, [dbs.to_json])
+        end
+      end
+
+      def head
+        # FIXME
+      end
+    end
+  end
+end

data/lib/couchproxy/rack/all_docs.rb

@@ -0,0 +1,9 @@
+# encoding: UTF-8
+
+module CouchProxy
+  module Rack
+    class AllDocs < Base
+      # FIXME
+    end
+  end
+end

data/lib/couchproxy/rack/base.rb

@@ -0,0 +1,197 @@
+module CouchProxy
+  module Rack
+    class Base
+      DESIGN_ID = /^_design\/.+/
+      METHODS = [:get, :put, :post, :delete, :head].freeze
+      INVALID_JSON = '{"error":"bad_request","reason":"invalid UTF-8 JSON"}'.freeze
+
+      attr_reader :request, :cluster
+
+      def initialize(request, cluster)
+        @request, @cluster = request, cluster
+      end
+
+      def method_missing(name)
+        allowed = methods.map {|m| m.to_sym } & METHODS
+        allowed = allowed.map {|m| m.to_s.upcase }.join(',')
+        body = "{\"error\:\"method_not_allowed\",\"reason\":\"Only #{allowed} allowed\"}"
+        send_response(405, response_headers, [body])
+      end
+
+      def proxy_to(node, &finish)
+        head_proxy_to(node, finish) if @request.request_method == 'HEAD'
+
+        body, started = DeferrableBody.new, false
+        uri = "#{node.uri}#{@request.fullpath}"
+        http = EM::HttpRequest.new(uri)
+        res = http.send(@request.request_method.downcase,
+          :head => proxy_headers, :body => @request.content)
+        res.stream do |chunk|
+          unless started
+            started = true
+            head = normalize(res.response_header).tap do |h|
+              h['Server'] = "CouchProxy/#{CouchProxy::VERSION}"
+              if res.response_header.location
+                h['Location'] = rewrite_location(res.response_header.location)
+              end
+            end
+            send_response(res.response_header.status, head, body)
+          end
+          body.call([chunk])
+        end
+        res.callback do
+          body.succeed
+          finish.call if finish
+        end
+        res.errback { send_error_response }
+      end
+
+      def rewrite_location(uri)
+        URI.parse(request.url).tap do |req|
+          req.query, req.user, req.password = nil
+          req.path = URI.parse(uri).path.gsub(
+            /^\/#{request.db_name}_\d+/, "/#{request.db_name}")
+        end.to_s
+      end
+
+      def proxy_to_any_node
+        proxy_to(@cluster.any_node)
+      end
+
+      def proxy_to_any_partition
+        partition = cluster.any_partition
+        request.rewrite_proxy_url!(partition.num)
+        proxy_to(partition.node)
+      end
+
+      def proxy_to_all_nodes(&callback)
+        method = request.request_method.downcase
+        multi = EM::MultiRequest.new
+        cluster.nodes.each do |n|
+          uri = "#{n.uri}#{@request.fullpath}"
+          req = EM::HttpRequest.new(uri).send(method,
+            :head => proxy_headers, :body => @request.content)
+          multi.add(req)
+        end
+        multi.callback(&multi(&callback)) if callback
+      end
+
+      def proxy_to_all_partitions(&callback)
+        method = request.request_method.downcase
+        multi = EM::MultiRequest.new
+        cluster.partitions.each do |p|
+          uri = "#{p.node.uri}#{@request.rewrite_proxy_url(p.num)}"
+          uri << "?#{@request.query_string}" unless @request.query_string.empty?
+          multi.add EM::HttpRequest.new(uri).send(method,
+            :head => proxy_headers, :body => @request.content)
+        end
+        multi.callback(&multi(&callback)) if callback
+      end
+
+      def replicate_to_all_partitions(source, *doc_ids, &callback)
+        multi = EM::MultiRequest.new
+        (@cluster.partitions - [source]).each do |p|
+          task = {
+            :source => source.uri(request.db_name),
+            :target => p.uri(request.db_name),
+            :doc_ids => doc_ids.flatten}
+          multi.add EM::HttpRequest.new("#{p.node.uri}/_replicate").post(
+            :head => proxy_headers, :body => task.to_json)
+        end
+        multi.callback(&multi(&callback)) if callback
+      end
+
+      def uuids(count, &callback)
+        http = EM::HttpRequest.new("#{cluster.any_node.uri}/_uuids?count=#{count}").get
+        http.errback { callback.call(nil) }
+        http.callback do |res|
+          if res.response_header.status == 200
+            uuids = JSON.parse(res.response)['uuids']
+            callback.call(uuids)
+          else
+            callback.call(nil)
+          end
+        end
+      end
+
+      def send_response(*args)
+        @request.env['async.callback'].call(args)
+      end
+
+      def send_error_response
+        send_response(503, response_headers, [])
+      end
+
+      private
+
+      def multi(&callback)
+        proc do |multi|
+          if multi.responses[:failed].empty?
+            err = multi.responses[:succeeded].find do |res|
+              res.response_header.status >= 400
+            end
+            if err
+              send_response(err.response_header.status,
+                response_headers, err.response)
+            else
+              callback.call(multi.responses[:succeeded])
+            end
+          else
+            send_error_response
+          end
+        end
+      end
+
+      def head_proxy_to(node, &finish)
+        uri = "#{node.uri}#{@request.fullpath}"
+        http = EM::HttpRequest.new(uri).head(:head => proxy_headers)
+        http.callback do 
+          status = http.response_header.status
+          headers = normalize(http.response_header)
+          send_response(status, headers, [])
+          finish.call if finish
+        end
+        http.errback { send_error_response }
+        throw :async
+      end
+
+      def normalize(headers)
+        headers.keys.inject({}) do |acc, k|
+          normalized = k.sub('HTTP_', '').split('_').map {|p| p.capitalize }.join('-')
+          acc.tap {|h| h[normalized] = headers[k] }
+        end
+      end
+
+      def proxy_headers
+        keys = @request.env.keys.select {|k| k.start_with?('HTTP_') || k == 'CONTENT_TYPE' }
+        keys -= %w[HTTP_HOST HTTP_VERSION]
+        headers = keys.inject({}) do |acc, k|
+          acc.tap {|h| h[k] = @request.env[k] }
+        end
+        normalize(headers)
+      end
+
+      def response_headers
+        type = @request.json? ? "application/json" : "text/plain;charset=utf-8"
+        {
+          "Server" => "CouchProxy/#{CouchProxy::VERSION}",
+          "Date" => Time.now.httpdate,
+          "Content-Type" => type,
+          "Cache-Control" => "must-revalidate"
+        }
+      end
+
+      def design?(doc_id)
+        doc_id =~ DESIGN_ID
+      end
+
+      def delete_query_param(param)
+        value = @request.GET.delete(param)
+        if value
+          @request.env['QUERY_STRING'] = ::Rack::Utils.build_query(@request.GET)
+        end
+        value
+      end
+    end
+  end
+end

data/lib/couchproxy/rack/bulk_docs.rb

@@ -0,0 +1,68 @@
+# encoding: UTF-8
+
+module CouchProxy
+  module Rack
+    class BulkDocs < Base
+      def post
+        begin
+          docs = JSON.parse(request.content)['docs']
+        rescue
+          send_response(400, response_headers, INVALID_JSON)
+          return
+        end
+
+        missing = docs.select {|doc| !doc['_id'] }
+        if missing.any?
+          uuids(missing.size) do |uuids|
+            if uuids
+              missing.each {|doc| doc['_id'] = uuids.shift }
+              save(docs)
+            else
+              send_error_response
+            end
+          end
+        else
+          save(docs)
+        end
+      end
+
+      private
+
+      def save(docs)
+        designs, normals = docs.partition {|d| design?(d['_id']) }
+
+        partitions = Hash.new {|h, k| h[k] = [] }
+        normals.each do |doc|
+          partitions[cluster.partition(doc['_id'])] << doc
+        end
+        design_partition = cluster.any_partition
+        partitions[design_partition] += designs
+
+        req = EM::MultiRequest.new
+        partitions.each do |p, d|
+          url = "#{p.uri(request.db_name)}/_bulk_docs"
+          req.add EM::HttpRequest.new(url).post(:head => proxy_headers,
+            :body => {:docs => d}.to_json)
+        end
+
+        callback = multi do |responses|
+          total = responses.map {|res| JSON.parse(res.response) }.flatten
+          total = docs.map do |doc|
+            total.find {|d| d['id'] == doc['_id'] }
+          end
+          sender = proc do
+            send_response(responses.first.response_header.status,
+              response_headers, [total.to_json])
+          end
+          if designs.any?
+            replicate_to_all_partitions(design_partition,
+              designs.map {|d| d['_id'] }, &sender)
+          else
+            sender.call
+          end
+        end
+        req.callback(&callback)
+      end
+    end
+  end
+end

data/lib/couchproxy/rack/changes.rb

@@ -0,0 +1,9 @@
+# encoding: UTF-8
+
+module CouchProxy
+  module Rack
+    class Changes < Base
+      # FIXME
+    end
+  end
+end