couchproxy 0.1.0 → 0.2.0

data/README

@@ -21,10 +21,11 @@ memory.
 
 == Dependencies
 
-* em-http-request >= 0.2.11
+* em-http-request >= 0.2.15
 * json >= 1.4.6
 * json-stream >= 0.1.0
 * thin >= 1.2.7
+* rbtree >= 0.3.0
 * ruby >= 1.9.1
 
 == Contact

data/Rakefile

@@ -19,7 +19,7 @@ returning the results to the client."
   s.homepage = "http://github.com/dgraham/couchproxy"
   s.authors = ["David Graham"]
   s.files = FileList['[A-Z]*', '{bin,lib,conf}/**/*']
-  s.test_files = FileList["test/**/*test.rb"]
+  s.test_files = FileList["test/**/*"]
   s.executables = %w[couchproxy]
   s.require_path = "lib"
   s.has_rdoc = true
@@ -27,6 +27,7 @@ returning the results to the client."
   s.add_dependency('json', '~> 1.4')
   s.add_dependency('json-stream', '~> 0.1')
   s.add_dependency('thin', '~> 1.2')
+  s.add_dependency('rbtree', '~> 0.3')
   s.required_ruby_version = '>= 1.9.1'
 end
  
@@ -35,6 +36,7 @@ Rake::GemPackageTask.new(spec) do |pkg|
 end 
 
 Rake::TestTask.new(:test) do |test|
+  test.libs << 'test'
   test.pattern = 'test/**/*_test.rb'
   test.warning = true
 end

data/lib/couchproxy.rb

@@ -3,10 +3,16 @@
 $:.unshift File.dirname(__FILE__) unless
   $:.include?(File.dirname(__FILE__))
 
+module CouchProxy
+  VERSION = '0.2.0'
+end
+
 %w[
+  digest
   em-http
   json
   json/stream
+  rbtree
   thin
   time
   uri
@@ -18,9 +24,14 @@ $:.unshift File.dirname(__FILE__) unless
   couchproxy/node
   couchproxy/partition
   couchproxy/deferrable_body
-  couchproxy/reducer
   couchproxy/request
   couchproxy/router
+  couchproxy/row_filter
+
+  couchproxy/reducer
+  couchproxy/reduce/base_reducer
+  couchproxy/reduce/map_reducer
+  couchproxy/reduce/reduce_reducer
 
   couchproxy/rack/base
   couchproxy/rack/all_databases
@@ -42,7 +53,3 @@ $:.unshift File.dirname(__FILE__) unless
   couchproxy/rack/uuids
   couchproxy/rack/view_cleanup
 ].each {|f| require f }
-
-module CouchProxy
-  VERSION = '0.1.0'
-end

data/lib/couchproxy.ru

@@ -1,3 +1,5 @@
+# encoding: UTF-8
+
 $:.unshift File.dirname(__FILE__) unless
   $:.include?(File.dirname(__FILE__))
 

data/lib/couchproxy/collator.rb

@@ -1,8 +1,10 @@
+# encoding: UTF-8
+
 module CouchProxy
+
   # Implements the JSON sorting rules defined at
   # http://wiki.apache.org/couchdb/View_collation.
   class Collator
-    CLASSES = [NilClass, FalseClass, TrueClass, Numeric, String, Array, Hash]
 
     def initialize(reverse=false)
       @reverse = reverse
@@ -27,11 +29,24 @@ module CouchProxy
     private
 
     def compare_class(a, b)
-      aix = CLASSES.find_index {|c| a.is_a?(c) }
-      bix = CLASSES.find_index {|c| b.is_a?(c) }
+      # optimize common case
+      return 0 if a.class == b.class
+      aix, bix = class_index(a), class_index(b)
       aix == bix ? 0 : aix < bix ? -1 : 1
     end
 
+    def class_index(value)
+      case value
+        when NilClass   then 0
+        when FalseClass then 1
+        when TrueClass  then 2
+        when Numeric    then 3
+        when String     then 4
+        when Array      then 5
+        when Hash       then 6
+      end
+    end
+
     # FIXME Implement UCA sorting with ICU
     def compare_string(a, b)
       a <=> b

data/lib/couchproxy/rack/base.rb

@@ -1,9 +1,14 @@
+# encoding: UTF-8
+
 module CouchProxy
   module Rack
     class Base
+      APPLICATION_JSON = "application/json".freeze
+      TEXT_PLAIN = "text/plain;charset=utf-8".freeze
       DESIGN_ID = /^_design\/.+/
       METHODS = [:get, :put, :post, :delete, :head].freeze
       INVALID_JSON = '{"error":"bad_request","reason":"invalid UTF-8 JSON"}'.freeze
+      SERVER_VERSION = "CouchProxy/#{CouchProxy::VERSION}".freeze
 
       attr_reader :request, :cluster
 
@@ -12,14 +17,14 @@ module CouchProxy
       end
 
       def method_missing(name)
-        allowed = methods.map {|m| m.to_sym } & METHODS
-        allowed = allowed.map {|m| m.to_s.upcase }.join(',')
+        allowed = (methods & METHODS).map {|m| m.to_s.upcase }.sort.join(',')
         body = "{\"error\:\"method_not_allowed\",\"reason\":\"Only #{allowed} allowed\"}"
-        send_response(405, response_headers, [body])
+        headers = response_headers.tap {|h| h['Allow'] = allowed }
+        send_response(405, headers, [body])
       end
 
       def proxy_to(node, &finish)
-        head_proxy_to(node, finish) if @request.request_method == 'HEAD'
+        head_proxy_to(node, &finish) if @request.head?
 
         body, started = DeferrableBody.new, false
         uri = "#{node.uri}#{@request.fullpath}"
@@ -30,7 +35,7 @@ module CouchProxy
           unless started
             started = true
             head = normalize(res.response_header).tap do |h|
-              h['Server'] = "CouchProxy/#{CouchProxy::VERSION}"
+              h['Server'] = SERVER_VERSION
               if res.response_header.location
                 h['Location'] = rewrite_location(res.response_header.location)
               end
@@ -59,7 +64,7 @@ module CouchProxy
       end
 
       def proxy_to_any_partition
-        partition = cluster.any_partition
+        partition = @cluster.any_partition
         request.rewrite_proxy_url!(partition.num)
         proxy_to(partition.node)
       end
@@ -67,7 +72,7 @@ module CouchProxy
       def proxy_to_all_nodes(&callback)
         method = request.request_method.downcase
         multi = EM::MultiRequest.new
-        cluster.nodes.each do |n|
+        @cluster.nodes.each do |n|
           uri = "#{n.uri}#{@request.fullpath}"
           req = EM::HttpRequest.new(uri).send(method,
             :head => proxy_headers, :body => @request.content)
@@ -79,9 +84,8 @@ module CouchProxy
       def proxy_to_all_partitions(&callback)
         method = request.request_method.downcase
         multi = EM::MultiRequest.new
-        cluster.partitions.each do |p|
-          uri = "#{p.node.uri}#{@request.rewrite_proxy_url(p.num)}"
-          uri << "?#{@request.query_string}" unless @request.query_string.empty?
+        @cluster.partitions.each do |p|
+          uri = "#{p.node.uri}#{@request.rewrite_proxy_url(p.num)}#{query_string}"
           multi.add EM::HttpRequest.new(uri).send(method,
             :head => proxy_headers, :body => @request.content)
         end
@@ -102,7 +106,7 @@ module CouchProxy
       end
 
       def uuids(count, &callback)
-        http = EM::HttpRequest.new("#{cluster.any_node.uri}/_uuids?count=#{count}").get
+        http = EM::HttpRequest.new("#{@cluster.any_node.uri}/_uuids?count=#{count}").get
         http.errback { callback.call(nil) }
         http.callback do |res|
           if res.response_header.status == 200
@@ -147,7 +151,9 @@ module CouchProxy
         http = EM::HttpRequest.new(uri).head(:head => proxy_headers)
         http.callback do 
           status = http.response_header.status
-          headers = normalize(http.response_header)
+          headers = normalize(http.response_header).tap do |h|
+            h['Server'] = SERVER_VERSION
+          end
           send_response(status, headers, [])
           finish.call if finish
         end
@@ -172,11 +178,10 @@ module CouchProxy
       end
 
       def response_headers
-        type = @request.json? ? "application/json" : "text/plain;charset=utf-8"
         {
-          "Server" => "CouchProxy/#{CouchProxy::VERSION}",
+          "Server" => SERVER_VERSION,
           "Date" => Time.now.httpdate,
-          "Content-Type" => type,
+          "Content-Type" => @request.json? ? APPLICATION_JSON : TEXT_PLAIN,
           "Cache-Control" => "must-revalidate"
         }
       end
@@ -186,11 +191,18 @@ module CouchProxy
       end
 
       def delete_query_param(param)
-        value = @request.GET.delete(param)
-        if value
+        @request.GET.delete(param).tap do |value|
           @request.env['QUERY_STRING'] = ::Rack::Utils.build_query(@request.GET)
         end
-        value
+      end
+
+      def update_query_param(param, value)
+        @request[param] = value        
+        @request.env['QUERY_STRING'] = ::Rack::Utils.build_query(@request.GET)
+      end
+
+      def query_string
+        @request.query_string.empty? ? '' : "?#{@request.query_string}"
       end
     end
   end

data/lib/couchproxy/rack/design_doc.rb

@@ -6,9 +6,6 @@ module CouchProxy
       QUERY     = /_view\/.+$/
       INFO      = /\/_info$/
       VIEW_NAME = /_view\/(.*)$/
-      COUNT     = '_count'.freeze
-      SUM       = '_sum'.freeze
-      STATS     = '_stats'.freeze
       REDUCE_ERROR = '{"error":"query_parse_error","reason":"Invalid URL parameter `reduce` for map view."}'.freeze
 
       def get
@@ -19,6 +16,20 @@ module CouchProxy
         end
       end
 
+      def head
+        case request.path_info
+          when QUERY
+            proxy_to_all_partitions do |responses|
+              etags = responses.map {|r| r.response_header.etag }
+              head = response_headers.tap do |h|
+                h['ETag'] = etag(etags)
+              end
+              send_response(responses.first.response_header.status, head, [])
+            end
+          else proxy_to_any_partition
+        end
+      end
+
       def post
         # FIXME same as get, but body can have keys in it
       end
@@ -34,7 +45,7 @@ module CouchProxy
           sender = proc do
             send_response(res.response_header.status, head, [res.response])
           end
-          if (200...300).include?(res.response_header.status)
+          if success?(res)
             head.tap do |h|
               h['ETag'] = res.response_header.etag
               h['Location'] = rewrite_location(res.response_header.location)
@@ -57,10 +68,6 @@ module CouchProxy
         end
       end
 
-      def head
-        # FIXME
-      end
-
       private
 
       def query_params
@@ -70,115 +77,118 @@ module CouchProxy
           params[:descending] = (request['descending'] == 'true')
           params[:limit] = request['limit'] || ''
           params[:limit] = params[:limit].empty? ? nil : params[:limit].to_i
-          params[:skip] = (params[:limit] == 0) ? 0 : delete_query_param('skip').to_i
-          delete_query_param('limit') if params[:skip] > (params[:limit] || 0)
+          params[:skip] = delete_query_param('skip').to_i
+          params[:skip] = 0 if params[:limit] == 0
+          update_query_param('limit', params[:limit] + params[:skip]) if params[:limit]
           params[:collator] = CouchProxy::Collator.new(params[:descending])
         end
       end
 
+      def send_chunk(body, chunk)
+        body.call(["%s\r\n%s\r\n" % [chunk.bytesize.to_s(16), chunk]])
+      end
+
       def query
         params = query_params
-        proxy_to_all_partitions do |responses|
-          view_doc do |doc|
-            if doc
-              fn = doc['views'][view_name]['reduce']
-              if request['reduce'] && fn.nil?
-                send_response(400, response_headers, [REDUCE_ERROR])
-              elsif params[:reduce] && fn
-                reduce(params, responses, fn)
-              else
-                map(params, responses)
-              end
+        view_doc do |doc|
+          if doc
+            fn = doc['views'][view_name]['reduce']
+            if request['reduce'] && fn.nil?
+              send_response(400, response_headers, [REDUCE_ERROR])
+            elsif params[:reduce] && fn
+              reduce(params, fn)
             else
-              send_error_response
+              map(params)
             end
+          else
+            send_error_response
           end
         end
       end
 
-      def map(params, responses)
-        total = {:total_rows => 0, :offset => 0, :rows =>[]}
-        responses.each do |res|
-          result = JSON.parse(res.response)
-          %w[total_rows rows].each {|k| total[k.to_sym] += result[k] }
-        end
-        total[:rows].sort! do |a, b|
-          key = params[:collator].compare(a['key'], b['key'])
-          (key == 0) ? params[:collator].compare(a['id'], b['id']) : key
-        end
-        total[:rows].slice!(0, params[:skip])
-        total[:rows].slice!(params[:limit], total[:rows].size) if params[:limit]
-        total[:offset] = [params[:skip], total[:total_rows]].min
-        send_response(responses.first.response_header.status,
-          response_headers, [total.to_json])
-      end
-
-      def reduce(params, responses, fn)
-        total = {:rows =>[]}
-        responses.each do |res|
-          result = JSON.parse(res.response)
-          total[:rows] += result['rows']
-        end
-        groups = total[:rows].group_by {|row| row['key'] }
-        case fn
-        when SUM, COUNT
-          sum(params, groups)
-        when STATS
-          stats(params, groups)
-        else
-          view_server(params, fn, groups)
-        end
-      end
-
-      def view_server(params, fn, groups)
-        reduced = {:rows => []}
-        groups.each do |key, rows|
-          values = rows.map {|row| row['value'] }
-          cluster.reducer.rereduce(fn, values) do |result|
-            success, value = result.flatten
-            if success
-              reduced[:rows] << {:key => key, :value => value}
-              if reduced[:rows].size == groups.size
-                reduced[:rows].sort! do |a, b|
-                  params[:collator].compare(a[:key], b[:key])
-                end
-                send_response(200, response_headers, [reduced.to_json])
-              end
-            else
-              send_error_response
-            end
-          end
+      def map(params)
+        reducer = proc do |sources|
+          args = params.merge({:sources => sources})
+          CouchProxy::Reduce::MapReducer.new(args)
+        end
+        spray(reducer) do |total_rows|
+          offset = [params[:skip], total_rows].min
+          "\n],\"total_rows\":%s,\"offset\":%s}" % [total_rows, offset]
         end
       end
 
-      def sum(params, groups)
-        reduced = {:rows => []}
-        groups.each do |key, rows|
-          value = rows.map {|row| row['value'] }.inject(:+)
-          reduced[:rows] << {:key => key, :value => value}
+      def reduce(params, fn)
+        reducer = proc do |sources|
+          args = params.merge({:sources => sources, :fn => fn,
+            :reducers => cluster.method(:reducer)})
+          CouchProxy::Reduce::ReduceReducer.new(args)
         end
-        reduced[:rows].sort! do |a, b|
-          params[:collator].compare(a[:key], b[:key])
-        end
-        send_response(200, response_headers, [reduced.to_json])
+        spray(reducer) {|total_rows| "\n]}" }
       end
 
-      def stats(groups)
-        reduced = {:rows => []}
-        groups.each do |key, rows|
-          values = rows.map {|row| row['value'] }
-          min, max = values.map {|v| [v['min'], v['max']] }.flatten.minmax
-          sum, count, sumsqr = %w[sum count sumsqr].map do |k|
-            values.map {|v| v[k] }.inject(:+)
+      def spray(reducer, &finish)
+        body, etags = DeferrableBody.new, []
+
+        requests = cluster.partitions.map do |p|
+          uri = "#{p.node.uri}#{request.rewrite_proxy_url(p.num)}#{query_string}"
+          EM::HttpRequest.new(uri).send(request.request_method.downcase,
+            :head => proxy_headers, :body => request.content, :timeout => 300)
+        end
+
+        started = false
+        start = proc do
+          started = true
+          headers = response_headers.tap do |h|
+            h['Transfer-Encoding'] = 'chunked'
+            h['ETag'] = etag(etags)
+          end
+          send_response(200, headers, body)
+          send_chunk(body, "{\"rows\":[\n")
+        end
+
+        closed = false
+        close = proc do
+          unless closed
+            closed = true
+            requests.each {|req| req.close_connection }
+            send_error_response
           end
-          value = {:sum => sum, :count => count, :min => min, :max => max,
-            :sumsqr => sumsqr}
-          reduced[:rows] << {:key => key, :value => value}
         end
-        reduced[:rows].sort! do |a, b|
-          params[:collator].compare(a[:key], b[:key])
+
+        total_rows = 0
+        reducer = reducer.call(requests)
+        reducer.error(&close)
+        reducer.results do |results|
+          start.call unless started
+          json = results.map {|row| row.to_json }.join(",\n")
+          json << ",\n" unless reducer.complete?
+          send_chunk(body, json)
+        end
+        reducer.complete do
+          start.call unless started
+          requests.each {|req| req.close_connection }
+          chunk = finish.call(total_rows)
+          [chunk, ''].each {|c| send_chunk(body, c) }
+          body.succeed
+        end
+
+        multi = EM::MultiRequest.new
+        requests.each do |req|
+          parser = JSON::Stream::Parser.new
+          CouchProxy::RowFilter.new(parser) do
+            total_rows {|total| total_rows += total }
+            rows do |rows, complete|
+              reducer.reduce(rows, req, complete)
+            end
+          end
+          req.stream {|chunk| parser << chunk unless closed }
+          req.errback(&close)
+          req.headers do |h|
+            etags << h['ETAG']
+            close.call unless success?(req)
+          end
+          multi.add(req)
         end
-        send_response(200, response_headers, [reduced.to_json])
       end
 
       def info
@@ -202,6 +212,15 @@ module CouchProxy
         end
       end
 
+      def success?(response)
+        (200...300).include?(response.response_header.status)
+      end
+
+      def etag(etags)
+        etags = etags.map {|etag| etag || '' }.sort.join
+        '"%s"' % Digest::SHA256.hexdigest(etags)
+      end
+
       def view_doc_id
         request.doc_id.split('/')[0..1].join('/')
       end

data/lib/couchproxy/rack/doc.rb

@@ -8,8 +8,9 @@ module CouchProxy
         request.rewrite_proxy_url!(partition.num)
         proxy_to(partition.node)
       end
-      alias :put :get
+      alias :put    :get
       alias :delete :get
+      alias :head   :get
     end
   end
 end

data/lib/couchproxy/rack/stats.rb

@@ -31,6 +31,10 @@ module CouchProxy
         end
       end
 
+      def head
+        # FIXME
+      end
+
       private
 
       def parse(body)

data/lib/couchproxy/rack/uuids.rb

@@ -3,7 +3,8 @@
 module CouchProxy
   module Rack
     class Uuids < Base
-      alias :get :proxy_to_any_node
+      alias :get  :proxy_to_any_node
+      alias :head :proxy_to_any_node
     end
   end
 end

data/lib/couchproxy/reduce/base_reducer.rb

@@ -0,0 +1,121 @@
+# encoding: UTF-8
+
+module CouchProxy
+  module Reduce
+
+    # Sorts and merges results from many different source streams as the data
+    # arrives from CouchDB over the network. This uses constant memory space to
+    # do the merge so we can handle huge datasets streaming back from the
+    # databases. Subclasses must provide a @sorter member variable, used to
+    # sort streaming rows before they're processed.
+    class BaseReducer
+      KEY = 'key'.freeze
+      ID  = 'id'.freeze
+
+      # Args should contain the following keys:
+      #  sources: List of stream sources used to identify from where
+      #           streaming rows are arriving.
+      #    limit: Maximum number of rows to return. If not specified, all
+      #           rows are returned.
+      #     skip: Number of rows at the start of the stream to skip before
+      #           returning the rest. If not specified, no rows are skipped.
+      def initialize(args)
+        @sources, @limit, @skip = args.values_at(:sources, :limit, :skip)
+        @sources = Hash[@sources.map {|s| [s, 0] }]
+        @listeners = Hash.new {|h, k| h[k] = [] }
+        @skip ||= 0
+        @returned, @skipped_rows = 0, 0
+        @rows = MultiRBTree.new.tap {|t| t.readjust(@sorter) }
+      end
+
+      %w[results complete error].each do |name|
+        define_method(name) do |&block|
+          @listeners[name] << block
+        end
+
+        define_method("notify_#{name}") do |*args|
+          @listeners[name].each do |block|
+            block.call(*args)
+          end
+        end
+        private "notify_#{name}"
+      end
+
+      # Gives the reducer more rows to process with their source connection.
+      # Complete must be a boolean, signaling whether this stream of rows has
+      # finished.
+      def reduce(rows, source, complete)
+        return if complete?
+        rows.each do |row|
+          row[:proxy_source] = source
+          key = [row[KEY], row[ID]]
+          @rows[key] = row
+        end
+        @sources[source] += rows.size
+        @sources.delete(source) if complete
+        source.pause unless complete
+        process do |results|
+          if results
+            results = limit(skip(results))
+            notify_results(results) if results.any?
+            notify_complete if complete?
+            resume_streams unless complete?
+          else
+            notify_error
+          end
+        end if process?
+      end
+
+      # Returns true if all streams of rows have arrived and the reduce
+      # processing is complete.
+      def complete?
+        @sources.empty?
+      end
+
+      private
+
+      def resume_streams
+        paused = @sources.select {|k, v| k.paused? }.keys
+        empty  = @sources.select {|k, v| k.paused? && v == 0 }.keys
+        (empty.any? ? empty : paused).each {|source| source.resume }
+      end
+
+      def skip(sorted)
+        if @skip > @skipped_rows
+          @skipped_rows += sorted.slice!(0, @skip - @skipped_rows).size
+        end
+        sorted
+      end
+
+      def limit(sorted)
+        return sorted unless @limit
+        if @returned + sorted.size > @limit
+          sorted = sorted[0, @limit - @returned]
+        end
+        @returned += sorted.size
+        if @returned == @limit
+          [@sources, @rows].each {|arr| arr.clear }
+        end
+        sorted
+      end
+
+      def process(&callback)
+        sorted = [].tap do |rows|
+          rows << shift while @rows.any? && process?
+        end
+        callback.call(sorted)
+      end
+
+      def shift
+        @rows.shift.tap do |key, row|
+          source = row.delete(:proxy_source)
+          @sources[source] -= 1 if @sources.key?(source)
+        end[1]
+      end
+
+      def process?
+        !@sources.values.include?(0)
+      end
+    end
+  end
+end

data/lib/couchproxy/reduce/map_reducer.rb

@@ -0,0 +1,28 @@
+# encoding: UTF-8
+
+module CouchProxy
+  module Reduce
+
+    # Sorts and merges map query results from many different source streams.
+    class MapReducer < BaseReducer
+
+      # Args should contain the following keys:
+      #  sources: List of stream sources used to identify from where
+      #           streaming rows are arriving.
+      #    limit: Maximum number of rows to return. If not specified, all
+      #           rows are returned.
+      #     skip: Number of rows at the start of the stream to skip before
+      #           returning the rest. If not specified, no rows are skipped.
+      # collator: A CouchProxy::Collator instance used to sort rows.
+      def initialize(args)
+        collator = args[:collator]
+        # key = 0, id = 1
+        @sorter = proc do |a, b|
+          key = collator.compare(a[0], b[0])
+          (key == 0) ? collator.compare(a[1], b[1]) : key
+        end
+        super(args)
+      end
+    end
+  end
+end

data/lib/couchproxy/reduce/reduce_reducer.rb

@@ -0,0 +1,95 @@
+# encoding: UTF-8
+
+module CouchProxy
+  module Reduce
+
+    # Sorts and merges reduce query results from many different source streams.
+    class ReduceReducer < BaseReducer
+      KEY       = 'key'.freeze
+      VALUE     = 'value'.freeze
+      COUNT     = '_count'.freeze
+      SUM       = '_sum'.freeze
+      STATS     = '_stats'.freeze
+      BUILT_INS = [COUNT, SUM, STATS]
+      NONE      = Struct.new(:none) 
+
+      # Args should contain the following keys:
+      #  sources: List of stream sources used to identify from where
+      #           streaming rows are arriving.
+      #    limit: Maximum number of rows to return. If not specified, all
+      #           rows are returned.
+      #     skip: Number of rows at the start of the stream to skip before
+      #           returning the rest. If not specified, no rows are skipped.
+      # collator: A CouchProxy::Collator instance used to sort rows.
+      #       fn: The JavaScript reduce function to apply to the rows.
+      # reducers: A block that, when called, returns a CouchProxy::Reducer
+      #           instance.
+      def initialize(args)
+        @fn, @reducers, collator = args.values_at(:fn, :reducers, :collator)
+        # key = 0, id = 1
+        @sorter = proc {|a, b| collator.compare(a[0], b[0]) }
+        @processes = []
+        super(args)
+      end
+
+      def complete?
+        super && @processes.empty?
+      end
+
+      private
+
+      def process(&callback)
+        sorted = [].tap do |rows|
+          while @rows.any? && process?
+            case @fn
+              when SUM, COUNT then rows << sum(next_group)
+              when STATS      then rows << stats(next_group)
+              else view_server(next_group, callback)
+            end
+          end
+        end
+        callback.call(sorted) if built_in?
+      end
+
+      def next_group
+        key, row = @rows.first
+        @rows.bound(key, key).map { shift }
+      end
+
+      def view_server(rows, callback)
+        tracker = (@processes << {:value => NONE}).last
+        values = rows.map {|row| row[VALUE] }
+        @reducers.call.rereduce(@fn, values) do |result|
+          success, value = result.flatten
+          if success
+            tracker[:value] = {:key => rows.first[KEY], :value => value}
+            ix = @processes.index {|t| t[:value] == NONE } || @processes.size
+            finished = @processes.slice!(0, ix).map {|t| t[:value] }
+            callback.call(finished)
+          else
+            callback.call(nil)
+          end
+        end
+      end
+
+      def sum(rows)
+        value = rows.map {|row| row[VALUE] }.inject(:+)
+        {:key => rows.first[KEY], :value => value}
+      end
+
+      def stats(rows)
+        values = rows.map {|row| row[VALUE] }
+        min, max = values.map {|v| [v['min'], v['max']] }.flatten.minmax
+        sum, count, sumsqr = %w[sum count sumsqr].map do |k|
+          values.map {|v| v[k] }.inject(:+)
+        end
+        value = {:sum => sum, :count => count, :min => min, :max => max, :sumsqr => sumsqr}
+        {:key => rows.first[KEY], :value => value}
+      end
+
+      def built_in?
+        BUILT_INS.include?(@fn)
+      end
+    end
+  end
+end

data/lib/couchproxy/reducer.rb

@@ -23,7 +23,7 @@ module CouchProxy
   end
 
   class ReduceProcess < EventMachine::Connection
-    def initialize(unbind)
+    def initialize(unbind=nil)
       @unbind, @connected, @callbacks, @deferred = unbind, false, [], []
     end
 
@@ -51,7 +51,7 @@ module CouchProxy
 
     def unbind
       @connected = false
-      @unbind.call
+      @unbind.call if @unbind
     end
   end
 end

data/lib/couchproxy/request.rb

@@ -1,4 +1,7 @@
+# encoding: UTF-8
+
 module Rack
+
   # Add a few helper methods to Rack's Request class.
   class Request
     def json?

data/lib/couchproxy/row_filter.rb

@@ -0,0 +1,69 @@
+# encoding: UTF-8
+
+module CouchProxy
+
+  # A JSON::Stream::Parser listener that listens for the 'rows' key
+  # in a CouchDB map/reduce result stream.  As row objects are parsed
+  # they are sent to callbacks for processing.  Typically the callback
+  # will perform some kind of reduce on the rows before sending them
+  # to the client.
+  #
+  # Example usage:
+  # parser = JSON::Stream::Parser.new
+  # filter = CouchProxy::RowFilter.new(parser) do
+  #   total_rows {|total| puts total }
+  #   rows do |rows, complete|
+  #     # process rows, complete tells us if this is the last row
+  #   end
+  # end
+  class RowFilter < JSON::Stream::Builder
+    TOTAL_ROWS = 'total_rows'.freeze
+    MAX_ROWS   = 100
+
+    def initialize(parser, &block)
+      @listeners = Hash.new {|h, k| h[k] = [] }
+      @total_rows_key = false
+      super(parser)
+      instance_eval(&block) if block_given?
+    end
+
+    %w[total_rows rows].each do |name|
+      define_method(name) do |&block|
+        @listeners[name] << block
+      end
+
+      define_method("notify_#{name}") do |*args|
+        @listeners[name].each do |block|
+          block.call(*args)
+        end
+      end
+      private "notify_#{name}"
+    end
+
+    def key(key)
+      super
+      @total_rows_key = (@stack.size == 1 && key == TOTAL_ROWS)
+    end
+
+    def value(value)
+      super
+      if @total_rows_key
+        @total_rows_key = false
+        notify_total_rows(value)
+      end
+    end
+
+    def end_document
+      notify_rows(@stack.pop.obj['rows'], true)
+    end
+
+    def end_object
+      super
+      # row object complete
+      if @stack.size == 2 && @stack[-1].obj.size >= MAX_ROWS
+        notify_rows(@stack.pop.obj, false)
+        @stack.push(JSON::Stream::ArrayNode.new)
+      end
+    end
+  end
+end

data/test/map_reducer_test.rb

@@ -0,0 +1,93 @@
+# encoding: UTF-8
+
+require 'couchproxy'
+require 'mock_source'
+require 'test/unit'
+
+class MapReducerTest < Test::Unit::TestCase
+  def setup
+    @collator = CouchProxy::Collator.new
+  end
+
+  def test_no_rows_from_one_source
+    source = MockSource.new('1')
+    reducer = CouchProxy::Reduce::MapReducer.new(
+      :sources => [source], :collator => @collator)
+    reducer.results do |results|
+      flunk("No results expected")
+    end
+    complete = []
+    reducer.complete { complete << 1 }
+    reducer.reduce([], source, true)
+    assert_equal(1, complete.inject(:+))
+    assert(reducer.complete?)
+  end
+
+  def test_no_rows_from_two_sources
+    sources = %w[1 2].map {|uri| MockSource.new(uri) }
+    reducer = CouchProxy::Reduce::MapReducer.new(
+      :sources => sources, :collator => @collator)
+    reducer.results do |results|
+      flunk("No results expected")
+    end
+    complete = []
+    reducer.complete { complete << 1 }
+    reducer.reduce([], sources[0], true)
+    assert(!reducer.complete?)
+    reducer.reduce([], sources[1], true)
+    assert_equal(1, complete.inject(:+))
+    assert(reducer.complete?)
+  end
+
+  def test_row_sorting
+    sources = %w[1 2].map {|uri| MockSource.new(uri) }
+    reducer = CouchProxy::Reduce::MapReducer.new(
+      :sources => sources, :collator => @collator)
+    rows, complete = [], []
+    reducer.results {|results| rows += results }
+    reducer.complete { complete << 1 }
+    reducer.reduce([{'id' => '2', 'key' => 'a', 'value' => 'v2'}], sources[1], false)
+    reducer.reduce([{'id' => '1', 'key' => 'a', 'value' => 'v1'}], sources[0], false)
+    reducer.reduce([{'id' => '3', 'key' => 'c', 'value' => 'v4'}], sources[0], true)
+    reducer.reduce([{'id' => '4', 'key' => 'b', 'value' => 'v3'}], sources[1], true)
+    assert_equal(1, complete.inject(:+))
+    assert(reducer.complete?)
+    assert_equal(4, rows.size)
+    assert_equal(%w[v1 v2 v3 v4], rows.map {|r| r['value'] })
+  end
+
+  def test_limit
+    sources = %w[1 2].map {|uri| MockSource.new(uri) }
+    reducer = CouchProxy::Reduce::MapReducer.new(
+      :sources => sources, :collator => @collator, :limit => 2)
+    rows, complete = [], []
+    reducer.results {|results| rows += results }
+    reducer.complete { complete << 1 }
+    reducer.reduce([{'id' => '1', 'key' => 'a', 'value' => 'v1'}], sources[0], false)
+    reducer.reduce([{'id' => '3', 'key' => 'c', 'value' => 'v4'}], sources[0], false)
+    assert(!reducer.complete?)
+    reducer.reduce([{'id' => '2', 'key' => 'a', 'value' => 'v2'}], sources[1], false)
+    assert(reducer.complete?)
+    reducer.reduce([{'id' => '4', 'key' => 'b', 'value' => 'v3'}], sources[1], false)
+    assert_equal(1, complete.inject(:+))
+    assert_equal(2, rows.size)
+    assert_equal(%w[v1 v2], rows.map {|r| r['value'] })
+  end
+
+  def test_skip
+    sources = %w[1 2].map {|uri| MockSource.new(uri) }
+    reducer = CouchProxy::Reduce::MapReducer.new(
+      :sources => sources, :collator => @collator, :skip => 2)
+    rows, complete = [], []
+    reducer.results {|results| rows += results }
+    reducer.complete { complete << 1 }
+    reducer.reduce([{'id' => '1', 'key' => 'a', 'value' => 'v1'}], sources[0], false)
+    reducer.reduce([{'id' => '3', 'key' => 'c', 'value' => 'v4'}], sources[0], true)
+    reducer.reduce([{'id' => '2', 'key' => 'a', 'value' => 'v2'}], sources[1], false)
+    reducer.reduce([{'id' => '4', 'key' => 'b', 'value' => 'v3'}], sources[1], true)
+    assert(reducer.complete?)
+    assert_equal(1, complete.inject(:+))
+    assert_equal(2, rows.size)
+    assert_equal(%w[v3 v4], rows.map {|r| r['value'] })
+  end
+end

data/test/mock_source.rb

@@ -0,0 +1,32 @@
+# encoding: UTF-8
+
+# A source of rows that responds to EventMachine::Connection pause and resume
+# methods. The reducer pauses sources to allow rows from slower connections
+# to be read.
+class MockSource
+  attr_reader :uri
+
+  def initialize(uri)
+    @uri, @paused = uri, false
+  end
+
+  def pause
+    @paused = true
+  end
+
+  def paused?
+    @paused
+  end
+
+  def resume
+    @paused = false
+  end
+
+  def ==(source)
+    @uri == source.uri
+  end
+
+  def <=>(source)
+    @uri <=> source.uri
+  end
+end

data/test/reduce_reducer_test.rb

@@ -0,0 +1,60 @@
+# encoding: UTF-8
+
+require 'couchproxy'
+require 'mock_source'
+require 'test/unit'
+
+class ReduceReducerTest < Test::Unit::TestCase
+  def setup
+    @collator = CouchProxy::Collator.new
+  end
+
+  def test_sum
+    sources = %w[1 2].map {|uri| MockSource.new(uri) }
+    reducer = CouchProxy::Reduce::ReduceReducer.new(
+      :sources => sources, :collator => @collator, :fn => '_sum')
+    rows, complete = [], []
+    reducer.results {|results| rows += results }
+    reducer.complete { complete << 1 }
+    reducer.reduce([{'key' => 'a', 'value' => 2}], sources[1], false)
+    reducer.reduce([{'key' => 'a', 'value' => 4}], sources[0], false)
+    reducer.reduce([{'key' => 'c', 'value' => 6}], sources[0], true)
+    reducer.reduce([{'key' => 'b', 'value' => 8}], sources[1], true)
+    assert_equal(1, complete.inject(:+))
+    assert(reducer.complete?)
+    assert_equal(3, rows.size)
+    results = rows.map {|r| r.values_at(:key, :value) }.flatten
+    assert_equal(['a', 6, 'b', 8, 'c', 6], results)
+  end
+
+  def test_stats
+    sources = %w[1 2].map {|uri| MockSource.new(uri) }
+    reducer = CouchProxy::Reduce::ReduceReducer.new(
+      :sources => sources, :collator => @collator, :fn => '_stats')
+    rows, complete = [], []
+    reducer.results {|results| rows += results }
+    reducer.complete { complete << 1 }
+    values = [
+      {'sum' => 2, 'count' => 3, 'min' => 0, 'max' => 2, 'sumsqr' => 1},
+      {'sum' => 4, 'count' => 6, 'min' => 1, 'max' => 3, 'sumsqr' => 2},
+      {'sum' => 2, 'count' => 3, 'min' => 0, 'max' => 2, 'sumsqr' => 3},
+      {'sum' => 2, 'count' => 3, 'min' => 0, 'max' => 2, 'sumsqr' => 4}
+    ]
+    reducer.reduce([{'key' => 'a', 'value' => values[0]}], sources[1], false)
+    reducer.reduce([{'key' => 'a', 'value' => values[1]}], sources[0], false)
+    reducer.reduce([{'key' => 'c', 'value' => values[2]}], sources[0], true)
+    reducer.reduce([{'key' => 'b', 'value' => values[3]}], sources[1], true)
+    assert_equal(1, complete.inject(:+))
+    assert(reducer.complete?)
+    assert_equal(3, rows.size)
+    results = rows.map {|r| r.values_at(:key, :value) }.flatten
+    combined = {:sum => 6, :count => 9, :min => 0, :max => 3, :sumsqr => 3}
+    assert_equal(['a', combined, 'b', to_sym(values[3]), 'c', to_sym(values[2])], results)
+  end
+
+  private
+
+  def to_sym(hash)
+    Hash[hash.map {|k,v| [k.to_sym, v]}]
+  end
+end

data/test/row_filter_test.rb

@@ -0,0 +1,51 @@
+# encoding: UTF-8
+
+require 'couchproxy'
+require 'test/unit'
+
+class RowFilterTest < Test::Unit::TestCase
+  def setup
+    @parser = JSON::Stream::Parser.new
+    @filter = CouchProxy::RowFilter.new(@parser)
+  end
+
+  def test_total_rows
+    total_rows = -1
+    @filter.total_rows {|total| total_rows = total }
+    @parser << {:total_rows => 2, :offset => 0, :rows => [
+        {:id => "1", :key => {:total_rows => 42}, :value => {:total_rows => 42}},
+        {:id => "2", :key => {:total_rows => 42}, :value => {:total_rows => 42}}
+      ]}.to_json
+    assert_equal(2, total_rows)
+  end
+
+  def test_total_rows_missing
+    total_rows = -1
+    @filter.total_rows {|total| total_rows = total }
+    @parser << {:offset => 0, :rows => [
+        {:id => "1", :key => nil, :value => {:total_rows => 42}}
+      ]}.to_json
+    assert_equal(-1, total_rows)
+  end
+
+  def test_rows_with_small_dataset
+    test_rows(3)
+  end
+
+  def test_rows_with_large_dataset
+    test_rows(5003)
+  end
+
+  private
+
+  def test_rows(count)
+    all_rows = []
+    @filter.rows {|rows| all_rows += rows }
+    rows = Array.new(count) do |i|
+      {:id => i.to_s, :key => {:rows => [42]}, :value => {:rows => [42]}}
+    end
+    @parser << {:total_rows => 2, :offset => 0, :rows => rows}.to_json
+    assert_equal(count, all_rows.size)
+    assert_equal(Array.new(count) {|i| i.to_s }, all_rows.map {|r| r['id'] })
+  end
+end

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
-  - 1
+  - 2
   - 0
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors: 
 - David Graham
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-09-06 00:00:00 -06:00
+date: 2011-01-03 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -73,6 +73,20 @@ dependencies:
         version: "1.2"
   type: :runtime
   version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: rbtree
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 3
+        version: "0.3"
+  type: :runtime
+  version_requirements: *id005
 description: |-
   CouchProxy is a simple proxy server that distributes reads and writes to a
   cluster of Apache CouchDB servers so they appear to be a single huge database.
@@ -119,13 +133,21 @@ files:
 - lib/couchproxy/rack/users.rb
 - lib/couchproxy/rack/uuids.rb
 - lib/couchproxy/rack/view_cleanup.rb
+- lib/couchproxy/reduce/base_reducer.rb
+- lib/couchproxy/reduce/map_reducer.rb
+- lib/couchproxy/reduce/reduce_reducer.rb
 - lib/couchproxy/reducer.rb
 - lib/couchproxy/request.rb
 - lib/couchproxy/router.rb
+- lib/couchproxy/row_filter.rb
 - lib/couchproxy.rb
 - lib/couchproxy.ru
 - conf/couchproxy.yml
 - test/collator_test.rb
+- test/map_reducer_test.rb
+- test/mock_source.rb
+- test/reduce_reducer_test.rb
+- test/row_filter_test.rb
 has_rdoc: true
 homepage: http://github.com/dgraham/couchproxy
 licenses: []
@@ -162,3 +184,7 @@ specification_version: 3
 summary: A proxy server for Apache CouchDB clusters.
 test_files: 
 - test/collator_test.rb
+- test/map_reducer_test.rb
+- test/mock_source.rb
+- test/reduce_reducer_test.rb
+- test/row_filter_test.rb