couchproxy 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. data/LICENSE +19 -0
  2. data/README +36 -0
  3. data/Rakefile +42 -0
  4. data/bin/couchproxy +88 -0
  5. data/conf/couchproxy.yml +21 -0
  6. data/lib/couchproxy/cluster.rb +43 -0
  7. data/lib/couchproxy/collator.rb +60 -0
  8. data/lib/couchproxy/deferrable_body.rb +15 -0
  9. data/lib/couchproxy/node.rb +25 -0
  10. data/lib/couchproxy/partition.rb +15 -0
  11. data/lib/couchproxy/rack/active_tasks.rb +9 -0
  12. data/lib/couchproxy/rack/all_databases.rb +23 -0
  13. data/lib/couchproxy/rack/all_docs.rb +9 -0
  14. data/lib/couchproxy/rack/base.rb +197 -0
  15. data/lib/couchproxy/rack/bulk_docs.rb +68 -0
  16. data/lib/couchproxy/rack/changes.rb +9 -0
  17. data/lib/couchproxy/rack/compact.rb +16 -0
  18. data/lib/couchproxy/rack/config.rb +16 -0
  19. data/lib/couchproxy/rack/database.rb +83 -0
  20. data/lib/couchproxy/rack/design_doc.rb +227 -0
  21. data/lib/couchproxy/rack/doc.rb +15 -0
  22. data/lib/couchproxy/rack/ensure_full_commit.rb +16 -0
  23. data/lib/couchproxy/rack/not_found.rb +13 -0
  24. data/lib/couchproxy/rack/replicate.rb +9 -0
  25. data/lib/couchproxy/rack/revs_limit.rb +18 -0
  26. data/lib/couchproxy/rack/root.rb +10 -0
  27. data/lib/couchproxy/rack/stats.rb +53 -0
  28. data/lib/couchproxy/rack/temp_view.rb +9 -0
  29. data/lib/couchproxy/rack/update.rb +11 -0
  30. data/lib/couchproxy/rack/users.rb +9 -0
  31. data/lib/couchproxy/rack/uuids.rb +9 -0
  32. data/lib/couchproxy/rack/view_cleanup.rb +16 -0
  33. data/lib/couchproxy/reducer.rb +57 -0
  34. data/lib/couchproxy/request.rb +50 -0
  35. data/lib/couchproxy/router.rb +62 -0
  36. data/lib/couchproxy.rb +48 -0
  37. data/lib/couchproxy.ru +22 -0
  38. data/test/collator_test.rb +100 -0
  39. metadata +164 -0
@@ -0,0 +1,16 @@
1
+ # encoding: UTF-8
2
+
3
+ module CouchProxy
4
+ module Rack
5
+ class Compact < Base
6
+ def post
7
+ proxy_to_all_partitions do |responses|
8
+ ok = responses.map {|res| JSON.parse(res.response)['ok'] }
9
+ body = {:ok => !ok.include?(false)}
10
+ send_response(responses.first.response_header.status,
11
+ response_headers, [body.to_json])
12
+ end
13
+ end
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,16 @@
1
+ # encoding: UTF-8
2
+
3
+ module CouchProxy
4
+ module Rack
5
+ class Config < Base
6
+ alias :get :proxy_to_any_node
7
+
8
+ def put
9
+ proxy_to_all_nodes do |responses|
10
+ send_response(responses.first.response_header.status,
11
+ response_headers, [responses.first.response])
12
+ end
13
+ end
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,83 @@
1
+ # encoding: UTF-8
2
+
3
+ module CouchProxy
4
+ module Rack
5
+ class Database < Base
6
+ def get
7
+ proxy_to_all_partitions do |responses|
8
+ doc = {
9
+ :db_name => request.db_name,
10
+ :disk_size => 0,
11
+ :doc_count => 0,
12
+ :doc_del_count => 0,
13
+ :compact_running => false,
14
+ :compact_running_partitions => []}
15
+
16
+ responses.each do |res|
17
+ result = JSON.parse(res.response)
18
+ doc[:compact_running] ||= result['compact_running']
19
+ doc[:compact_running_partitions] << result['db_name'] if result['compact_running']
20
+ %w[disk_size doc_count doc_del_count].each do |k|
21
+ doc[k.to_sym] += result[k]
22
+ end
23
+ doc[:compact_running_partitions].sort!
24
+ end
25
+ send_response(responses.first.response_header.status,
26
+ response_headers, [doc.to_json])
27
+ end
28
+ end
29
+
30
+ def post
31
+ begin
32
+ doc = JSON.parse(request.content)
33
+ rescue
34
+ send_response(400, response_headers, INVALID_JSON)
35
+ return
36
+ end
37
+
38
+ unless doc['_id']
39
+ uuids(1) do |uuids|
40
+ if uuids
41
+ doc['_id'] = uuids.first
42
+ partition = cluster.partition(doc['_id'])
43
+ request.content = doc.to_json
44
+ request.rewrite_proxy_url!(partition.num)
45
+ proxy_to(partition.node)
46
+ else
47
+ send_error_response
48
+ end
49
+ end
50
+ else
51
+ partition = cluster.partition(doc['_id'])
52
+ request.rewrite_proxy_url!(partition.num)
53
+ proxy_to(partition.node) do
54
+ if design?(doc['_id'])
55
+ replicate_to_all_partitions(partition, doc['_id'])
56
+ end
57
+ end
58
+ end
59
+ end
60
+
61
+ def put
62
+ proxy_to_all_partitions do |responses|
63
+ res = responses.first
64
+ head = response_headers.tap do |h|
65
+ h['Location'] = rewrite_location(res.response_header.location)
66
+ end
67
+ send_response(res.response_header.status, head, res.response)
68
+ end
69
+ end
70
+
71
+ def delete
72
+ proxy_to_all_partitions do |responses|
73
+ send_response(responses.first.response_header.status,
74
+ response_headers, responses.first.response)
75
+ end
76
+ end
77
+
78
+ def head
79
+ # FIXME
80
+ end
81
+ end
82
+ end
83
+ end
@@ -0,0 +1,227 @@
1
+ # encoding: UTF-8
2
+
3
+ module CouchProxy
4
+ module Rack
5
+ class DesignDoc < Base
6
+ QUERY = /_view\/.+$/
7
+ INFO = /\/_info$/
8
+ VIEW_NAME = /_view\/(.*)$/
9
+ COUNT = '_count'.freeze
10
+ SUM = '_sum'.freeze
11
+ STATS = '_stats'.freeze
12
+ REDUCE_ERROR = '{"error":"query_parse_error","reason":"Invalid URL parameter `reduce` for map view."}'.freeze
13
+
14
+ def get
15
+ case request.path_info
16
+ when QUERY then query
17
+ when INFO then info
18
+ else proxy_to_any_partition
19
+ end
20
+ end
21
+
22
+ def post
23
+ # FIXME same as get, but body can have keys in it
24
+ end
25
+
26
+ def put
27
+ partition = cluster.any_partition
28
+ request.rewrite_proxy_url!(partition.num)
29
+ uri = "#{partition.node.uri}#{@request.fullpath}"
30
+ http = EM::HttpRequest.new(uri).put(:head => proxy_headers,
31
+ :body => request.content)
32
+ http.callback do |res|
33
+ head = response_headers
34
+ sender = proc do
35
+ send_response(res.response_header.status, head, [res.response])
36
+ end
37
+ if (200...300).include?(res.response_header.status)
38
+ head.tap do |h|
39
+ h['ETag'] = res.response_header.etag
40
+ h['Location'] = rewrite_location(res.response_header.location)
41
+ end
42
+ replicate_to_all_partitions(partition, request.doc_id, &sender)
43
+ else
44
+ sender.call
45
+ end
46
+ end
47
+ http.errback { send_error_response }
48
+ end
49
+
50
+ def delete
51
+ proxy_to_all_partitions do |responses|
52
+ head = response_headers.tap do |h|
53
+ h['ETag'] = responses.first.response_header.etag
54
+ end
55
+ send_response(responses.first.response_header.status,
56
+ head, responses.first.response)
57
+ end
58
+ end
59
+
60
+ def head
61
+ # FIXME
62
+ end
63
+
64
+ private
65
+
66
+ def query_params
67
+ {}.tap do |params|
68
+ params[:reduce] = [nil, 'true'].include?(request['reduce'])
69
+ params[:group] = (request['group'] == 'true')
70
+ params[:descending] = (request['descending'] == 'true')
71
+ params[:limit] = request['limit'] || ''
72
+ params[:limit] = params[:limit].empty? ? nil : params[:limit].to_i
73
+ params[:skip] = (params[:limit] == 0) ? 0 : delete_query_param('skip').to_i
74
+ delete_query_param('limit') if params[:skip] > (params[:limit] || 0)
75
+ params[:collator] = CouchProxy::Collator.new(params[:descending])
76
+ end
77
+ end
78
+
79
+ def query
80
+ params = query_params
81
+ proxy_to_all_partitions do |responses|
82
+ view_doc do |doc|
83
+ if doc
84
+ fn = doc['views'][view_name]['reduce']
85
+ if request['reduce'] && fn.nil?
86
+ send_response(400, response_headers, [REDUCE_ERROR])
87
+ elsif params[:reduce] && fn
88
+ reduce(params, responses, fn)
89
+ else
90
+ map(params, responses)
91
+ end
92
+ else
93
+ send_error_response
94
+ end
95
+ end
96
+ end
97
+ end
98
+
99
+ def map(params, responses)
100
+ total = {:total_rows => 0, :offset => 0, :rows =>[]}
101
+ responses.each do |res|
102
+ result = JSON.parse(res.response)
103
+ %w[total_rows rows].each {|k| total[k.to_sym] += result[k] }
104
+ end
105
+ total[:rows].sort! do |a, b|
106
+ key = params[:collator].compare(a['key'], b['key'])
107
+ (key == 0) ? params[:collator].compare(a['id'], b['id']) : key
108
+ end
109
+ total[:rows].slice!(0, params[:skip])
110
+ total[:rows].slice!(params[:limit], total[:rows].size) if params[:limit]
111
+ total[:offset] = [params[:skip], total[:total_rows]].min
112
+ send_response(responses.first.response_header.status,
113
+ response_headers, [total.to_json])
114
+ end
115
+
116
+ def reduce(params, responses, fn)
117
+ total = {:rows =>[]}
118
+ responses.each do |res|
119
+ result = JSON.parse(res.response)
120
+ total[:rows] += result['rows']
121
+ end
122
+ groups = total[:rows].group_by {|row| row['key'] }
123
+ case fn
124
+ when SUM, COUNT
125
+ sum(params, groups)
126
+ when STATS
127
+ stats(params, groups)
128
+ else
129
+ view_server(params, fn, groups)
130
+ end
131
+ end
132
+
133
+ def view_server(params, fn, groups)
134
+ reduced = {:rows => []}
135
+ groups.each do |key, rows|
136
+ values = rows.map {|row| row['value'] }
137
+ cluster.reducer.rereduce(fn, values) do |result|
138
+ success, value = result.flatten
139
+ if success
140
+ reduced[:rows] << {:key => key, :value => value}
141
+ if reduced[:rows].size == groups.size
142
+ reduced[:rows].sort! do |a, b|
143
+ params[:collator].compare(a[:key], b[:key])
144
+ end
145
+ send_response(200, response_headers, [reduced.to_json])
146
+ end
147
+ else
148
+ send_error_response
149
+ end
150
+ end
151
+ end
152
+ end
153
+
154
+ def sum(params, groups)
155
+ reduced = {:rows => []}
156
+ groups.each do |key, rows|
157
+ value = rows.map {|row| row['value'] }.inject(:+)
158
+ reduced[:rows] << {:key => key, :value => value}
159
+ end
160
+ reduced[:rows].sort! do |a, b|
161
+ params[:collator].compare(a[:key], b[:key])
162
+ end
163
+ send_response(200, response_headers, [reduced.to_json])
164
+ end
165
+
166
+ def stats(groups)
167
+ reduced = {:rows => []}
168
+ groups.each do |key, rows|
169
+ values = rows.map {|row| row['value'] }
170
+ min, max = values.map {|v| [v['min'], v['max']] }.flatten.minmax
171
+ sum, count, sumsqr = %w[sum count sumsqr].map do |k|
172
+ values.map {|v| v[k] }.inject(:+)
173
+ end
174
+ value = {:sum => sum, :count => count, :min => min, :max => max,
175
+ :sumsqr => sumsqr}
176
+ reduced[:rows] << {:key => key, :value => value}
177
+ end
178
+ reduced[:rows].sort! do |a, b|
179
+ params[:collator].compare(a[:key], b[:key])
180
+ end
181
+ send_response(200, response_headers, [reduced.to_json])
182
+ end
183
+
184
+ def info
185
+ proxy_to_all_partitions do |responses|
186
+ status = total = nil
187
+ responses.shift.tap do |res|
188
+ status = res.response_header.status
189
+ total = JSON.parse(res.response)
190
+ end
191
+ responses.each do |res|
192
+ doc = JSON.parse(res.response)
193
+ %w[disk_size waiting_clients].each do |k|
194
+ total['view_index'][k] += doc['view_index'][k]
195
+ end
196
+ %w[compact_running updater_running waiting_commit].each do |k|
197
+ total['view_index'][k] ||= doc['view_index'][k]
198
+ end
199
+ end
200
+ %w[update_seq purge_seq].each {|k| total['view_index'].delete(k) }
201
+ send_response(status, response_headers, [total.to_json])
202
+ end
203
+ end
204
+
205
+ def view_doc_id
206
+ request.doc_id.split('/')[0..1].join('/')
207
+ end
208
+
209
+ def view_name
210
+ request.doc_id.match(VIEW_NAME)[1]
211
+ end
212
+
213
+ def view_doc(&callback)
214
+ db = cluster.any_partition.uri(request.db_name)
215
+ http = EM::HttpRequest.new("#{db}/#{view_doc_id}").get
216
+ http.errback { callback.call(nil) }
217
+ http.callback do |res|
218
+ if res.response_header.status == 200
219
+ callback.call(JSON.parse(res.response))
220
+ else
221
+ callback.call(nil)
222
+ end
223
+ end
224
+ end
225
+ end
226
+ end
227
+ end
@@ -0,0 +1,15 @@
1
+ # encoding: UTF-8
2
+
3
+ module CouchProxy
4
+ module Rack
5
+ class Doc < Base
6
+ def get
7
+ partition = cluster.partition(request.doc_id)
8
+ request.rewrite_proxy_url!(partition.num)
9
+ proxy_to(partition.node)
10
+ end
11
+ alias :put :get
12
+ alias :delete :get
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,16 @@
1
+ # encoding: UTF-8
2
+
3
+ module CouchProxy
4
+ module Rack
5
+ class EnsureFullCommit < Base
6
+ def post
7
+ proxy_to_all_partitions do |responses|
8
+ ok = responses.map {|res| JSON.parse(res.response)['ok'] }
9
+ body = {:ok => !ok.include?(false)}
10
+ send_response(responses.first.response_header.status,
11
+ response_headers, [body.to_json])
12
+ end
13
+ end
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,13 @@
1
+ # encoding: UTF-8
2
+
3
+ module CouchProxy
4
+ module Rack
5
+ class NotFound < Base
6
+ NOT_FOUND = '{"error":"not_found","reason":"missing"}'.freeze
7
+
8
+ def method_missing(name)
9
+ send_response(404, response_headers, [NOT_FOUND])
10
+ end
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,9 @@
1
+ # encoding: UTF-8
2
+
3
+ module CouchProxy
4
+ module Rack
5
+ class Replicate < Base
6
+ # FIXME
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,18 @@
1
+ # encoding: UTF-8
2
+
3
+ module CouchProxy
4
+ module Rack
5
+ class RevsLimit < Base
6
+ alias :get :proxy_to_any_partition
7
+
8
+ def put
9
+ proxy_to_all_partitions do |responses|
10
+ ok = responses.map {|res| JSON.parse(res.response)['ok'] }
11
+ body = {:ok => !ok.include?(false)}
12
+ send_response(responses.first.response_header.status,
13
+ response_headers, [body.to_json])
14
+ end
15
+ end
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,10 @@
1
+ # encoding: UTF-8
2
+
3
+ module CouchProxy
4
+ module Rack
5
+ class Root < Base
6
+ alias :get :proxy_to_any_node
7
+ alias :head :proxy_to_any_node
8
+ end
9
+ end
10
+ end
@@ -0,0 +1,53 @@
1
+ # encoding: UTF-8
2
+
3
+ module CouchProxy
4
+ module Rack
5
+ class Stats < Base
6
+ def get
7
+ proxy_to_all_nodes do |responses|
8
+ docs = responses.map {|res| parse(res.response) }
9
+ total = docs.shift.tap do |doc|
10
+ each_stat(doc) do |group, name, values|
11
+ %w[means stddevs].each {|k| values[k] = [values[k.chop]] }
12
+ end
13
+ end
14
+ docs.each do |doc|
15
+ each_stat(total) do |group, name, values|
16
+ %w[current sum].each {|k| values[k] += doc[group][name][k] }
17
+ %w[means stddevs].each {|k| values[k] << doc[group][name][k.chop] }
18
+ %w[min max].each {|k| values[k] = [values[k], doc[group][name][k]].send(k) }
19
+ end
20
+ end
21
+ each_stat(total) do |group, name, values|
22
+ means, stddevs = %w[means stddevs].map {|k| values.delete(k) }
23
+ mean = means.inject(:+) / means.size.to_f
24
+ sums = means.zip(stddevs).map {|m, sd| m**2 + sd**2 }
25
+ stddev = Math.sqrt(sums.inject(:+) / means.size.to_f - mean**2)
26
+ mean, stddev = [mean, stddev].map {|f| sprintf('%.3f', f).to_f }
27
+ values.update('stddev' => stddev, 'mean' => mean)
28
+ end
29
+ send_response(responses.first.response_header.status,
30
+ response_headers, [total.to_json])
31
+ end
32
+ end
33
+
34
+ private
35
+
36
+ def parse(body)
37
+ JSON.parse(body).tap do |doc|
38
+ each_stat(doc) do |group, name, values|
39
+ values.each {|k, v| values[k] = 0 unless v }
40
+ end
41
+ end
42
+ end
43
+
44
+ def each_stat(doc)
45
+ doc.each do |group, stats|
46
+ stats.each do |name, values|
47
+ yield group, name, values
48
+ end
49
+ end
50
+ end
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,9 @@
1
+ # encoding: UTF-8
2
+
3
+ module CouchProxy
4
+ module Rack
5
+ class TempView < Base
6
+ # FIXME
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,11 @@
1
+ # encoding: UTF-8
2
+
3
+ module CouchProxy
4
+ module Rack
5
+ class Update < Base
6
+ def post
7
+ # FIXME http://wiki.apache.org/couchdb/Document_Update_Handlers
8
+ end
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,9 @@
1
+ # encoding: UTF-8
2
+
3
+ module CouchProxy
4
+ module Rack
5
+ class Users < Base
6
+ # FIXME
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,9 @@
1
+ # encoding: UTF-8
2
+
3
+ module CouchProxy
4
+ module Rack
5
+ class Uuids < Base
6
+ alias :get :proxy_to_any_node
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,16 @@
1
+ # encoding: UTF-8
2
+
3
+ module CouchProxy
4
+ module Rack
5
+ class ViewCleanup < Base
6
+ def post
7
+ proxy_to_all_partitions do |responses|
8
+ ok = responses.map {|res| JSON.parse(res.response)['ok'] }
9
+ body = {:ok => !ok.include?(false)}
10
+ send_response(responses.first.response_header.status,
11
+ response_headers, [body.to_json])
12
+ end
13
+ end
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,57 @@
1
+ # encoding: UTF-8
2
+
3
+ module CouchProxy
4
+ class Reducer
5
+ def initialize(couchjs)
6
+ @couchjs, @conn = "#{couchjs} #{mainjs(couchjs)}", nil
7
+ end
8
+
9
+ def rereduce(fn, values, &callback)
10
+ connect unless @conn
11
+ @conn.rereduce(fn, values, callback)
12
+ end
13
+
14
+ private
15
+
16
+ def mainjs(couchjs)
17
+ File.expand_path('../../share/couchdb/server/main.js', couchjs)
18
+ end
19
+
20
+ def connect
21
+ @conn = EM.popen(@couchjs, ReduceProcess, proc { @conn = nil })
22
+ end
23
+ end
24
+
25
+ class ReduceProcess < EventMachine::Connection
26
+ def initialize(unbind)
27
+ @unbind, @connected, @callbacks, @deferred = unbind, false, [], []
28
+ end
29
+
30
+ def post_init
31
+ @connected = true
32
+ @deferred.slice!(0, @deferred.size).each do |fn, values, callback|
33
+ rereduce(fn, values, callback)
34
+ end
35
+ end
36
+
37
+ def rereduce(fn, values, callback)
38
+ if @connected
39
+ @callbacks << callback
40
+ send_data(["rereduce", [fn], values].to_json + "\n")
41
+ else
42
+ @deferred << [fn, values, callback]
43
+ end
44
+ end
45
+
46
+ def receive_data(data)
47
+ data.split("\n").each do |line|
48
+ @callbacks.shift.call(JSON.parse(line))
49
+ end
50
+ end
51
+
52
+ def unbind
53
+ @connected = false
54
+ @unbind.call
55
+ end
56
+ end
57
+ end
@@ -0,0 +1,50 @@
1
+ module Rack
2
+ # Add a few helper methods to Rack's Request class.
3
+ class Request
4
+ def json?
5
+ if accept = @env['HTTP_ACCEPT']
6
+ accept.tr(' ', '').split(',').include?('application/json')
7
+ end
8
+ end
9
+
10
+ def db_name
11
+ parse_db_name_and_doc_id[0]
12
+ end
13
+
14
+ def doc_id
15
+ parse_db_name_and_doc_id[1]
16
+ end
17
+
18
+ def rewrite_proxy_url(partition_num)
19
+ path_info.sub(/^\/#{db_name}/, "/#{db_name}_#{partition_num}")
20
+ end
21
+
22
+ def rewrite_proxy_url!(partition_num)
23
+ @env['PATH_INFO'] = rewrite_proxy_url(partition_num)
24
+ end
25
+
26
+ def content
27
+ unless defined? @proxy_content
28
+ body.rewind
29
+ @proxy_content = body.read
30
+ body.rewind
31
+ end
32
+ @proxy_content
33
+ end
34
+
35
+ def content=(body)
36
+ @proxy_content = body
37
+ @env['CONTENT_LENGTH'] = body.bytesize
38
+ end
39
+
40
+ private
41
+
42
+ def parse_db_name_and_doc_id
43
+ unless defined? @db_name
44
+ path = @env['REQUEST_PATH'][1..-1].chomp('/')
45
+ @db_name, @doc_id = path.split('/', 2).map {|n| ::Rack::Utils.unescape(n) }
46
+ end
47
+ [@db_name, @doc_id]
48
+ end
49
+ end
50
+ end