mongo_delegate 0.1.2 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README.rdoc +2 -2
- data/Rakefile +13 -0
- data/VERSION +1 -1
- data/lib/mongo_delegate/composite_cursor.rb +60 -3
- data/spec/delegating_collection_spec.rb +44 -67
- data/spec/id_time.rb +49 -0
- data/spec/spec_helper.rb +1 -0
- metadata +3 -2
data/README.rdoc
CHANGED
@@ -6,11 +6,11 @@ A DelegateCollection wraps a local collection and remote collection in one objec
|
|
6
6
|
|
7
7
|
This was inspired by things I've been told about Goldman Sachs' proprietary object database. You can point one database at another remote database, and it will function in this manner.
|
8
8
|
|
9
|
-
The classic use case is debugging a production problem, or developing while using production data.
|
9
|
+
=== The classic use case is debugging a production problem, or developing while using production data.
|
10
10
|
|
11
11
|
The annoying ways
|
12
12
|
* Generate data that mimics production
|
13
|
-
*
|
13
|
+
* Get a production dump and load it locally
|
14
14
|
* "Test" in production while walking on eggshells to make sure not to break anything
|
15
15
|
|
16
16
|
The easy way
|
data/Rakefile
CHANGED
@@ -50,3 +50,16 @@ Rake::RDocTask.new do |rdoc|
|
|
50
50
|
end
|
51
51
|
|
52
52
|
Jeweler::GemcutterTasks.new
|
53
|
+
|
54
|
+
task :run_specs do
|
55
|
+
require 'directory_watcher'
|
56
|
+
|
57
|
+
dw = DirectoryWatcher.new '.'
|
58
|
+
dw.glob = "**/*"
|
59
|
+
dw.interval = 0.25
|
60
|
+
dw.add_observer {|*args| Rake::Task['spec'].execute }
|
61
|
+
|
62
|
+
dw.start
|
63
|
+
gets # when the user hits "enter" the script will terminate
|
64
|
+
dw.stop
|
65
|
+
end
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.2.0
|
@@ -1,3 +1,34 @@
|
|
1
|
+
class Mongo::Cursor
|
2
|
+
def foc
|
3
|
+
formatted_order_clause
|
4
|
+
rescue => exp
|
5
|
+
return nil
|
6
|
+
end
|
7
|
+
end
|
8
|
+
|
9
|
+
class Numeric
|
10
|
+
def sortflip
|
11
|
+
self * -1.0
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
class String
|
16
|
+
def num_val
|
17
|
+
return 0 if length == 0
|
18
|
+
self[-1] + 500*self[0..-2].num_val
|
19
|
+
end
|
20
|
+
def sortflip
|
21
|
+
opposite_word
|
22
|
+
end
|
23
|
+
def opposite_word
|
24
|
+
res = " " * length
|
25
|
+
for i in (0...length)
|
26
|
+
res[i] = 255-self[i]
|
27
|
+
end
|
28
|
+
res
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
1
32
|
class CompositeCursor
|
2
33
|
class FindOps
|
3
34
|
attr_accessor :res, :cursor
|
@@ -5,10 +36,22 @@ class CompositeCursor
|
|
5
36
|
def options; cursor.options; end
|
6
37
|
fattr(:num_using) { res.map { |x| x[0].to_af.size }.sum }
|
7
38
|
fattr(:found_ids) { res.map { |x| x[1].to_af.map { |x| x['_id'] } }.flatten }
|
39
|
+
fattr(:limit) do
|
40
|
+
if options[:limit]
|
41
|
+
if options[:sort]
|
42
|
+
options[:limit]
|
43
|
+
else
|
44
|
+
options[:limit] - num_using
|
45
|
+
end
|
46
|
+
else
|
47
|
+
nil
|
48
|
+
end
|
49
|
+
end
|
8
50
|
fattr(:ops) do
|
9
51
|
r = {}
|
10
|
-
r[:limit] =
|
11
|
-
r[:skip] = [(options[:skip] - found_ids.size ),0].max if options[:skip]
|
52
|
+
r[:limit] = limit if limit
|
53
|
+
r[:skip] = [(options[:skip] - found_ids.size ),0].max if options[:skip] && !options[:sort]
|
54
|
+
r[:sort] = options[:sort] if options[:sort]
|
12
55
|
r
|
13
56
|
end
|
14
57
|
def zero_limit?
|
@@ -20,9 +63,23 @@ class CompositeCursor
|
|
20
63
|
include FromHash
|
21
64
|
include Enumerable
|
22
65
|
def to_af; rows; end
|
23
|
-
|
66
|
+
def sort_proc
|
67
|
+
foc = cursors.first.foc
|
68
|
+
return nil unless foc
|
69
|
+
lambda do |doc|
|
70
|
+
foc.map { |k,v| (v == 1) ? doc[k] : doc[k].sortflip }
|
71
|
+
end
|
72
|
+
end
|
73
|
+
fattr(:raw_rows) do
|
24
74
|
cursors.map { |x| x.to_af }.flatten
|
25
75
|
end
|
76
|
+
fattr(:rows) do
|
77
|
+
res = raw_rows
|
78
|
+
res = res.sort_by(&sort_proc) if sort_proc
|
79
|
+
res = res[0...(options[:limit])] if options[:limit]
|
80
|
+
res = res[(options[:skip])..-1] if options[:skip] && options[:sort]
|
81
|
+
res
|
82
|
+
end
|
26
83
|
def each(&b)
|
27
84
|
rows.each(&b)
|
28
85
|
end
|
@@ -1,55 +1,5 @@
|
|
1
1
|
require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
|
2
2
|
|
3
|
-
class Array
|
4
|
-
def num_from_bytes
|
5
|
-
return 0 if empty?
|
6
|
-
self[-1] + 256 * self[0..-2].num_from_bytes
|
7
|
-
end
|
8
|
-
def time_from_bytes
|
9
|
-
Time.at(num_from_bytes)
|
10
|
-
end
|
11
|
-
end
|
12
|
-
|
13
|
-
class Numeric
|
14
|
-
def to_bytes
|
15
|
-
return [] if self == 0
|
16
|
-
(self/256).to_bytes + [self%256]
|
17
|
-
end
|
18
|
-
def to_bytes4
|
19
|
-
res = to_bytes
|
20
|
-
while res.length < 4
|
21
|
-
res = [0] + res
|
22
|
-
end
|
23
|
-
res
|
24
|
-
end
|
25
|
-
end
|
26
|
-
|
27
|
-
class Time
|
28
|
-
def to_small_mongo_id
|
29
|
-
res = Mongo::ObjectID.new
|
30
|
-
res.data = to_i.to_bytes4 + (0...8).map { |x| 0 }
|
31
|
-
res
|
32
|
-
end
|
33
|
-
def to_big_mongo_id
|
34
|
-
res = Mongo::ObjectID.new
|
35
|
-
res.data = to_i.to_bytes4 + (0...8).map { |x| 255 }
|
36
|
-
res
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
class Mongo::ObjectID
|
41
|
-
attr_accessor :data
|
42
|
-
def time
|
43
|
-
data[0...4].time_from_bytes
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
|
-
class Hash
|
48
|
-
def created_at
|
49
|
-
self['_id'].time
|
50
|
-
end
|
51
|
-
end
|
52
|
-
|
53
3
|
context Mongo::DelegatingCollection do
|
54
4
|
def setup_coll
|
55
5
|
@start_time = Time.now
|
@@ -57,8 +7,8 @@ context Mongo::DelegatingCollection do
|
|
57
7
|
@db = Mongo::DelegatingDatabase.new(:remote => @conn.db('dc-remote'), :local => @conn.db('dc-local'))
|
58
8
|
@d = @db.collection('abc')
|
59
9
|
@d.remove
|
60
|
-
@remotes = %w(
|
61
|
-
@locals = %w(
|
10
|
+
@remotes = %w(Barbara Danny Frank).each { |x| @d.remote.save(:name => x) }
|
11
|
+
@locals = %w(Adam Chris Eric).each { |x| @d.local.save(:name => x) }
|
62
12
|
@all = @remotes + @locals
|
63
13
|
end
|
64
14
|
context 'find' do
|
@@ -67,7 +17,7 @@ context Mongo::DelegatingCollection do
|
|
67
17
|
@d.find.count.should == @all.size
|
68
18
|
end
|
69
19
|
it 'find doesnt return dups' do
|
70
|
-
@d.save(@d.remote.find_one(:name =>
|
20
|
+
@d.save(@d.remote.find_one(:name => @remotes.first))
|
71
21
|
@d.find.count.should == @all.size
|
72
22
|
@d.find.unpruned_rows.size.should == @all.size + 1
|
73
23
|
end
|
@@ -77,9 +27,9 @@ context Mongo::DelegatingCollection do
|
|
77
27
|
@d.find.rows
|
78
28
|
end
|
79
29
|
it 'find returns local precedence' do
|
80
|
-
r = @d.remote.find_one(:name =>
|
30
|
+
r = @d.remote.find_one(:name => @remotes.first).merge(:age => 'old')
|
81
31
|
@d.local.save(r)
|
82
|
-
@d.find_one(:name =>
|
32
|
+
@d.find_one(:name => @remotes.first)['age'].should == 'old'
|
83
33
|
end
|
84
34
|
it 'respects limit' do
|
85
35
|
@d.find({},:limit => 4).to_af.size.should == 4
|
@@ -91,25 +41,52 @@ context Mongo::DelegatingCollection do
|
|
91
41
|
@d.find({},:limit => 2).rows
|
92
42
|
end
|
93
43
|
it 'respects skip' do
|
94
|
-
@d.find({},:skip => 1).to_af.size.should ==
|
95
|
-
@d.find({},:skip => 1).map { |x| x['name'] }.sort.should == @all.reject { |x| x ==
|
44
|
+
@d.find({},:skip => 1).to_af.size.should == @all.size - 1
|
45
|
+
@d.find({},:skip => 1).map { |x| x['name'] }.sort.should == @all.reject { |x| x == @locals.first }.sort
|
96
46
|
end
|
97
47
|
it 'respects skip whole collection' do
|
98
|
-
@d.find({},:skip => 4).to_af.size.should ==
|
48
|
+
@d.find({},:skip => 4).to_af.size.should == @all.size - 4
|
99
49
|
end
|
100
50
|
it 'wont return remote records that were skipped locally' do
|
101
|
-
@d.local.save(@d.remote.find_one(:name =>
|
102
|
-
@d.find({},:skip => 5).to_af.size.should ==
|
51
|
+
@d.local.save(@d.remote.find_one(:name => @remotes.first))
|
52
|
+
@d.find({},:skip => 5).to_af.size.should == @all.size - 5
|
103
53
|
end
|
104
54
|
it 'respects skip and limit' do
|
105
55
|
@d.find({},:skip => 1, :limit => 4).to_af.size.should == 4
|
106
56
|
end
|
107
57
|
it 'count doesnt fetch records' do
|
108
58
|
mock.instance_of(Mongo::Cursor).to_a.times(0)
|
109
|
-
@d.count.should ==
|
59
|
+
@d.count.should == @all.size
|
110
60
|
end
|
111
|
-
it 'abc' do
|
112
|
-
|
61
|
+
# it 'abc' do
|
62
|
+
# @d.local.scope_gte('_id' => @start_time.to_small_mongo_id).count.should == 3
|
63
|
+
# end
|
64
|
+
context 'sorting' do
|
65
|
+
it 'honors sort order when retreiving all records' do
|
66
|
+
@d.find({},:sort => [['name','ascending']]).map { |x| x['name'] }.should == @all.sort
|
67
|
+
end
|
68
|
+
it 'honors reverse sort order when retreiving all records' do
|
69
|
+
@d.find({},:sort => [['name','descending']]).map { |x| x['name'] }.should == @all.sort.reverse
|
70
|
+
end
|
71
|
+
it 'honors sort order with a limit' do
|
72
|
+
exp = @all.sort[0...4]
|
73
|
+
@d.find({},:sort => [['name','ascending']], :limit => 4).map { |x| x['name'] }.should == exp
|
74
|
+
end
|
75
|
+
it 'honors sort order with a limit desc' do
|
76
|
+
exp = @all.sort.reverse[0...4]
|
77
|
+
@d.find({},:sort => [['name','descending']], :limit => 4).map { |x| x['name'] }.should == exp
|
78
|
+
end
|
79
|
+
it 'honors sort order with a skip' do
|
80
|
+
exp = @all.sort[1..-1]
|
81
|
+
@d.find({},:sort => [['name','ascending']], :skip => 1).map { |x| x['name'] }.should == exp
|
82
|
+
end
|
83
|
+
it 'honors sort order with a skip desc' do
|
84
|
+
exp = @all.sort.reverse[1..-1]
|
85
|
+
@d.find({},:sort => [['name','descending']], :skip => 1).map { |x| x['name'] }.should == exp
|
86
|
+
end
|
87
|
+
it 'when sorting with a limit, never retreives more than the limit from any one collection' do
|
88
|
+
@d.find({},:sort => [['name','ascending']], :limit => 2).raw_rows.size.should == 4
|
89
|
+
end
|
113
90
|
end
|
114
91
|
# it 'count honors remote deletes' do
|
115
92
|
# @d.remote.find.each { |x| @d.save(x) }
|
@@ -126,16 +103,16 @@ context Mongo::DelegatingCollection do
|
|
126
103
|
@d.local.count.should == @locals.size + 1
|
127
104
|
end
|
128
105
|
it 'updating a non-dup shouldnt mark it as a dup' do
|
129
|
-
r = @d.local.find_one(:name =>
|
106
|
+
r = @d.local.find_one(:name => @locals.first).merge(:foo => :bar)
|
130
107
|
@d.save(r)
|
131
108
|
@d.count.should == 6
|
132
|
-
@d.local.find_one(:name =>
|
109
|
+
@d.local.find_one(:name => @locals.first)['_duplicate'].should be_nil
|
133
110
|
end
|
134
111
|
it 'saving a dup for first time should mark it as a dup' do
|
135
|
-
r = @d.remote.find_one(:name =>
|
112
|
+
r = @d.remote.find_one(:name => @remotes.first)
|
136
113
|
@d.save(r)
|
137
114
|
@d.count.should == 6
|
138
|
-
@d.local.find_one(:name =>
|
115
|
+
@d.local.find_one(:name => @remotes.first)['_duplicate'].should == true
|
139
116
|
end
|
140
117
|
end
|
141
118
|
end
|
data/spec/id_time.rb
ADDED
@@ -0,0 +1,49 @@
|
|
1
|
+
class Array
|
2
|
+
def num_from_bytes
|
3
|
+
return 0 if empty?
|
4
|
+
self[-1] + 256 * self[0..-2].num_from_bytes
|
5
|
+
end
|
6
|
+
def time_from_bytes
|
7
|
+
Time.at(num_from_bytes)
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
class Numeric
|
12
|
+
def to_bytes
|
13
|
+
return [] if self == 0
|
14
|
+
(self/256).to_bytes + [self%256]
|
15
|
+
end
|
16
|
+
def to_bytes4
|
17
|
+
res = to_bytes
|
18
|
+
while res.length < 4
|
19
|
+
res = [0] + res
|
20
|
+
end
|
21
|
+
res
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
class Time
|
26
|
+
def to_small_mongo_id
|
27
|
+
res = Mongo::ObjectID.new
|
28
|
+
res.data = to_i.to_bytes4 + (0...8).map { |x| 0 }
|
29
|
+
res
|
30
|
+
end
|
31
|
+
def to_big_mongo_id
|
32
|
+
res = Mongo::ObjectID.new
|
33
|
+
res.data = to_i.to_bytes4 + (0...8).map { |x| 255 }
|
34
|
+
res
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
class Mongo::ObjectID
|
39
|
+
attr_accessor :data
|
40
|
+
def time
|
41
|
+
data[0...4].time_from_bytes
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
class Hash
|
46
|
+
def created_at
|
47
|
+
self['_id'].time
|
48
|
+
end
|
49
|
+
end
|
data/spec/spec_helper.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mongo_delegate
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Mike Harris
|
@@ -9,7 +9,7 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2010-01-
|
12
|
+
date: 2010-01-15 00:00:00 -05:00
|
13
13
|
default_executable:
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
@@ -151,5 +151,6 @@ specification_version: 3
|
|
151
151
|
summary: restful interface to delegate mongodb records
|
152
152
|
test_files:
|
153
153
|
- spec/delegating_collection_spec.rb
|
154
|
+
- spec/id_time.rb
|
154
155
|
- spec/mongo_delegate_spec.rb
|
155
156
|
- spec/spec_helper.rb
|