mongo_mapper_parallel 1.0.6 → 1.0.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/lib/mongo_mapper_parallel.rb +20 -11
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
YzRjYTA1ZjY1MDJlYjhlYmRmN2JjYjQ1NjYwOTM5ZDg2NDUxYmQ1YQ==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
NzI4MDNmNTExZmQ5NTUyZWUyOTRkZGM0ZDExMDlmNTNlNGZlNTViNg==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
OTk4MDUzNzA3M2Y3MzU2NjdkNTRlODBmN2NjY2I0NDAxNjE2ZTliMGYxYjIy
|
10
|
+
MGY1Y2VlNjUwZmVjNWFhYjNhMDM4YmIzZjFiYTVjNWI2ZDJhNGM1Y2MzNDRk
|
11
|
+
YTFkNDU3MjE5Y2VlODhiNzZjNGFlM2YxOWE4MWJjYmQ3MGMxZmY=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
NjFjN2U4Yzg4OWI3Y2Y5YWVhYzA0NDk0ZWUyYTU1ODllMGM0ZjA3MGFjYzZh
|
14
|
+
MmMzOTE5NjQ1N2Y0ODkxODMyM2ZhNTcxODc0YzUzZDgxYTI0ZjBhZmI1MGY1
|
15
|
+
YzQwM2YzMjQ4N2JiY2Y4OGI0ZTQxMjJhOTdlZTZiZTg1YmYwNmY=
|
@@ -11,6 +11,7 @@ class MongoMapperParallel
|
|
11
11
|
attr_accessor :command_class
|
12
12
|
attr_accessor :javascript
|
13
13
|
attr_accessor :args
|
14
|
+
attr_accessor :debug
|
14
15
|
|
15
16
|
class Key
|
16
17
|
# A chunk that will be parallelized
|
@@ -18,6 +19,7 @@ class MongoMapperParallel
|
|
18
19
|
attr_accessor :key
|
19
20
|
attr_accessor :completed
|
20
21
|
attr_reader :compiler
|
22
|
+
attr_accessor :debug
|
21
23
|
|
22
24
|
# A chunk that will be parallelized
|
23
25
|
#
|
@@ -25,12 +27,13 @@ class MongoMapperParallel
|
|
25
27
|
# @option opts [String] :key the lower bound of the range of resources to retrieve
|
26
28
|
# @option opts [String] :future_key the upper bound for the range of resources to retrieve
|
27
29
|
# @option opts [MongoMapperParallel] :compiler the Parallel execution object that holds the keys, javascript, and arguments.
|
28
|
-
#
|
30
|
+
# @option opts [Boolean] :debug whether to show messages during the process.
|
29
31
|
def initialize(opts={})
|
30
32
|
@key = opts[:key]
|
31
33
|
@compiler = opts[:compiler]
|
32
34
|
@future_key = opts[:future_key]
|
33
35
|
@completed = false
|
36
|
+
@debug = opts[:debug].nil? true : opts[:debug]
|
34
37
|
end
|
35
38
|
|
36
39
|
# The javascript function to run on the resources
|
@@ -54,13 +57,17 @@ class MongoMapperParallel
|
|
54
57
|
def compile
|
55
58
|
search_opts = {:name => {:$gte => @key}}
|
56
59
|
if @future_key then search_opts[:name][:$lte] = @future_key end
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
60
|
+
begin
|
61
|
+
command_class.database.command({
|
62
|
+
:"$eval" => javascript,
|
63
|
+
:args => [@key, @future_key, args],
|
64
|
+
:nolock => true
|
65
|
+
})
|
66
|
+
rescue Mongo::ConnectionFailure
|
67
|
+
raise Mongo::ConnectionFailure.new "Connection disconnected at position #{@position}"
|
68
|
+
end
|
62
69
|
@completed = true
|
63
|
-
puts "Completed chunk".green
|
70
|
+
puts "Completed chunk (#{@position})".green if @debug
|
64
71
|
end
|
65
72
|
end
|
66
73
|
|
@@ -71,7 +78,7 @@ class MongoMapperParallel
|
|
71
78
|
def get_split_keys
|
72
79
|
@split_keys, splits = [], @command_class.database.command({splitVector: "#{@command_class.database.name}.#{@command_class.collection.name}", keyPattern: {@split.to_sym => 1}, maxChunkSizeBytes: @splitSize })["splitKeys"]
|
73
80
|
splits.each_with_index do |split_key,k|
|
74
|
-
@split_keys << MongoMapperParallel::Key.new(:compiler => self, :key => split_key[@split.to_s], :future_key => (splits[k+1] ? splits[k+1][@split.to_s] : nil))
|
81
|
+
@split_keys << MongoMapperParallel::Key.new(:position => k, :compiler => self, :key => split_key[@split.to_s], :future_key => (splits[k+1] ? splits[k+1][@split.to_s] : nil),:debug => @debug)
|
75
82
|
end
|
76
83
|
if @split_keys.length == 0 and @command_class.count > 0 then get_extreme_split_keys end
|
77
84
|
end
|
@@ -81,7 +88,7 @@ class MongoMapperParallel
|
|
81
88
|
# @return [Array<MongoMapperParallel::Key>] the list of the keys that will be used for parallel operation
|
82
89
|
def get_extreme_split_keys
|
83
90
|
split_key = @command_class.where().order(@split.to_sym).fields(@split.to_sym).first.send(@split.to_sym)
|
84
|
-
@split_keys << MongoMapperParallel::Key.new(:compiler => self, :key => split_key, :future_key => nil)
|
91
|
+
@split_keys << MongoMapperParallel::Key.new(:position => 0, :compiler => self, :key => split_key, :future_key => nil, :debug => @debug)
|
85
92
|
end
|
86
93
|
|
87
94
|
# Instantiates the parallel operation object with the right class, javascript function, and field
|
@@ -92,6 +99,7 @@ class MongoMapperParallel
|
|
92
99
|
# @option opts [Array, Hash] :args the arguments to pass to the Javascript function
|
93
100
|
# @option opts [String, Symbol] :split the field to split the computation on -- typically an indexed unique property of the resources in the collection.
|
94
101
|
# @option opts [Fixnum] :maxChunkSizeBytes the size of the chunks to parallelize. Defaults to `32*1024*1024 = 33554432`.
|
102
|
+
# @option opts [Boolean] :debug whether to show messages during the process.
|
95
103
|
# @return [MongoMapperParallel]
|
96
104
|
#
|
97
105
|
def initialize(opts={})
|
@@ -100,6 +108,7 @@ class MongoMapperParallel
|
|
100
108
|
@args = opts[:args]
|
101
109
|
@split = opts[:split] # name, title, etc...
|
102
110
|
@splitSize = opts[:maxChunkSizeBytes] || 32*1024*1024
|
111
|
+
@debug = opts[:debug].nil? true : opts[:debug]
|
103
112
|
get_split_keys()
|
104
113
|
self
|
105
114
|
end
|
@@ -110,9 +119,9 @@ class MongoMapperParallel
|
|
110
119
|
total = @split_keys.length
|
111
120
|
Parallel.each_with_index(@split_keys) do |section,k|
|
112
121
|
if !section.completed then section.compile end
|
113
|
-
JRProgressBar.show(k,total)
|
122
|
+
JRProgressBar.show(k,total) if @debug
|
114
123
|
end
|
115
|
-
puts "Success".green
|
124
|
+
puts "Success".green if @debug
|
116
125
|
end
|
117
126
|
|
118
127
|
# In case of stalled progress you can skip ahead by a percentage and mark the keys as `completed`.
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mongo_mapper_parallel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jonathan Raiman
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2013-11-
|
11
|
+
date: 2013-11-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: mongo_mapper
|