mongo_mapper_parallel 1.0.3 → 1.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/lib/mongo_mapper_parallel.rb +31 -11
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
MDI2MWE3ZWJlMDg3ZDYyNzExZWJkNzgwYmE3MzJjNGQ2ZWJlMmZiMw==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
ODA5ZjFiMTc3NzMwODAyODY1OWVjOGI3NmJhZjFkODQ3ZGYyYzhlZQ==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
YjM5MDkyNTg2NzVjZmY3OWI3MmNkMTM2ZDg4MjQyMmUzYTRhMTUxZDVmZmRh
|
10
|
+
NWUzM2I0ZDg4MGQyMTNiNTYwNTQzNDE5MDJkNjRmOTE4ZDljNTlhYjc0NTcx
|
11
|
+
ZTQ4MTliNjE0OWMxMDdmY2Q5NmM0NTE1NTgwMjEwYjZiZjViMjE=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
ZjkzZDg2Y2EwNTBkNTA5Y2VlYjgwNjhjYmMzZTE0Y2Q0N2RhYTAzOGFlMjIy
|
14
|
+
OWVlOTEwYTA4NDYyNTNjNzMxYTk1YjNlMTFhZTBmZmE0MTc4NjM1YTlmOGZj
|
15
|
+
NjMzZTg0M2ZiMjU5ZWRjOWRiYWM2YTE2MTRmZDBjNTYyNTk2ZjU=
|
@@ -1,4 +1,3 @@
|
|
1
|
-
# @title Mongo Mapper Parallel
|
2
1
|
# @author Jonathan Raiman
|
3
2
|
require 'parallel' # for parallel processing
|
4
3
|
require 'colorize' # for colored output
|
@@ -22,9 +21,10 @@ class MongoMapperParallel
|
|
22
21
|
|
23
22
|
# A chunk that will be parallelized
|
24
23
|
#
|
25
|
-
# @param
|
26
|
-
# @
|
27
|
-
# @
|
24
|
+
# @param [Hash] opts the options to create the chunk.
|
25
|
+
# @option opts [String] :key the lower bound of the range of resources to retrieve
|
26
|
+
# @option opts [String] :future_key the upper bound for the range of resources to retrieve
|
27
|
+
# @option opts [MongoMapperParallel] :compiler the Parallel execution object that holds the keys, javascript, and arguments.
|
28
28
|
#
|
29
29
|
def initialize(opts={})
|
30
30
|
@key = opts[:key]
|
@@ -66,10 +66,10 @@ class MongoMapperParallel
|
|
66
66
|
|
67
67
|
# Obtains the splitVectors keys to find chunks to parallelize via the MongoDB `splitVector` command.
|
68
68
|
#
|
69
|
-
# @return
|
69
|
+
# @return [Array<MongoMapperParallel::Key>] the list of the keys that will be used for parallel operation
|
70
70
|
#
|
71
71
|
def get_split_keys
|
72
|
-
@split_keys, splits = [], @command_class.database.command({splitVector: "#{@command_class.database.name}.#{@command_class.collection.name}", keyPattern: {@split.to_sym => 1}, maxChunkSizeBytes:
|
72
|
+
@split_keys, splits = [], @command_class.database.command({splitVector: "#{@command_class.database.name}.#{@command_class.collection.name}", keyPattern: {@split.to_sym => 1}, maxChunkSizeBytes: @splitSize })["splitKeys"]
|
73
73
|
splits.each_with_index do |split_key,k|
|
74
74
|
@split_keys << MongoMapperParallel::Key.new(:compiler => self, :key => split_key[@split.to_s], :future_key => (splits[k+1] ? splits[k+1][@split.to_s] : nil))
|
75
75
|
end
|
@@ -77,17 +77,20 @@ class MongoMapperParallel
|
|
77
77
|
|
78
78
|
# Instantiates the parallel operation object with the right class, javascript function, and field
|
79
79
|
#
|
80
|
-
# @param
|
81
|
-
# @
|
82
|
-
# @
|
83
|
-
# @
|
84
|
-
# @
|
80
|
+
# @param opts [Hash] the options to initialize the parallel script.
|
81
|
+
# @option opts [Class] :class the Mongo collection's Ruby Class to execute operations on.
|
82
|
+
# @option opts [String] :javascript the Javascript function in String format
|
83
|
+
# @option opts [Array, Hash] :args the arguments to pass to the Javascript function
|
84
|
+
# @option opts [String, Symbol] :split the field to split the computation on -- typically an indexed unique property of the resources in the collection.
|
85
|
+
# @option opts [Fixnum] :maxChunkSizeBytes the size of the chunks to parallelize. Defaults to `32*1024*1024 = 33554432`.
|
86
|
+
# @return [MongoMapperParallel]
|
85
87
|
#
|
86
88
|
def initialize(opts={})
|
87
89
|
@command_class = opts[:class]
|
88
90
|
@javascript = opts[:javascript]
|
89
91
|
@args = opts[:args]
|
90
92
|
@split = opts[:split] # name, title, etc...
|
93
|
+
@splitSize = opts[:maxChunkSizeBytes] || 32*1024*1024
|
91
94
|
get_split_keys()
|
92
95
|
self
|
93
96
|
end
|
@@ -103,4 +106,21 @@ class MongoMapperParallel
|
|
103
106
|
puts "Success".green
|
104
107
|
end
|
105
108
|
|
109
|
+
# Occurs when {#advance} receives an out of bounds percentage
|
110
|
+
class ProgressError < Error; end
|
111
|
+
|
112
|
+
# In case of stalled progress you can skip ahead by a percentage and mark the keys as `completed`.
|
113
|
+
#
|
114
|
+
# @param percentage [Float] how far along you want to advance, a value between 0.0 and 1.0
|
115
|
+
# @return [MongoMapperParallel]
|
116
|
+
def advance percentage
|
117
|
+
if percentage.class != Float
|
118
|
+
raise TypeError.new "Can only advance by a Float value."
|
119
|
+
elsif percentage > 1.0 or percentage < 0.0
|
120
|
+
raise ProgressError.new "Can only advance by a Float between 0.0 and 1.0."
|
121
|
+
end
|
122
|
+
@split_keys[0..(@split_keys.length*percentage).to_i].each {|i| i.completed = true}
|
123
|
+
self
|
124
|
+
end
|
125
|
+
|
106
126
|
end
|