mongo_agent 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.rspec +2 -0
- data/Gemfile +4 -0
- data/Rakefile +12 -0
- data/lib/mongo_agent/agent.rb +270 -0
- data/lib/mongo_agent/db.rb +27 -0
- data/lib/mongo_agent/error.rb +8 -0
- data/lib/mongo_agent.rb +24 -0
- data/spec/mongo_agent_spec.rb +353 -0
- data/spec/spec_helper.rb +87 -0
- metadata +62 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: ee39eefc1decdd3c2831ccad3130d9e6f2836ba3
|
4
|
+
data.tar.gz: be4191785f44ef768946182a95b0cca390ad5229
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 8943b3c9a745020ddf853ff1f89d637c0a09a874987d4d8256527cdd143582d75922d5a9170fcbc8fa9dd54c9a8d2e8e882981778ff7b1aa0396660c1242eb49
|
7
|
+
data.tar.gz: 641d079a4dc1a092b9a4d566d2fafb90af2e46d0542b55664ece1f9a2b1c8618a61dc21d773d65106824063ed0f8b02c1422ab9e7a9c417f4396a772c6099cdd
|
data/.rspec
ADDED
data/Gemfile
ADDED
data/Rakefile
ADDED
@@ -0,0 +1,270 @@
|
|
1
|
+
require 'moped'
|
2
|
+
|
3
|
+
module MongoAgent
|
4
|
+
|
5
|
+
# @note The license of this source is "MIT Licence"
|
6
|
+
|
7
|
+
# MongoAgent::Agent is designed to make it easy to create an agent that works on tasks
|
8
|
+
# queued up in a specified queue. A task is a document in a queue defined in
|
9
|
+
# the MONGO_HOST MONGO_DB. A task must be created by another agent (MongoAgent::Agent,
|
10
|
+
# or Human) in the queue. It must have, at minimum, the following fields:
|
11
|
+
# agent_name: string, name of the agent that should process the task
|
12
|
+
# ready: must be true for an agent to process the task
|
13
|
+
# When the agent finds a ready task for its agent_name, it registers itself with the
|
14
|
+
# task document by updating its agent_host field to the host that is running the
|
15
|
+
# agent, and sets ready to false (meaning it is actually running). When the agent
|
16
|
+
# completes the task, it sets the complete flag to true. If the agent encounters
|
17
|
+
# an error during processing, it can signal failure by setting the errors_encountered
|
18
|
+
# field to true at the time its complete flag is updated to true.
|
19
|
+
#
|
20
|
+
# When an Agent creates a task, it can pass a variety of messages as JSON in the task
|
21
|
+
# that can be used by the processing agent. The processing agent can also pass a
|
22
|
+
# variety of messages to other agents when they review the task. These messages
|
23
|
+
# are just fields added to the task Document when it is created or updated by
|
24
|
+
# an agent.
|
25
|
+
# Any MongoAgent::Agent depends on the following environment variables to configure its
|
26
|
+
# MongoDB connection:
|
27
|
+
# MONGO_HOST: host URL for the MongoDB, can be in any form that mongod itself can
|
28
|
+
# use, e.g. host, host:port, etc.
|
29
|
+
# MONGO_DB: the name of the Document store in the MongoDB to use to find its queue.
|
30
|
+
# it will be created if it does not exist
|
31
|
+
|
32
|
+
# @author Darin London Copyright 2014
|
33
|
+
class Agent
|
34
|
+
|
35
|
+
# This holds the Moped::Session object that can be used to query information from the MongoDB
|
36
|
+
# hosted by the MONGO_HOST environment variable
|
37
|
+
# @return [Moped::Session]
|
38
|
+
attr_reader :db
|
39
|
+
|
40
|
+
# This holds the log while work! is running
|
41
|
+
# log will be a Hash with the following keys:
|
42
|
+
# tasks_processed: int number of tasks processed (success of failure)
|
43
|
+
# failed_tasks: int number of tasks that have failed
|
44
|
+
# The log is passed to the block that is assigned to process_while (see below)
|
45
|
+
# @return [Hash]
|
46
|
+
attr_reader :log
|
47
|
+
|
48
|
+
# This holds a block that will be passed the log as an argument and return true
|
49
|
+
# as long as the agent should continue to process tasks when work! is called,
|
50
|
+
# and false when work! should stop and return.
|
51
|
+
# If not set, the agent will continue to process tasks until it is killed when
|
52
|
+
# work! is called
|
53
|
+
# @return [Block]
|
54
|
+
attr_accessor :process_while
|
55
|
+
|
56
|
+
# The name of the agent for which tasks will be taken from the queue
|
57
|
+
# @return [String]
|
58
|
+
attr_accessor :name
|
59
|
+
|
60
|
+
# The name of the task queue that contains the tasks on which this agent will work.
|
61
|
+
# @return [String]
|
62
|
+
attr_accessor :queue
|
63
|
+
|
64
|
+
# number of seconds to sleep between each call to process! when running agent.work! or agent.process_while
|
65
|
+
# default 5
|
66
|
+
attr_accessor :sleep_between
|
67
|
+
|
68
|
+
# create a new MongoAgent::Agent
|
69
|
+
# @param attributes [Hash] with name, queue, and optional sleep_between
|
70
|
+
# @option attributes [String] name REQUIRED
|
71
|
+
# @option attributes [String] queue REQUIRED
|
72
|
+
# @option attributes [Int] sleep_between OPTIONAL
|
73
|
+
# @raise [MongoAgent::Error] name and queue are missing
|
74
|
+
def initialize(attributes = nil)
|
75
|
+
if attributes.nil?
|
76
|
+
raise MongoAgent::Error, "attributes Hash required with name and queue keys required"
|
77
|
+
end
|
78
|
+
@name = attributes[:name]
|
79
|
+
@queue = attributes[:queue]
|
80
|
+
unless @name && @queue
|
81
|
+
raise MongoAgent::Error, "attributes[:name] and attributes[:queue] are required!"
|
82
|
+
end
|
83
|
+
build_db()
|
84
|
+
if attributes[:sleep_between]
|
85
|
+
@sleep_between = attributes[:sleep_between]
|
86
|
+
else
|
87
|
+
@sleep_between = 5
|
88
|
+
end
|
89
|
+
@log = {
|
90
|
+
tasks_processed: 0,
|
91
|
+
failed_tasks: 0
|
92
|
+
}
|
93
|
+
@process_while = ->(log) { true }
|
94
|
+
end
|
95
|
+
|
96
|
+
# If a task for the agent is found that is ready, process! registers itself with the task
|
97
|
+
# by setting ready to false, and setting its hostname on the :agent_host field, and then
|
98
|
+
# passes the task to the supplied block. This block must return a required boolean field
|
99
|
+
# indicating success or failure, and an optional hash of key - value fields that will be
|
100
|
+
# updated on the task Document. Note, the updates are made regardless of the value of
|
101
|
+
# success. In fact, the agent can be configured to update different fields based on
|
102
|
+
# success or failure. Also, note that any key, value supported by JSON can be stored
|
103
|
+
# in the hash. This allows the agent to communicate any useful information to the task
|
104
|
+
# for other agents (MongoAgent::Agent or human) to use. The block must try at all costs
|
105
|
+
# to avoid terminating. If an error is encountered, block should return false for the
|
106
|
+
# success field to signal that the process failed. If no errors are encountered block
|
107
|
+
# should return true for the success field.
|
108
|
+
#
|
109
|
+
# @example Exit successfully and sets :complete to true on the task
|
110
|
+
# @agent->process! do |task_hash|
|
111
|
+
# foo = task_hash[:foo]
|
112
|
+
# # do something with foo to perform a task
|
113
|
+
# true
|
114
|
+
# end
|
115
|
+
#
|
116
|
+
# @example Same, but also sets the 'files_processed' field
|
117
|
+
# @agent->process! { |task_hash|
|
118
|
+
# # ... operation using task_hash for information
|
119
|
+
# [true, {:files_processed => 30}]
|
120
|
+
# }
|
121
|
+
#
|
122
|
+
# @example Fails, sets :complete to true, and :error_encountered to true
|
123
|
+
# @failure = ->(task_hash){
|
124
|
+
# begin
|
125
|
+
# # ... failing operation using task_hash for information
|
126
|
+
# return true
|
127
|
+
# rescue
|
128
|
+
# return false
|
129
|
+
# end
|
130
|
+
# }
|
131
|
+
#
|
132
|
+
# @agent->process!(&@failure)
|
133
|
+
#
|
134
|
+
# @example Same, but also sets the 'notice' field
|
135
|
+
# @agent->process! do |task_hash|
|
136
|
+
# ...
|
137
|
+
# [false, {:notice => 'There were 10 files left to process!' }]
|
138
|
+
# end
|
139
|
+
#
|
140
|
+
# @example This agent passes different parameters based on success or failure
|
141
|
+
# $agent->process! { |task_hash|
|
142
|
+
# # ... process and set $success true or false
|
143
|
+
# if $success
|
144
|
+
# [ $success, {:files_processed => 100} ]
|
145
|
+
# else
|
146
|
+
# [ $success, {:files_remaining => 10}]
|
147
|
+
# end
|
148
|
+
# }
|
149
|
+
#
|
150
|
+
# @param agent_code [Block, Lambda, or Method] Code to process a task
|
151
|
+
# @yieldparam Task [Hash]
|
152
|
+
# @yieldreturn [Boolean, Hash] success, (optional) hash of fields to update and values to update on the task
|
153
|
+
def process!(&agent_code)
|
154
|
+
(runnable, task) = register()
|
155
|
+
return unless runnable
|
156
|
+
(success, update) = agent_code.call(task)
|
157
|
+
@log[:tasks_processed] += 1
|
158
|
+
if success
|
159
|
+
complete_task(task, update)
|
160
|
+
else
|
161
|
+
fail_task(task, update)
|
162
|
+
end
|
163
|
+
return
|
164
|
+
end
|
165
|
+
|
166
|
+
# Iteratively runs process! on the supplied Block, then sleeps :sleep_between
|
167
|
+
# between each attempt. Block should match the specifications of what can
|
168
|
+
# be passed to process! (see above).
|
169
|
+
#
|
170
|
+
# If @process_while is set to a Block, Lambda, or Method, then it is called after
|
171
|
+
# each task is processed, and passed the current @log. As long as the
|
172
|
+
# Block returns true, work! will continue to process. work! will stop processing
|
173
|
+
# tasks when the Block returns false.
|
174
|
+
#
|
175
|
+
# @example process 3 entries and then exit
|
176
|
+
# @agent.process_while = ->(log) {
|
177
|
+
# (log[:tasks_processed] < 3)
|
178
|
+
# }
|
179
|
+
# @agent.work! { |task_hash|
|
180
|
+
# #... do something with task_hash and return true of false just as in process!
|
181
|
+
# }
|
182
|
+
#
|
183
|
+
# @example process until errors are encountered and then exit
|
184
|
+
# @agent.process_while = ->(log) {
|
185
|
+
# not(log[:errors_encountered])
|
186
|
+
# }
|
187
|
+
# @agent.work! { |task_hash|
|
188
|
+
# #... do something with task_hash and return true of false just as in process!
|
189
|
+
# }
|
190
|
+
# $stderr.puts " #{ @agent.log[:errors_encountered ] } errors were encountered during work."
|
191
|
+
# @param agent_code [Block, Lambda, or Method] Code to process a task
|
192
|
+
# @yieldparam Task Hash
|
193
|
+
# @yieldreturn [Boolean, Hash] success, (optional) hash of fields to update and values to update on the task
|
194
|
+
def work!(&agent_code)
|
195
|
+
|
196
|
+
while (@process_while.call(@log))
|
197
|
+
process!(&agent_code)
|
198
|
+
sleep @sleep_between
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
202
|
+
# get A MONGO_DB[queue] Moped::Query, either for the specified query Hash, or, when
|
203
|
+
# query is nil, all that are currently ready for the @name. This can be used to
|
204
|
+
# scan through the tasks on the @queue to perform aggregation tasks:
|
205
|
+
# @example collecting information
|
206
|
+
# @agent->get_tasks({
|
207
|
+
# agent_name: @agent->name,
|
208
|
+
# error_encountered: true
|
209
|
+
# }).each do |task|
|
210
|
+
# $stderr.puts "ERROR:\n#{ task.inspect }\n"
|
211
|
+
# end
|
212
|
+
#
|
213
|
+
# @example update ready to true for tasks that need intervention before they can run
|
214
|
+
# @agent->get_tasks({
|
215
|
+
# agent_name: @agent->name,
|
216
|
+
# waiting_for_information: true
|
217
|
+
# }).each do |task|
|
218
|
+
# task.update('$set' => {ready: true, waiting_form_information: false})
|
219
|
+
# end
|
220
|
+
#
|
221
|
+
# @param query [Hash] (optional) any query to find tasks
|
222
|
+
# @return [Moped::Query]
|
223
|
+
def get_tasks(query = nil)
|
224
|
+
if query.nil?
|
225
|
+
return @db[@queue].find({agent_name: @name, ready: true})
|
226
|
+
else
|
227
|
+
return @db[@queue].find(query)
|
228
|
+
end
|
229
|
+
end
|
230
|
+
|
231
|
+
private
|
232
|
+
|
233
|
+
def register
|
234
|
+
task = get_tasks().first
|
235
|
+
unless task
|
236
|
+
$stderr.puts "there are no ready tasks for #{@name} in queue #{@queue}"
|
237
|
+
return false
|
238
|
+
end
|
239
|
+
|
240
|
+
hostname = Socket.gethostname
|
241
|
+
get_tasks({ _id: task[:_id] }).update('$set' => {ready: false, agent_host: "#{hostname}" })
|
242
|
+
return true, task
|
243
|
+
end
|
244
|
+
|
245
|
+
def complete_task(task, update = nil)
|
246
|
+
if update.nil?
|
247
|
+
update = {}
|
248
|
+
end
|
249
|
+
update[:complete] = true
|
250
|
+
update[:error_encountered] = false
|
251
|
+
get_tasks({ _id: task[:_id] }).update('$set' => update)
|
252
|
+
end
|
253
|
+
|
254
|
+
def fail_task(task, update = nil)
|
255
|
+
@log[:failed_tasks] += 1
|
256
|
+
if update.nil?
|
257
|
+
update = {}
|
258
|
+
end
|
259
|
+
update[:complete] = true
|
260
|
+
update[:error_encountered] = true
|
261
|
+
get_tasks({ _id: task[:_id] }).update('$set' => update)
|
262
|
+
end
|
263
|
+
|
264
|
+
def build_db
|
265
|
+
@db = Moped::Session.new([ ENV['MONGO_HOST'] ])
|
266
|
+
@db.use ENV['MONGO_DB']
|
267
|
+
end
|
268
|
+
|
269
|
+
end #MongoAgent::Agent
|
270
|
+
end #MongoAgent
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'moped'
|
2
|
+
|
3
|
+
module MongoAgent
|
4
|
+
|
5
|
+
# MongoAgent::Db is a class that is meant to be extended by MongoAgent classes. It
|
6
|
+
# stores shared code to instantiate and provide access to a MongoDB Document Store and
|
7
|
+
# Moped::Session object for use by the extending classes to access their MongoDB Document Store
|
8
|
+
#
|
9
|
+
# It depends on the following environment variables to configure its MongoDB connect:
|
10
|
+
# MONGO_HOST: host URL for the MongoDB, can be in any form that mongod itself can
|
11
|
+
# use, e.g. host:port, etc.
|
12
|
+
# MONGO_DB: the name of the Document store in the MongoDB to use for all activities
|
13
|
+
# will be created if does not exist
|
14
|
+
|
15
|
+
class Db
|
16
|
+
|
17
|
+
# This holds the Moped::Session object that can be used to query information from the MongoDB
|
18
|
+
# hosted by the MONGO_HOST environment variable
|
19
|
+
attr_reader :db
|
20
|
+
|
21
|
+
# This is for internal use by SpreadsheetAgent classes that extend SpreadsheetAgent::Db
|
22
|
+
def build_db
|
23
|
+
@db = Moped::Session.new([ ENV['MONGO_HOST'] ])
|
24
|
+
@db.use ENV['MONGO_DB']
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
data/lib/mongo_agent.rb
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'mongo_agent/agent'
|
2
|
+
require 'mongo_agent/error'
|
3
|
+
require 'socket'
|
4
|
+
|
5
|
+
# @note The license of this source is "MIT Licence"
|
6
|
+
# A Distributed Agent System using MongoDB
|
7
|
+
#
|
8
|
+
# MongoAgent is a framework for creating massively distributed pipelines
|
9
|
+
# across many different servers, each using the same MongoDB as a
|
10
|
+
# control panel. It is extensible, and flexible. It doesnt specify what
|
11
|
+
# goals any pipeline should be working towards, or which goals are prerequisites
|
12
|
+
# for other goals, but it does provide logic for easily defining these relationships
|
13
|
+
# based on your own needs. It does this by providing a subsumption architecture,
|
14
|
+
# whereby many small, highly focused agents are written to perform specific goals,
|
15
|
+
# and also know what resources they require to perform them. Agents can be coded to
|
16
|
+
# subsume other agents upon successful completion. In addition, it is
|
17
|
+
# designed from the beginning to support the creation of simple human-computational
|
18
|
+
# workflows.
|
19
|
+
#
|
20
|
+
# MongoAgent requires MongoDB and Moped
|
21
|
+
# @version 0.01
|
22
|
+
# @author Darin London Copyright 2014
|
23
|
+
module MongoAgent
|
24
|
+
end
|
@@ -0,0 +1,353 @@
|
|
1
|
+
require 'mongo_agent'
|
2
|
+
require 'moped'
|
3
|
+
require 'psych'
|
4
|
+
require 'socket'
|
5
|
+
|
6
|
+
RSpec.describe "MongoAgent::Agent" do
|
7
|
+
let (:test_agent_name) { 'test_agent' }
|
8
|
+
let (:test_agent_queue) { 'testqueue' }
|
9
|
+
let (:test_agent_sleep_between) { 10 }
|
10
|
+
let (:expected_default_sleep_between) { 5 }
|
11
|
+
let (:test_task_params) {[
|
12
|
+
'foo',
|
13
|
+
'bar',
|
14
|
+
'baz',
|
15
|
+
'bleb'
|
16
|
+
]}
|
17
|
+
let (:expected_hostname) { Socket.gethostname }
|
18
|
+
before (:each) do
|
19
|
+
@db = Moped::Session.new([ ENV['MONGO_HOST'] ])
|
20
|
+
@db.use ENV['MONGO_DB']
|
21
|
+
end
|
22
|
+
|
23
|
+
after (:each) do
|
24
|
+
@db.drop
|
25
|
+
end
|
26
|
+
|
27
|
+
context "initialization" do
|
28
|
+
it "throws MongoAgent::Error without an attributes Hash" do
|
29
|
+
expect {
|
30
|
+
MongoAgent::Agent.new
|
31
|
+
}.to raise_error { |error|
|
32
|
+
expect(error).to be_a(MongoAgent::Error)
|
33
|
+
}
|
34
|
+
end
|
35
|
+
|
36
|
+
it "throws MongoAgent::Error without attributes[:name]" do
|
37
|
+
expect {
|
38
|
+
MongoAgent::Agent.new({queue: test_agent_queue})
|
39
|
+
}.to raise_error { |error|
|
40
|
+
expect(error).to be_a(MongoAgent::Error)
|
41
|
+
}
|
42
|
+
end
|
43
|
+
|
44
|
+
it 'throws MongoAgent::Error without attributes[:queue]' do
|
45
|
+
expect {
|
46
|
+
MongoAgent::Agent.new({ name: test_agent_name })
|
47
|
+
}.to raise_error { |error|
|
48
|
+
expect(error).to be_a(MongoAgent::Error)
|
49
|
+
}
|
50
|
+
end
|
51
|
+
|
52
|
+
it 'creates MongoAgent::Agent with default sleep_between when provided a name and queue' do
|
53
|
+
expect {
|
54
|
+
@agent = MongoAgent::Agent.new({
|
55
|
+
name: test_agent_name,
|
56
|
+
queue: test_agent_queue
|
57
|
+
})
|
58
|
+
}.to_not raise_error
|
59
|
+
expect(@agent.sleep_between).to eq(expected_default_sleep_between)
|
60
|
+
end
|
61
|
+
|
62
|
+
it 'allows optional attributes[:sleep_between]' do
|
63
|
+
expect {
|
64
|
+
@agent = MongoAgent::Agent.new({
|
65
|
+
name: test_agent_name,
|
66
|
+
queue: test_agent_queue,
|
67
|
+
sleep_between: test_agent_sleep_between
|
68
|
+
})
|
69
|
+
}.to_not raise_error
|
70
|
+
expect(@agent.sleep_between).to eq(test_agent_sleep_between)
|
71
|
+
end
|
72
|
+
end #initialization
|
73
|
+
|
74
|
+
context "process!" do
|
75
|
+
subject {
|
76
|
+
MongoAgent::Agent.new({
|
77
|
+
name: test_agent_name,
|
78
|
+
queue: test_agent_queue
|
79
|
+
})
|
80
|
+
}
|
81
|
+
|
82
|
+
context "with no tasks in queue" do
|
83
|
+
it 'returns without processing any tasks' do
|
84
|
+
called = false
|
85
|
+
subject.process! {
|
86
|
+
called = true
|
87
|
+
true
|
88
|
+
}
|
89
|
+
expect(called).to eq(false)
|
90
|
+
expect(subject.log[:tasks_processed]).to eq(0)
|
91
|
+
expect(subject.log[:failed_tasks]).to eq(0)
|
92
|
+
end
|
93
|
+
end # with no tasks in queue
|
94
|
+
|
95
|
+
context "with no ready tasks in queue" do
|
96
|
+
before(:each) do
|
97
|
+
@db[test_agent_queue].insert( test_task_params.collect{|param|
|
98
|
+
{agent_name: test_agent_name, test_param: param}
|
99
|
+
})
|
100
|
+
end
|
101
|
+
|
102
|
+
it 'return without processing any tasks' do
|
103
|
+
called = false
|
104
|
+
subject.process! {
|
105
|
+
called = true
|
106
|
+
true
|
107
|
+
}
|
108
|
+
expect(called).to eq(false)
|
109
|
+
expect(subject.log[:tasks_processed]).to eq(0)
|
110
|
+
expect(subject.log[:failed_tasks]).to eq(0)
|
111
|
+
end
|
112
|
+
end # with no ready tasks in queue
|
113
|
+
|
114
|
+
context "with ready tasks" do
|
115
|
+
before (:each) do
|
116
|
+
@db[test_agent_queue].insert( test_task_params.collect{|param|
|
117
|
+
{agent_name: test_agent_name, ready: true, test_param: param}
|
118
|
+
})
|
119
|
+
end
|
120
|
+
|
121
|
+
context "success" do
|
122
|
+
context "default" do
|
123
|
+
it 'updates complete, agent_host but not :error_encountered' do
|
124
|
+
called = false
|
125
|
+
processed_param = nil
|
126
|
+
subject.process! { |task|
|
127
|
+
processed_param = task[:test_param]
|
128
|
+
called = true
|
129
|
+
true
|
130
|
+
}
|
131
|
+
expect(processed_param).to be
|
132
|
+
expect(test_task_params.include?(processed_param)).to eq(true)
|
133
|
+
expect(called).to eq(true)
|
134
|
+
expect(subject.log[:tasks_processed]).to eq(1)
|
135
|
+
expect(subject.log[:failed_tasks]).to eq(0)
|
136
|
+
processed_task = @db[test_agent_queue].find(
|
137
|
+
{agent_name: test_agent_name, test_param: processed_param}
|
138
|
+
).first
|
139
|
+
expect(processed_task).to be
|
140
|
+
expect(processed_task[:complete]).to eq(true)
|
141
|
+
expect(processed_task[:error_encountered]).to eq(false)
|
142
|
+
expect(processed_task[:ready]).to eq(false)
|
143
|
+
expect(processed_task[:agent_host]).to eq(expected_hostname)
|
144
|
+
end
|
145
|
+
end #default
|
146
|
+
|
147
|
+
context "with update" do
|
148
|
+
it 'updates complete, agent_host, and update params, but not error_encountered' do
|
149
|
+
called = false
|
150
|
+
processed_param = nil
|
151
|
+
subject.process! { |task|
|
152
|
+
processed_param = task[:test_param]
|
153
|
+
called = true
|
154
|
+
[true, {test_update: 'updated'}]
|
155
|
+
}
|
156
|
+
expect(processed_param).to be
|
157
|
+
expect(test_task_params.include?(processed_param)).to eq(true)
|
158
|
+
expect(called).to eq(true)
|
159
|
+
expect(subject.log[:tasks_processed]).to eq(1)
|
160
|
+
expect(subject.log[:failed_tasks]).to eq(0)
|
161
|
+
processed_task = @db[test_agent_queue].find(
|
162
|
+
{agent_name: test_agent_name, test_param: processed_param}
|
163
|
+
).first
|
164
|
+
expect(processed_task).to be
|
165
|
+
expect(processed_task[:complete]).to eq(true)
|
166
|
+
expect(processed_task[:error_encountered]).to eq(false)
|
167
|
+
expect(processed_task[:ready]).to eq(false)
|
168
|
+
expect(processed_task[:agent_host]).to eq(expected_hostname)
|
169
|
+
expect(processed_task[:test_update]).to be
|
170
|
+
expect(processed_task[:test_update]).to eq('updated')
|
171
|
+
end
|
172
|
+
end #with update
|
173
|
+
end #success
|
174
|
+
|
175
|
+
context "failure" do
|
176
|
+
context "default" do
|
177
|
+
it 'updates complete, agent_host, and error_encountered' do
|
178
|
+
called = false
|
179
|
+
processed_param = nil
|
180
|
+
subject.process! { |task|
|
181
|
+
processed_param = task[:test_param]
|
182
|
+
called = true
|
183
|
+
false
|
184
|
+
}
|
185
|
+
expect(processed_param).to be
|
186
|
+
expect(test_task_params.include?(processed_param)).to eq(true)
|
187
|
+
expect(called).to eq(true)
|
188
|
+
expect(subject.log[:tasks_processed]).to eq(1)
|
189
|
+
expect(subject.log[:failed_tasks]).to eq(1)
|
190
|
+
processed_task = @db[test_agent_queue].find(
|
191
|
+
{agent_name: test_agent_name, test_param: processed_param}
|
192
|
+
).first
|
193
|
+
expect(processed_task).to be
|
194
|
+
expect(processed_task[:complete]).to eq(true)
|
195
|
+
expect(processed_task[:error_encountered]).to eq(true)
|
196
|
+
expect(processed_task[:ready]).to eq(false)
|
197
|
+
expect(processed_task[:agent_host]).to eq(expected_hostname)
|
198
|
+
end
|
199
|
+
end #default
|
200
|
+
|
201
|
+
context "with update" do
|
202
|
+
it 'updates complete, agent_host, update params, and error_encountered' do
|
203
|
+
called = false
|
204
|
+
processed_param = nil
|
205
|
+
subject.process! { |task|
|
206
|
+
processed_param = task[:test_param]
|
207
|
+
called = true
|
208
|
+
[false, {test_update: 'updated'}]
|
209
|
+
}
|
210
|
+
expect(processed_param).to be
|
211
|
+
expect(test_task_params.include?(processed_param)).to eq(true)
|
212
|
+
expect(called).to eq(true)
|
213
|
+
expect(subject.log[:tasks_processed]).to eq(1)
|
214
|
+
expect(subject.log[:failed_tasks]).to eq(1)
|
215
|
+
processed_task = @db[test_agent_queue].find(
|
216
|
+
{agent_name: test_agent_name, test_param: processed_param}
|
217
|
+
).first
|
218
|
+
expect(processed_task).to be
|
219
|
+
expect(processed_task[:complete]).to eq(true)
|
220
|
+
expect(processed_task[:error_encountered]).to eq(true)
|
221
|
+
expect(processed_task[:ready]).to eq(false)
|
222
|
+
expect(processed_task[:agent_host]).to eq(expected_hostname)
|
223
|
+
expect(processed_task[:test_update]).to be
|
224
|
+
expect(processed_task[:test_update]).to eq('updated')
|
225
|
+
end
|
226
|
+
end #with update
|
227
|
+
end #failure
|
228
|
+
end # with ready tasks
|
229
|
+
end #process!
|
230
|
+
|
231
|
+
context "work!" do
|
232
|
+
subject {
|
233
|
+
MongoAgent::Agent.new({
|
234
|
+
name: test_agent_name,
|
235
|
+
queue: test_agent_queue
|
236
|
+
})
|
237
|
+
}
|
238
|
+
|
239
|
+
context "with no tasks in queue" do
|
240
|
+
before (:each) do
|
241
|
+
@attempts = 0
|
242
|
+
subject.process_while = -> (log) {
|
243
|
+
@attempts += 1
|
244
|
+
( @attempts < 4 )
|
245
|
+
}
|
246
|
+
end
|
247
|
+
|
248
|
+
it 'returns without processing any tasks' do
|
249
|
+
called = false
|
250
|
+
subject.work! {
|
251
|
+
called = true
|
252
|
+
true
|
253
|
+
}
|
254
|
+
expect(called).to eq(false)
|
255
|
+
expect(subject.log[:tasks_processed]).to eq(0)
|
256
|
+
expect(subject.log[:failed_tasks]).to eq(0)
|
257
|
+
end
|
258
|
+
end # with no ready tasks
|
259
|
+
|
260
|
+
context "with no ready tasks in queue" do
|
261
|
+
before (:each) do
|
262
|
+
@db[test_agent_queue].insert( test_task_params.collect{|param|
|
263
|
+
{agent_name: test_agent_name, test_param: param}
|
264
|
+
})
|
265
|
+
@attempts = 0
|
266
|
+
subject.process_while = -> (log) {
|
267
|
+
@attempts += 1
|
268
|
+
( @attempts < 4 )
|
269
|
+
}
|
270
|
+
end
|
271
|
+
|
272
|
+
it 'returns without processing any tasks' do
|
273
|
+
called = false
|
274
|
+
subject.work! {
|
275
|
+
called = true
|
276
|
+
true
|
277
|
+
}
|
278
|
+
expect(called).to eq(false)
|
279
|
+
expect(subject.log[:tasks_processed]).to eq(0)
|
280
|
+
expect(subject.log[:failed_tasks]).to eq(0)
|
281
|
+
end
|
282
|
+
end # with no ready tasks in queue
|
283
|
+
|
284
|
+
context "with ready tasks" do
|
285
|
+
before (:each) do
|
286
|
+
@db[test_agent_queue].insert( test_task_params.collect{|param|
|
287
|
+
{agent_name: test_agent_name, ready: true, test_param: param}
|
288
|
+
})
|
289
|
+
subject.process_while = -> (log) {
|
290
|
+
(log[:tasks_processed] < 2)
|
291
|
+
}
|
292
|
+
end
|
293
|
+
|
294
|
+
it "will process! until process_while returns false" do
|
295
|
+
called = 0
|
296
|
+
subject.work! {
|
297
|
+
called += 1
|
298
|
+
true
|
299
|
+
}
|
300
|
+
expect(called).to eq(2)
|
301
|
+
expect(subject.log[:tasks_processed] < 2).to eq(false)
|
302
|
+
expect(subject.log[:tasks_processed]).to eq(2)
|
303
|
+
expect(subject.log[:failed_tasks]).to eq(0)
|
304
|
+
end
|
305
|
+
end # with ready tasks
|
306
|
+
end #work
|
307
|
+
|
308
|
+
context "get_tasks" do
|
309
|
+
let (:error_document) {{
|
310
|
+
agent_name: test_agent_name,
|
311
|
+
test_param: 'finished',
|
312
|
+
agent_host: 'daaedcedddeaaf',
|
313
|
+
test_param: 'failed_entry',
|
314
|
+
complete: true,
|
315
|
+
ready: false,
|
316
|
+
errors_encounterd: true
|
317
|
+
}}
|
318
|
+
|
319
|
+
subject {
|
320
|
+
MongoAgent::Agent.new({
|
321
|
+
name: test_agent_name,
|
322
|
+
queue: test_agent_queue
|
323
|
+
})
|
324
|
+
}
|
325
|
+
|
326
|
+
before (:each) do
|
327
|
+
@db[test_agent_queue].insert( test_task_params.collect{|param|
|
328
|
+
{agent_name: test_agent_name, ready: true, test_param: param}
|
329
|
+
})
|
330
|
+
@db[test_agent_queue].insert(error_document)
|
331
|
+
end
|
332
|
+
|
333
|
+
it "returns all ready tasks in queue by default" do
|
334
|
+
tasks = subject.get_tasks
|
335
|
+
expect(tasks.count).to eq(test_task_params.count)
|
336
|
+
tasks.each do |task|
|
337
|
+
expect(task[:ready]).to eq(true)
|
338
|
+
end
|
339
|
+
end
|
340
|
+
|
341
|
+
it "return only tasks that match the query provided" do
|
342
|
+
tasks = subject.get_tasks({agent_name: test_agent_name, agent_host: error_document[:agent_host]})
|
343
|
+
expect(tasks.count).to eq(1)
|
344
|
+
returned_task = tasks.first
|
345
|
+
returned_task.keys.each do |key|
|
346
|
+
if error_document.keys.include? key
|
347
|
+
expect(returned_task[key]).to eq(error_document[key])
|
348
|
+
end
|
349
|
+
end
|
350
|
+
end
|
351
|
+
end #get_task
|
352
|
+
|
353
|
+
end #MongoAgentTest::Agent
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,87 @@
|
|
1
|
+
# This file was generated by the `rspec --init` command. Conventionally, all
|
2
|
+
# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
|
3
|
+
# The generated `.rspec` file contains `--require spec_helper` which will cause this
|
4
|
+
# file to always be loaded, without a need to explicitly require it in any files.
|
5
|
+
#
|
6
|
+
# Given that it is always loaded, you are encouraged to keep this file as
|
7
|
+
# light-weight as possible. Requiring heavyweight dependencies from this file
|
8
|
+
# will add to the boot time of your test suite on EVERY test run, even for an
|
9
|
+
# individual file that may not need all of that loaded. Instead, consider making
|
10
|
+
# a separate helper file that requires the additional dependencies and performs
|
11
|
+
# the additional setup, and require it from the spec files that actually need it.
|
12
|
+
#
|
13
|
+
# The `.rspec` file also contains a few flags that are not defaults but that
|
14
|
+
# users commonly want.
|
15
|
+
#
|
16
|
+
# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
|
17
|
+
RSpec.configure do |config|
|
18
|
+
# rspec-expectations config goes here. You can use an alternate
|
19
|
+
# assertion/expectation library such as wrong or the stdlib/minitest
|
20
|
+
# assertions if you prefer.
|
21
|
+
config.expect_with :rspec do |expectations|
|
22
|
+
# This option will default to `true` in RSpec 4. It makes the `description`
|
23
|
+
# and `failure_message` of custom matchers include text for helper methods
|
24
|
+
# defined using `chain`, e.g.:
|
25
|
+
# be_bigger_than(2).and_smaller_than(4).description
|
26
|
+
# # => "be bigger than 2 and smaller than 4"
|
27
|
+
# ...rather than:
|
28
|
+
# # => "be bigger than 2"
|
29
|
+
expectations.include_chain_clauses_in_custom_matcher_descriptions = true
|
30
|
+
end
|
31
|
+
|
32
|
+
# rspec-mocks config goes here. You can use an alternate test double
|
33
|
+
# library (such as bogus or mocha) by changing the `mock_with` option here.
|
34
|
+
config.mock_with :rspec do |mocks|
|
35
|
+
# Prevents you from mocking or stubbing a method that does not exist on
|
36
|
+
# a real object. This is generally recommended, and will default to
|
37
|
+
# `true` in RSpec 4.
|
38
|
+
mocks.verify_partial_doubles = true
|
39
|
+
end
|
40
|
+
|
41
|
+
# The settings below are suggested to provide a good initial experience
|
42
|
+
# with RSpec, but feel free to customize to your heart's content.
|
43
|
+
# These two settings work together to allow you to limit a spec run
|
44
|
+
# to individual examples or groups you care about by tagging them with
|
45
|
+
# `:focus` metadata. When nothing is tagged with `:focus`, all examples
|
46
|
+
# get run.
|
47
|
+
config.filter_run :focus
|
48
|
+
config.run_all_when_everything_filtered = true
|
49
|
+
|
50
|
+
# Limits the available syntax to the non-monkey patched syntax that is recommended.
|
51
|
+
# For more details, see:
|
52
|
+
# - http://myronmars.to/n/dev-blog/2012/06/rspecs-new-expectation-syntax
|
53
|
+
# - http://teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
|
54
|
+
# - http://myronmars.to/n/dev-blog/2014/05/notable-changes-in-rspec-3#new__config_option_to_disable_rspeccore_monkey_patching
|
55
|
+
config.disable_monkey_patching!
|
56
|
+
|
57
|
+
# This setting enables warnings. It's recommended, but in some cases may
|
58
|
+
# be too noisy due to issues in dependencies.
|
59
|
+
config.warnings = true
|
60
|
+
|
61
|
+
# Many RSpec users commonly either run the entire suite or an individual
|
62
|
+
# file, and it's useful to allow more verbose output when running an
|
63
|
+
# individual spec file.
|
64
|
+
if config.files_to_run.one?
|
65
|
+
# Use the documentation formatter for detailed output,
|
66
|
+
# unless a formatter has already been configured
|
67
|
+
# (e.g. via a command-line flag).
|
68
|
+
config.default_formatter = 'doc'
|
69
|
+
end
|
70
|
+
|
71
|
+
# Print the 10 slowest examples and example groups at the
|
72
|
+
# end of the spec run, to help surface which specs are running
|
73
|
+
# particularly slow.
|
74
|
+
config.profile_examples = 10
|
75
|
+
|
76
|
+
# Run specs in random order to surface order dependencies. If you find an
|
77
|
+
# order dependency and want to debug it, you can fix the order by providing
|
78
|
+
# the seed, which is printed after each run.
|
79
|
+
# --seed 1234
|
80
|
+
config.order = :random
|
81
|
+
|
82
|
+
# Seed global randomization in this process using the `--seed` CLI option.
|
83
|
+
# Setting this allows you to use `--seed` to deterministically reproduce
|
84
|
+
# test failures related to randomization by passing the same `--seed` value
|
85
|
+
# as the one that triggered the failure.
|
86
|
+
Kernel.srand config.seed
|
87
|
+
end
|
metadata
ADDED
@@ -0,0 +1,62 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: mongo_agent
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Darin London
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-11-20 00:00:00.000000000 Z
|
12
|
+
dependencies: []
|
13
|
+
description: |
|
14
|
+
MongoAgent is a framework for creating distributed pipelines across many
|
15
|
+
different servers. It is extensible, and flexible. It does not specify what goals
|
16
|
+
should be processed. It simply provides the foundation for using a MongoDB as
|
17
|
+
a messaging queue between many different agents processing tasks defined in the
|
18
|
+
same queue. It is designed from the beginning to support the creation of simple
|
19
|
+
human-computational workflows.
|
20
|
+
email: darin.london@duke.edu
|
21
|
+
executables: []
|
22
|
+
extensions: []
|
23
|
+
extra_rdoc_files: []
|
24
|
+
files:
|
25
|
+
- ".rspec"
|
26
|
+
- Gemfile
|
27
|
+
- Rakefile
|
28
|
+
- lib/mongo_agent.rb
|
29
|
+
- lib/mongo_agent/agent.rb
|
30
|
+
- lib/mongo_agent/db.rb
|
31
|
+
- lib/mongo_agent/error.rb
|
32
|
+
- spec/mongo_agent_spec.rb
|
33
|
+
- spec/spec_helper.rb
|
34
|
+
homepage: http://rubygems.org/gems/mongo_agent
|
35
|
+
licenses:
|
36
|
+
- MIT
|
37
|
+
metadata: {}
|
38
|
+
post_install_message:
|
39
|
+
rdoc_options: []
|
40
|
+
require_paths:
|
41
|
+
- lib
|
42
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - ">="
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '0'
|
47
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
48
|
+
requirements:
|
49
|
+
- - ">="
|
50
|
+
- !ruby/object:Gem::Version
|
51
|
+
version: '0'
|
52
|
+
requirements: []
|
53
|
+
rubyforge_project:
|
54
|
+
rubygems_version: 2.2.2
|
55
|
+
signing_key:
|
56
|
+
specification_version: 4
|
57
|
+
summary: MongoAgent is a framework for creating distributed pipelines across many
|
58
|
+
different servers, each using the same MongoDB as a control panel.
|
59
|
+
test_files:
|
60
|
+
- spec/mongo_agent_spec.rb
|
61
|
+
- spec/spec_helper.rb
|
62
|
+
has_rdoc:
|