spreadsheet_agent 0.0.1 → 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- data/config/test.config.yml +13 -0
- data/lib/spreadsheet_agent.rb +6 -390
- data/lib/spreadsheet_agent/agent.rb +408 -0
- data/lib/spreadsheet_agent/db.rb +0 -3
- data/lib/spreadsheet_agent/error.rb +0 -3
- data/test/spreadsheet_agent_db_test.rb +4 -1
- data/test/spreadsheet_agent_runner_test.rb +1 -1
- data/test/spreadsheet_agent_test.rb +1 -1
- metadata +4 -1
data/lib/spreadsheet_agent.rb
CHANGED
@@ -1,16 +1,15 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
1
|
+
require 'spreadsheet_agent/agent'
|
2
|
+
require 'spreadsheet_agent/error'
|
4
3
|
require 'spreadsheet_agent/db'
|
4
|
+
require 'spreadsheet_agent/runner'
|
5
5
|
require 'socket'
|
6
6
|
require 'open3'
|
7
7
|
require 'capture_io'
|
8
8
|
require 'mail'
|
9
9
|
|
10
|
+
# @note The license of this source is "MIT Licence"
|
10
11
|
# A Distributed Agent System using Google Spreadsheets
|
11
12
|
#
|
12
|
-
# Version 0.01
|
13
|
-
#
|
14
13
|
# SpreadsheetAgent is a framework for creating massively distributed pipelines
|
15
14
|
# across many different servers, each using the same google spreadsheet as a
|
16
15
|
# control panel. It is extensible, and flexible. It doesnt specify what
|
@@ -29,390 +28,7 @@ require 'mail'
|
|
29
28
|
# * You can define any fields necessary, but you must specify a 'ready' and a 'complete' field
|
30
29
|
# * You must define at least 1 key field, and the key field must be specified as required in the :config (see SpreadsheetAgent::Db)
|
31
30
|
# * You should then define fields named for agent_bin/#{ field_name }_agent.rb for each agent that you plan to deploy in your pipeline
|
32
|
-
#
|
31
|
+
# @version 0.01
|
32
|
+
# @author Darin London Copyright 2013
|
33
33
|
module SpreadsheetAgent
|
34
|
-
|
35
|
-
# SpreadsheetAgent::Agent is designed to make it easy to create a single task which connects to
|
36
|
-
# a field within a record on a page within the configured SpreadsheetAgent compatible Google Spreadsheet,
|
37
|
-
# runs, and reports whether the job completed or ended in error. An agent can be configured to only run
|
38
|
-
# when certain prerequisite fields have completed. The data in these fields can be filled in by other
|
39
|
-
# SpreadsheetAgent::Agents, SpreadsheetAgent::Runners, or humans. Compute node configuration is available
|
40
|
-
# to prevent the agent from running more than a certain number of instances of itself, or not run if certain
|
41
|
-
# other agents or processes are running on the node. Finally, an agent can be configured to subsume another
|
42
|
-
# agent, and fill in the completion field for that agent in addition to its own when it completes successfully.
|
43
|
-
#
|
44
|
-
# extends SpreadsheetAgent::Db
|
45
|
-
class Agent < SpreadsheetAgent::Db
|
46
|
-
|
47
|
-
# The name of the field in the page to which the agent should report status
|
48
|
-
attr_accessor :agent_name
|
49
|
-
|
50
|
-
# The name of the Page on the Google Spreadsheet that contains the record to be worked on by the agent
|
51
|
-
attr_accessor :page_name
|
52
|
-
|
53
|
-
# hash of key-value pairs. The keys are defined in config/agent.conf.yml. The values
|
54
|
-
# specify the values for those fields in the record on the page for which the agent is running.
|
55
|
-
# All keys configured as 'required: 1' in config/agent.conf.yml must be included in the keys hash
|
56
|
-
attr_accessor :keys
|
57
|
-
|
58
|
-
# Boolean. When true, the agent code will print verbosely to STDERR. When false, and the process!
|
59
|
-
# returns a failure status, the agent will email all stdout and stderr to the email specified in the
|
60
|
-
# :config send_to value
|
61
|
-
attr_accessor :debug
|
62
|
-
|
63
|
-
# Optional array of prerequisite fields that must contain a 1 in them for the record on the page before
|
64
|
-
# the agent will attempt to run
|
65
|
-
attr_accessor :prerequisites
|
66
|
-
|
67
|
-
# Optional integer. This works on Linux with ps. The agent will not attempt to run if there are
|
68
|
-
# max_selves instances running
|
69
|
-
attr_accessor :max_selves
|
70
|
-
|
71
|
-
# Hash of process_name to number of max_instances. This works on Linux with ps. If the agent detects
|
72
|
-
# the specified number of max_instances of the given process (based on a line match), it will not
|
73
|
-
# attempt to run
|
74
|
-
attr_accessor :conflicts_with
|
75
|
-
|
76
|
-
# Array of fields on the record which this agent subsumes. If the agent completes successfully these
|
77
|
-
# fields will be updated with a 1 in addition to the field for the agent
|
78
|
-
attr_accessor :subsumes
|
79
|
-
|
80
|
-
# Readonly access to the GoogleDrive::Worksheet that is being access by the agent.
|
81
|
-
attr_reader :worksheet
|
82
|
-
|
83
|
-
# create a new SpreadsheetAgent::Agent with the following:
|
84
|
-
# == required configuration parameters:
|
85
|
-
# * agent_name
|
86
|
-
# * page_name
|
87
|
-
# * keys
|
88
|
-
#
|
89
|
-
# == optional parameters:
|
90
|
-
# * config_file: (see SpreadsheetAgent::DB)
|
91
|
-
# * debug
|
92
|
-
# * prerequisites
|
93
|
-
# * max_selves
|
94
|
-
# * conflicts_with
|
95
|
-
# * subsumes
|
96
|
-
#
|
97
|
-
def initialize(attributes)
|
98
|
-
@agent_name = attributes[:agent_name]
|
99
|
-
@page_name = attributes[:page_name]
|
100
|
-
@keys = attributes[:keys].clone
|
101
|
-
unless @agent_name && @page_name && @keys
|
102
|
-
raise SpreadsheetAgentError, "agent_name, page_name, and keys attributes are required!"
|
103
|
-
end
|
104
|
-
@config_file = attributes[:config_file]
|
105
|
-
build_db()
|
106
|
-
|
107
|
-
@worksheet = @db.worksheet_by_title(@page_name)
|
108
|
-
@debug = attributes[:debug]
|
109
|
-
if attributes[:prerequisites]
|
110
|
-
@prerequisites = attributes[:prerequisites].clone
|
111
|
-
end
|
112
|
-
|
113
|
-
@max_selves = attributes[:max_selves]
|
114
|
-
if attributes[:conflicts_with]
|
115
|
-
@conflicts_with = attributes[:conflicts_with].clone
|
116
|
-
end
|
117
|
-
if attributes[:subsumes]
|
118
|
-
@subsumes = attributes[:subsumes].clone
|
119
|
-
end
|
120
|
-
end
|
121
|
-
|
122
|
-
# If the agent does not have any conflicting processes (max_selves or conflicts_with)
|
123
|
-
# and if the entry is ready (field 'ready' has a 1), and all prerequisite fields have a 1,
|
124
|
-
# gets the GoogleDrive::List record, and passes it to the supplied agent_code PROC as argument.
|
125
|
-
# This PROC must return a required boolean field indicating success or failure, and an optional
|
126
|
-
# hash of key - value fields that will be updated on the GoogleDrive::List record. Note, the updates
|
127
|
-
# are made regardless of the value of success. In fact, the agent can be configured to update
|
128
|
-
# different fields based on success or failure. Also, note that any value can be stored in the
|
129
|
-
# hash. This allows the agent to communicate any useful information to the google spreadsheet for other
|
130
|
-
# agents (SpreadsheetAgent::Agent, SpreadsheetAgent::Runner, or human) to use. The PROC must try at all
|
131
|
-
# costs to avoid terminating. If an error is encountered, it should return false for the success field
|
132
|
-
# to signal that the process failed. If no errors are encountered it should return true for the success
|
133
|
-
# field.
|
134
|
-
#
|
135
|
-
# Exits successfully, enters a 1 in the agent_name field
|
136
|
-
# $agent->process! do |entry|
|
137
|
-
# true
|
138
|
-
# end
|
139
|
-
#
|
140
|
-
# Same, but also updates the 'notice' field in the record along with the 1 in the agent_name field
|
141
|
-
# $agent->process! do |entry|
|
142
|
-
# [true, {:notice => 'There were 30 files processed'}]
|
143
|
-
# end
|
144
|
-
#
|
145
|
-
# Fails, enters f:#{hostname} in the agent_name field
|
146
|
-
# $agent->process! do |entry|
|
147
|
-
# false
|
148
|
-
#
|
149
|
-
# Same, but also updates the 'notice' field in the record along with the failure notice
|
150
|
-
# $agent->process! do |entry|
|
151
|
-
# [false, {:notice => 'There were 10 files left to process!' }]
|
152
|
-
# end
|
153
|
-
#
|
154
|
-
# This agent passes different parameters based on success or failure
|
155
|
-
# $agent->process! do |entry|
|
156
|
-
# if $success
|
157
|
-
# true
|
158
|
-
# else
|
159
|
-
# [ false, {:notice => 'there were 10 remaining files'}]
|
160
|
-
# end
|
161
|
-
# end
|
162
|
-
#
|
163
|
-
def process!(&agent_code)
|
164
|
-
@worksheet.reload
|
165
|
-
no_problems = true
|
166
|
-
capture_output = nil
|
167
|
-
unless @debug
|
168
|
-
capture_output = CaptureIO.new
|
169
|
-
capture_output.start
|
170
|
-
end
|
171
|
-
|
172
|
-
begin
|
173
|
-
return true if has_conflicts()
|
174
|
-
(runnable, entry) = run_entry()
|
175
|
-
return false unless entry
|
176
|
-
return true unless runnable
|
177
|
-
|
178
|
-
success, update_entry = agent_code.call(entry)
|
179
|
-
if success
|
180
|
-
complete_entry(update_entry)
|
181
|
-
else
|
182
|
-
fail_entry(update_entry)
|
183
|
-
end
|
184
|
-
rescue
|
185
|
-
$stderr.puts "#{ $! }"
|
186
|
-
no_problems = false
|
187
|
-
end
|
188
|
-
unless capture_output.nil?
|
189
|
-
if no_problems
|
190
|
-
capture_output.stop
|
191
|
-
else
|
192
|
-
mail_error(capture_output.stop)
|
193
|
-
end
|
194
|
-
end
|
195
|
-
return no_problems
|
196
|
-
end
|
197
|
-
|
198
|
-
# Returns the GoogleDrive::List object for the specified keys
|
199
|
-
def get_entry
|
200
|
-
this_entry = nil
|
201
|
-
if @worksheet
|
202
|
-
@worksheet.list.each do |this_row|
|
203
|
-
keep_row = true
|
204
|
-
|
205
|
-
@config['key_fields'].keys.reject { |key_field|
|
206
|
-
!(@config['key_fields'][key_field]["required"]) && !(@keys[key_field])
|
207
|
-
}.each do |key|
|
208
|
-
break unless keep_row
|
209
|
-
keep_row = (this_row[key] == @keys[key])
|
210
|
-
end
|
211
|
-
|
212
|
-
if keep_row
|
213
|
-
return this_row
|
214
|
-
end
|
215
|
-
end
|
216
|
-
end
|
217
|
-
end
|
218
|
-
|
219
|
-
private
|
220
|
-
|
221
|
-
def has_conflicts
|
222
|
-
return unless (@max_selves || @conflicts_with) # nothing conflicts here
|
223
|
-
|
224
|
-
running_conflicters = {}
|
225
|
-
self_name = File.basename $0
|
226
|
-
|
227
|
-
begin
|
228
|
-
conflicting_in = Open3.popen3('ps','-eo','pid,command')[1]
|
229
|
-
conflicting_in.lines.each do |line|
|
230
|
-
unless(
|
231
|
-
(line.match(/emacs\s+|vim*\s+|pico\s+/)) ||
|
232
|
-
(line.match("#{ $$ }"))
|
233
|
-
)
|
234
|
-
if @max_selves && line.match(self_name)
|
235
|
-
if running_conflicters[@agent_name].nil?
|
236
|
-
running_conflicters[@agent_name] = 1
|
237
|
-
else
|
238
|
-
running_conflicters[@agent_name] += 1
|
239
|
-
end
|
240
|
-
|
241
|
-
if running_conflicters[@agent_name] == @max_selves
|
242
|
-
$stderr.puts "max_selves limit reached" if @debug
|
243
|
-
conflicting_in.close
|
244
|
-
return true
|
245
|
-
end
|
246
|
-
end
|
247
|
-
|
248
|
-
if @conflicts_with
|
249
|
-
@conflicts_with.keys.each do |conflicter|
|
250
|
-
if line.match(conflicter)
|
251
|
-
if running_conflicters[conflicter].nil?
|
252
|
-
running_conflicters[conflicter] = 1
|
253
|
-
else
|
254
|
-
running_conflicters[conflicter] += 1
|
255
|
-
end
|
256
|
-
if running_conflicters[conflicter] >= @conflicts_with[conflicter]
|
257
|
-
$stderr.puts "conflicts with #{ conflicter }" if @debug
|
258
|
-
conflicting_in.close
|
259
|
-
return true
|
260
|
-
end
|
261
|
-
end
|
262
|
-
end
|
263
|
-
end
|
264
|
-
end
|
265
|
-
end
|
266
|
-
conflicting_in.close
|
267
|
-
return false
|
268
|
-
|
269
|
-
rescue
|
270
|
-
$stderr.puts "Couldnt check conflicts #{ $! }" if @debug
|
271
|
-
return true
|
272
|
-
end
|
273
|
-
|
274
|
-
end
|
275
|
-
|
276
|
-
# this call initiates a race resistant attempt to make sure that there is only 1
|
277
|
-
# clear 'winner' among N potential agents attempting to run the same goal on the
|
278
|
-
# same spreadsheet agent's cell
|
279
|
-
def run_entry
|
280
|
-
entry = get_entry()
|
281
|
-
output = '';
|
282
|
-
@keys.keys.select { |k| @config['key_fields'][k] && @keys[k] }.each do |key|
|
283
|
-
output += [ key, @keys[key] ].join(' ') + " "
|
284
|
-
end
|
285
|
-
|
286
|
-
unless entry
|
287
|
-
$stderr.puts "#{ output } is not supported on #{ @page_name }" if @debug
|
288
|
-
return
|
289
|
-
end
|
290
|
-
|
291
|
-
unless entry['ready'] == "1"
|
292
|
-
$stderr.puts "#{ output } is not ready to run #{ @agent_name }" if @debug
|
293
|
-
return false, entry
|
294
|
-
end
|
295
|
-
|
296
|
-
if entry['complete'] == "1"
|
297
|
-
$stderr.puts "All goals are completed for #{ output }" if @debug
|
298
|
-
return false, entry
|
299
|
-
end
|
300
|
-
|
301
|
-
if entry[@agent_name]
|
302
|
-
(status, running_hostname) = entry[@agent_name].split(':')
|
303
|
-
|
304
|
-
case status
|
305
|
-
when 'r'
|
306
|
-
$stderr.puts " #{ output } is already running #{ @agent_name } on #{ running_hostname }" if @debug
|
307
|
-
return false, entry
|
308
|
-
|
309
|
-
when "1"
|
310
|
-
$stderr.puts " #{ output } has already run #{ @agent_name }" if @debug
|
311
|
-
return false, entry
|
312
|
-
|
313
|
-
when 'F'
|
314
|
-
$stderr.puts " #{ output } has already Failed #{ @agent_name }" if @debug
|
315
|
-
return false, entry
|
316
|
-
end
|
317
|
-
end
|
318
|
-
|
319
|
-
if @prerequisites
|
320
|
-
@prerequisites.each do |prereq_field|
|
321
|
-
unless entry[prereq_field] == "1"
|
322
|
-
$stderr.puts " #{ output } has not finished #{ prereq_field }" if @debug
|
323
|
-
return false, entry
|
324
|
-
end
|
325
|
-
end
|
326
|
-
end
|
327
|
-
|
328
|
-
# first attempt to set the hostname of the machine as the value of the agent
|
329
|
-
hostname = Socket.gethostname;
|
330
|
-
begin
|
331
|
-
entry.update @agent_name => "r:#{ hostname }"
|
332
|
-
@worksheet.save
|
333
|
-
|
334
|
-
rescue GoogleDrive::Error
|
335
|
-
# this is a collision, which is to be treated as if it is not runnable
|
336
|
-
$stderr.puts " #{ output } lost #{ @agent_name } on #{hostname}" if @debug
|
337
|
-
return false, entry
|
338
|
-
end
|
339
|
-
|
340
|
-
sleep 3
|
341
|
-
begin
|
342
|
-
@worksheet.reload
|
343
|
-
rescue GoogleDrive::Error
|
344
|
-
# this is a collision, which is to be treated as if it is not runnable
|
345
|
-
$stderr.puts " #{ output } lost #{ @agent_name } on #{hostname}" if @debug
|
346
|
-
return false, entry
|
347
|
-
end
|
348
|
-
|
349
|
-
check = entry[@agent_name]
|
350
|
-
(status, running_hostname) = check.split(':')
|
351
|
-
if hostname == running_hostname
|
352
|
-
return true, entry
|
353
|
-
end
|
354
|
-
$stderr.puts " #{ output } lost #{ @agent_name } on #{hostname}" if @debug
|
355
|
-
return false, entry
|
356
|
-
end
|
357
|
-
|
358
|
-
def complete_entry(update_entry)
|
359
|
-
if update_entry.nil?
|
360
|
-
update_entry = {}
|
361
|
-
end
|
362
|
-
|
363
|
-
if @subsumes && @subsumes.length > 0
|
364
|
-
@subsumes.each do |subsumed_agent|
|
365
|
-
update_entry[subsumed_agent] = 1
|
366
|
-
end
|
367
|
-
end
|
368
|
-
|
369
|
-
update_entry[@agent_name] = 1
|
370
|
-
entry = get_entry()
|
371
|
-
entry.update update_entry
|
372
|
-
@worksheet.save
|
373
|
-
end
|
374
|
-
|
375
|
-
def fail_entry(update_entry)
|
376
|
-
if update_entry.nil?
|
377
|
-
update_entry = { }
|
378
|
-
end
|
379
|
-
hostname = Socket.gethostname
|
380
|
-
update_entry[@agent_name] = "F:#{ hostname }"
|
381
|
-
entry = get_entry()
|
382
|
-
entry.update update_entry
|
383
|
-
@worksheet.save
|
384
|
-
end
|
385
|
-
|
386
|
-
def mail_error(error_message)
|
387
|
-
output = ''
|
388
|
-
@keys.keys.each do |key|
|
389
|
-
output += [key, @keys[key] ].join(' ') + " "
|
390
|
-
end
|
391
|
-
|
392
|
-
prefix = [Socket.gethostname, output, @agent_name ].join(' ')
|
393
|
-
begin
|
394
|
-
Mail.defaults do
|
395
|
-
delivery_method :smtp, {
|
396
|
-
:address => "smtp.gmail.com",
|
397
|
-
:port => 587,
|
398
|
-
:domain => Socket.gethostname,
|
399
|
-
:user_name => @config['guser'],
|
400
|
-
:password => @config['gpass'],
|
401
|
-
:authentication => 'plain',
|
402
|
-
:enable_starttls_auto => true }
|
403
|
-
end
|
404
|
-
|
405
|
-
mail = Mail.new do
|
406
|
-
from @config['reply_email']
|
407
|
-
to @config['send_to']
|
408
|
-
subject prefix
|
409
|
-
body error_message.to_s
|
410
|
-
end
|
411
|
-
|
412
|
-
mail.deliver!
|
413
|
-
rescue
|
414
|
-
#DO NOTHING
|
415
|
-
end
|
416
|
-
end
|
417
|
-
end
|
418
34
|
end
|
@@ -0,0 +1,408 @@
|
|
1
|
+
require 'spreadsheet_agent/db'
|
2
|
+
|
3
|
+
module SpreadsheetAgent
|
4
|
+
|
5
|
+
# @note The license of this source is "MIT Licence"
|
6
|
+
# SpreadsheetAgent::Agent is designed to make it easy to create a single task which connects to
|
7
|
+
# a field within a record on a page within the configured SpreadsheetAgent compatible Google Spreadsheet,
|
8
|
+
# runs supplied code, and reports whether the job completed or ended in error. An agent can be configured
|
9
|
+
# to only run when certain prerequisite fields have completed. The data in these fields can be filled in by
|
10
|
+
# other SpreadsheetAgent::Agents, SpreadsheetAgent::Runners, or humans. Compute node configuration is available
|
11
|
+
# to prevent the agent from running more than a certain number of instances of itself, or not run if certain
|
12
|
+
# other agents or processes are running on the node. Finally, an agent can be configured to subsume another
|
13
|
+
# agent, and fill in the completion field for that agent in addition to its own when it completes successfully.
|
14
|
+
# @author Darin London Copyright 2013
|
15
|
+
class Agent < SpreadsheetAgent::Db
|
16
|
+
|
17
|
+
# The name of the field in the page to which the agent should report status
|
18
|
+
# @return [String]
|
19
|
+
attr_accessor :agent_name
|
20
|
+
|
21
|
+
# The name of the Page on the Google Spreadsheet that contains the record to be worked on by the agent
|
22
|
+
# @return [String]
|
23
|
+
attr_accessor :page_name
|
24
|
+
|
25
|
+
# Hash used to find the entry on the Google Spreadsheet Worksheet
|
26
|
+
# Keys are defined in config/agent.conf.yml.
|
27
|
+
# All keys configured as 'required: 1' must be included in the keys hash.
|
28
|
+
# Values specify values for those fields in the record on the page for which the agent is running.
|
29
|
+
# @return [Hash]
|
30
|
+
attr_accessor :keys
|
31
|
+
|
32
|
+
# Specify whether to print debug information (default false).
|
33
|
+
# When true, the agent code will print verbosely to STDERR. When false, and the process!
|
34
|
+
# returns a failure status, the agent will email all stdout and stderr to the email specified in the
|
35
|
+
# :config send_to value
|
36
|
+
# @return [Boolean]
|
37
|
+
attr_accessor :debug
|
38
|
+
|
39
|
+
# Optional array of prerequisites.
|
40
|
+
# If supplied, each entry is treated as a field name on the Google Worksheet which must contain a 1 in it for the record on the page before
|
41
|
+
# this agent will attempt to run.
|
42
|
+
# @return [Array]
|
43
|
+
attr_accessor :prerequisites
|
44
|
+
|
45
|
+
# @note This works on Linux with ps.
|
46
|
+
# Maximum number of instances of this agent to run on any particular server.
|
47
|
+
# If specified, newly instantiated agents will not attempt to run process! if
|
48
|
+
# there are max_selves instances already running on the same server. If not
|
49
|
+
# specified, all instances will attempt to run.
|
50
|
+
# @return [Integer]
|
51
|
+
attr_accessor :max_selves
|
52
|
+
|
53
|
+
# @note This works on Linux with ps.
|
54
|
+
# List of other processes, and the maximum number of running instances of the process that are allowed before this agent should avoid running on the given server.
|
55
|
+
# Hash of process_name => number of max_instances. If specified, each key is treated as a process name in ps. If the agent detects the specified number of
|
56
|
+
# max_instances of the given process (based on a line match), it will not attempt to run. If not specified, it will run regardless of the other processes
|
57
|
+
# already running on a server.
|
58
|
+
# @return [Hash]
|
59
|
+
attr_accessor :conflicts_with
|
60
|
+
|
61
|
+
# List of fields (agent or otherwise) that this agent should also complete when it completes successfully.
|
62
|
+
# Each entry is treated as a fields on the record which this agent subsumes. If the agent completes successfully these
|
63
|
+
# fields will be updated with a 1 in addition to the field for the agent.
|
64
|
+
# @return [Array]
|
65
|
+
attr_accessor :subsumes
|
66
|
+
|
67
|
+
# The GoogleDrive::Worksheet[http://rubydoc.info/gems/google_drive/0.3.6/GoogleDrive/Worksheet] that is being access by the agent.
|
68
|
+
# @return [GoogleDrive::Worksheet]
|
69
|
+
attr_reader :worksheet
|
70
|
+
|
71
|
+
# create a new SpreadsheetAgent::Agent
|
72
|
+
# @param attributes [Hash] keys are the attribute names, values are their values
|
73
|
+
# @option attributes [String] agent_name REQUIRED
|
74
|
+
# @option attributes [String] page_name REQUIRED
|
75
|
+
# @option attributes [Hash] keys REQUIRED
|
76
|
+
# @option attributes [String] config_file (see SpreadsheetAgent::DB)
|
77
|
+
# @option attributes [Boolean] debug
|
78
|
+
# @option attributes [Array] prerequisites
|
79
|
+
# @option attributes [Integer] max_selves
|
80
|
+
# @option attributes [Hash] conflicts_with
|
81
|
+
# @option attributes [Array] subsumes
|
82
|
+
def initialize(attributes)
|
83
|
+
@agent_name = attributes[:agent_name]
|
84
|
+
@page_name = attributes[:page_name]
|
85
|
+
@keys = attributes[:keys].clone
|
86
|
+
unless @agent_name && @page_name && @keys
|
87
|
+
raise SpreadsheetAgentError, "agent_name, page_name, and keys attributes are required!"
|
88
|
+
end
|
89
|
+
@config_file = attributes[:config_file]
|
90
|
+
build_db()
|
91
|
+
|
92
|
+
@worksheet = @db.worksheet_by_title(@page_name)
|
93
|
+
@debug = attributes[:debug]
|
94
|
+
if attributes[:prerequisites]
|
95
|
+
@prerequisites = attributes[:prerequisites].clone
|
96
|
+
end
|
97
|
+
|
98
|
+
@max_selves = attributes[:max_selves]
|
99
|
+
if attributes[:conflicts_with]
|
100
|
+
@conflicts_with = attributes[:conflicts_with].clone
|
101
|
+
end
|
102
|
+
if attributes[:subsumes]
|
103
|
+
@subsumes = attributes[:subsumes].clone
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
# If the agent does not have any conflicting processes (max_selves or conflicts_with)
|
108
|
+
# and if the entry field 'ready' has a 1, and any supplied prerequisite fields have a 1,
|
109
|
+
# gets the GoogleDrive::List[http://rubydoc.info/gems/google_drive/0.3.6/GoogleDrive/List] record, and
|
110
|
+
# passes it to the supplied Proc. This PROC must return a required boolean field indicating success or failure,
|
111
|
+
# and an optional hash of key - value fields that will be updated on the GoogleDrive::List record. Note, the updates
|
112
|
+
# are made regardless of the value of success. In fact, the agent can be configured to update
|
113
|
+
# different fields based on success or failure. Also, note that any value can be stored in the
|
114
|
+
# hash. This allows the agent to communicate any useful information to the google spreadsheet for other
|
115
|
+
# agents (SpreadsheetAgent::Agent, SpreadsheetAgent::Runner, or human) to use. The Proc must try at all
|
116
|
+
# costs to avoid terminating. If an error is encountered, it should return false for the success field
|
117
|
+
# to signal that the process failed. If no errors are encountered it should return true for the success
|
118
|
+
# field.
|
119
|
+
#
|
120
|
+
# @example Exit successfully, enters a 1 in the agent_name field
|
121
|
+
# $agent->process! do |entry|
|
122
|
+
# true
|
123
|
+
# end
|
124
|
+
#
|
125
|
+
# @example Same, but also updates the 'notice' field in the record along with the 1 in the agent_name field
|
126
|
+
# $agent->process! do |entry|
|
127
|
+
# [true, {:notice => 'There were 30 files processed'}]
|
128
|
+
# end
|
129
|
+
#
|
130
|
+
# @example Fails, enters f:server_hostname in the agent_name field
|
131
|
+
# $agent->process! do |entry|
|
132
|
+
# false
|
133
|
+
#
|
134
|
+
# @example Same, but also updates the 'notice' field in the record along with the failure notice
|
135
|
+
# $agent->process! do |entry|
|
136
|
+
# [false, {:notice => 'There were 10 files left to process!' }]
|
137
|
+
# end
|
138
|
+
#
|
139
|
+
# @example This agent passes different parameters based on success or failure
|
140
|
+
# $agent->process! do |entry|
|
141
|
+
# if $success
|
142
|
+
# true
|
143
|
+
# else
|
144
|
+
# [ false, {:notice => 'there were 10 remaining files'}]
|
145
|
+
# end
|
146
|
+
# end
|
147
|
+
#
|
148
|
+
# @param agent_code [Proc] Code to process entry
|
149
|
+
# @yieldparam [GoogleDrive::List] entry
|
150
|
+
# @yieldreturn [Boolean, Hash] success, (optional) hash of fields to update and values to update on the fields
|
151
|
+
def process!(&agent_code)
|
152
|
+
@worksheet.reload
|
153
|
+
no_problems = true
|
154
|
+
capture_output = nil
|
155
|
+
unless @debug
|
156
|
+
capture_output = CaptureIO.new
|
157
|
+
capture_output.start
|
158
|
+
end
|
159
|
+
|
160
|
+
begin
|
161
|
+
return true if has_conflicts()
|
162
|
+
(runnable, entry) = run_entry()
|
163
|
+
return false unless entry
|
164
|
+
return true unless runnable
|
165
|
+
|
166
|
+
success, update_entry = agent_code.call(entry)
|
167
|
+
if success
|
168
|
+
complete_entry(update_entry)
|
169
|
+
else
|
170
|
+
fail_entry(update_entry)
|
171
|
+
end
|
172
|
+
rescue
|
173
|
+
$stderr.puts "#{ $! }"
|
174
|
+
no_problems = false
|
175
|
+
end
|
176
|
+
unless capture_output.nil?
|
177
|
+
if no_problems
|
178
|
+
capture_output.stop
|
179
|
+
else
|
180
|
+
mail_error(capture_output.stop)
|
181
|
+
end
|
182
|
+
end
|
183
|
+
return no_problems
|
184
|
+
end
|
185
|
+
|
186
|
+
# The GoogleDrive::List[http://rubydoc.info/gems/google_drive/0.3.6/GoogleDrive/List] for the specified keys
|
187
|
+
# @return [GoogleDrive::List]
|
188
|
+
def get_entry
|
189
|
+
this_entry = nil
|
190
|
+
if @worksheet
|
191
|
+
@worksheet.list.each do |this_row|
|
192
|
+
keep_row = true
|
193
|
+
|
194
|
+
@config['key_fields'].keys.reject { |key_field|
|
195
|
+
!(@config['key_fields'][key_field]["required"]) && !(@keys[key_field])
|
196
|
+
}.each do |key|
|
197
|
+
break unless keep_row
|
198
|
+
keep_row = (this_row[key] == @keys[key])
|
199
|
+
end
|
200
|
+
|
201
|
+
if keep_row
|
202
|
+
return this_row
|
203
|
+
end
|
204
|
+
end
|
205
|
+
end
|
206
|
+
end
|
207
|
+
|
208
|
+
private
|
209
|
+
|
210
|
+
def has_conflicts
|
211
|
+
return unless (@max_selves || @conflicts_with) # nothing conflicts here
|
212
|
+
|
213
|
+
running_conflicters = {}
|
214
|
+
self_name = File.basename $0
|
215
|
+
|
216
|
+
begin
|
217
|
+
conflicting_in = Open3.popen3('ps','-eo','pid,command')[1]
|
218
|
+
conflicting_in.lines.each do |line|
|
219
|
+
unless(
|
220
|
+
(line.match(/emacs\s+|vim*\s+|pico\s+/)) ||
|
221
|
+
(line.match("#{ $$ }"))
|
222
|
+
)
|
223
|
+
if @max_selves && line.match(self_name)
|
224
|
+
if running_conflicters[@agent_name].nil?
|
225
|
+
running_conflicters[@agent_name] = 1
|
226
|
+
else
|
227
|
+
running_conflicters[@agent_name] += 1
|
228
|
+
end
|
229
|
+
|
230
|
+
if running_conflicters[@agent_name] == @max_selves
|
231
|
+
$stderr.puts "max_selves limit reached" if @debug
|
232
|
+
conflicting_in.close
|
233
|
+
return true
|
234
|
+
end
|
235
|
+
end
|
236
|
+
|
237
|
+
if @conflicts_with
|
238
|
+
@conflicts_with.keys.each do |conflicter|
|
239
|
+
if line.match(conflicter)
|
240
|
+
if running_conflicters[conflicter].nil?
|
241
|
+
running_conflicters[conflicter] = 1
|
242
|
+
else
|
243
|
+
running_conflicters[conflicter] += 1
|
244
|
+
end
|
245
|
+
if running_conflicters[conflicter] >= @conflicts_with[conflicter]
|
246
|
+
$stderr.puts "conflicts with #{ conflicter }" if @debug
|
247
|
+
conflicting_in.close
|
248
|
+
return true
|
249
|
+
end
|
250
|
+
end
|
251
|
+
end
|
252
|
+
end
|
253
|
+
end
|
254
|
+
end
|
255
|
+
conflicting_in.close
|
256
|
+
return false
|
257
|
+
|
258
|
+
rescue
|
259
|
+
$stderr.puts "Couldnt check conflicts #{ $! }" if @debug
|
260
|
+
return true
|
261
|
+
end
|
262
|
+
|
263
|
+
end
|
264
|
+
|
265
|
+
# this call initiates a race resistant attempt to make sure that there is only 1
|
266
|
+
# clear 'winner' among N potential agents attempting to run the same goal on the
|
267
|
+
# same spreadsheet agent's cell
|
268
|
+
def run_entry
|
269
|
+
entry = get_entry()
|
270
|
+
output = '';
|
271
|
+
@keys.keys.select { |k| @config['key_fields'][k] && @keys[k] }.each do |key|
|
272
|
+
output += [ key, @keys[key] ].join(' ') + " "
|
273
|
+
end
|
274
|
+
|
275
|
+
unless entry
|
276
|
+
$stderr.puts "#{ output } is not supported on #{ @page_name }" if @debug
|
277
|
+
return
|
278
|
+
end
|
279
|
+
|
280
|
+
unless entry['ready'] == "1"
|
281
|
+
$stderr.puts "#{ output } is not ready to run #{ @agent_name }" if @debug
|
282
|
+
return false, entry
|
283
|
+
end
|
284
|
+
|
285
|
+
if entry['complete'] == "1"
|
286
|
+
$stderr.puts "All goals are completed for #{ output }" if @debug
|
287
|
+
return false, entry
|
288
|
+
end
|
289
|
+
|
290
|
+
if entry[@agent_name]
|
291
|
+
(status, running_hostname) = entry[@agent_name].split(':')
|
292
|
+
|
293
|
+
case status
|
294
|
+
when 'r'
|
295
|
+
$stderr.puts " #{ output } is already running #{ @agent_name } on #{ running_hostname }" if @debug
|
296
|
+
return false, entry
|
297
|
+
|
298
|
+
when "1"
|
299
|
+
$stderr.puts " #{ output } has already run #{ @agent_name }" if @debug
|
300
|
+
return false, entry
|
301
|
+
|
302
|
+
when 'F'
|
303
|
+
$stderr.puts " #{ output } has already Failed #{ @agent_name }" if @debug
|
304
|
+
return false, entry
|
305
|
+
end
|
306
|
+
end
|
307
|
+
|
308
|
+
if @prerequisites
|
309
|
+
@prerequisites.each do |prereq_field|
|
310
|
+
unless entry[prereq_field] == "1"
|
311
|
+
$stderr.puts " #{ output } has not finished #{ prereq_field }" if @debug
|
312
|
+
return false, entry
|
313
|
+
end
|
314
|
+
end
|
315
|
+
end
|
316
|
+
|
317
|
+
# first attempt to set the hostname of the machine as the value of the agent
|
318
|
+
hostname = Socket.gethostname;
|
319
|
+
begin
|
320
|
+
entry.update @agent_name => "r:#{ hostname }"
|
321
|
+
@worksheet.save
|
322
|
+
|
323
|
+
rescue GoogleDrive::Error
|
324
|
+
# this is a collision, which is to be treated as if it is not runnable
|
325
|
+
$stderr.puts " #{ output } lost #{ @agent_name } on #{hostname}" if @debug
|
326
|
+
return false, entry
|
327
|
+
end
|
328
|
+
|
329
|
+
sleep 3
|
330
|
+
begin
|
331
|
+
@worksheet.reload
|
332
|
+
rescue GoogleDrive::Error
|
333
|
+
# this is a collision, which is to be treated as if it is not runnable
|
334
|
+
$stderr.puts " #{ output } lost #{ @agent_name } on #{hostname}" if @debug
|
335
|
+
return false, entry
|
336
|
+
end
|
337
|
+
|
338
|
+
check = entry[@agent_name]
|
339
|
+
(status, running_hostname) = check.split(':')
|
340
|
+
if hostname == running_hostname
|
341
|
+
return true, entry
|
342
|
+
end
|
343
|
+
$stderr.puts " #{ output } lost #{ @agent_name } on #{hostname}" if @debug
|
344
|
+
return false, entry
|
345
|
+
end
|
346
|
+
|
347
|
+
def complete_entry(update_entry)
|
348
|
+
if update_entry.nil?
|
349
|
+
update_entry = {}
|
350
|
+
end
|
351
|
+
|
352
|
+
if @subsumes && @subsumes.length > 0
|
353
|
+
@subsumes.each do |subsumed_agent|
|
354
|
+
update_entry[subsumed_agent] = 1
|
355
|
+
end
|
356
|
+
end
|
357
|
+
|
358
|
+
update_entry[@agent_name] = 1
|
359
|
+
entry = get_entry()
|
360
|
+
entry.update update_entry
|
361
|
+
@worksheet.save
|
362
|
+
end
|
363
|
+
|
364
|
+
def fail_entry(update_entry)
|
365
|
+
if update_entry.nil?
|
366
|
+
update_entry = { }
|
367
|
+
end
|
368
|
+
hostname = Socket.gethostname
|
369
|
+
update_entry[@agent_name] = "F:#{ hostname }"
|
370
|
+
entry = get_entry()
|
371
|
+
entry.update update_entry
|
372
|
+
@worksheet.save
|
373
|
+
end
|
374
|
+
|
375
|
+
def mail_error(error_message)
|
376
|
+
output = ''
|
377
|
+
@keys.keys.each do |key|
|
378
|
+
output += [key, @keys[key] ].join(' ') + " "
|
379
|
+
end
|
380
|
+
|
381
|
+
prefix = [Socket.gethostname, output, @agent_name ].join(' ')
|
382
|
+
begin
|
383
|
+
Mail.defaults do
|
384
|
+
delivery_method :smtp, {
|
385
|
+
:address => "smtp.gmail.com",
|
386
|
+
:port => 587,
|
387
|
+
:domain => Socket.gethostname,
|
388
|
+
:user_name => @config['guser'],
|
389
|
+
:password => @config['gpass'],
|
390
|
+
:authentication => 'plain',
|
391
|
+
:enable_starttls_auto => true }
|
392
|
+
end
|
393
|
+
|
394
|
+
mail = Mail.new do
|
395
|
+
from @config['reply_email']
|
396
|
+
to @config['send_to']
|
397
|
+
subject prefix
|
398
|
+
body error_message.to_s
|
399
|
+
end
|
400
|
+
|
401
|
+
mail.deliver!
|
402
|
+
rescue
|
403
|
+
#DO NOTHING
|
404
|
+
end
|
405
|
+
end
|
406
|
+
|
407
|
+
end #SpreadsheetAgent::Agent
|
408
|
+
end #SpreadsheetAgent
|
data/lib/spreadsheet_agent/db.rb
CHANGED
@@ -4,7 +4,10 @@ require 'spreadsheet_agent/db'
|
|
4
4
|
class TC_SpreadsheetAgentDbTest < Test::Unit::TestCase
|
5
5
|
def test_instantiate_db
|
6
6
|
conf_file = File.expand_path(File.dirname( __FILE__ )) + '/../config/agent.conf.yml'
|
7
|
-
|
7
|
+
unless File.exists?( conf_file )
|
8
|
+
$stderr.puts "You must create a valid test Google Spreadsheet and a valid #{ conf_file } configuration file pointing to it to run the tests. See README.txt file for more information on how to run the tests."
|
9
|
+
exit
|
10
|
+
end
|
8
11
|
|
9
12
|
google_db = SpreadsheetAgent::Db.new()
|
10
13
|
assert_not_nil google_db
|
@@ -13,7 +13,7 @@ class TC_SpreadsheetAgentRunnerTest < Test::Unit::TestCase
|
|
13
13
|
|
14
14
|
unless File.exists? @config_file
|
15
15
|
$stderr.puts "You must create a valid test Google Spreadsheet and a valid #{ @config_file } configuration file pointing to it to run the tests. See README.txt file for more information on how to run the tests."
|
16
|
-
exit
|
16
|
+
exit
|
17
17
|
end
|
18
18
|
@test_agent_bin = find_bin() + 'agent_bin'
|
19
19
|
@testing_pages = nil
|
@@ -14,7 +14,7 @@ class TC_SpreadsheetAgentTest < Test::Unit::TestCase
|
|
14
14
|
|
15
15
|
unless File.exists? @config_file
|
16
16
|
$stderr.puts "You must create a valid test Google Spreadsheet and a valid #{ @config_file } configuration file pointing to it to run the tests. See README.txt file for more information on how to run the tests."
|
17
|
-
exit
|
17
|
+
exit
|
18
18
|
end
|
19
19
|
|
20
20
|
@testing_page_name = 'testing'
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: spreadsheet_agent
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -116,6 +116,7 @@ executables: []
|
|
116
116
|
extensions: []
|
117
117
|
extra_rdoc_files: []
|
118
118
|
files:
|
119
|
+
- lib/spreadsheet_agent/agent.rb
|
119
120
|
- lib/spreadsheet_agent/db.rb
|
120
121
|
- lib/spreadsheet_agent/error.rb
|
121
122
|
- lib/spreadsheet_agent/runner.rb
|
@@ -125,6 +126,7 @@ files:
|
|
125
126
|
- test/spreadsheet_agent_db_test.rb
|
126
127
|
- test/spreadsheet_agent_runner_test.rb
|
127
128
|
- test/spreadsheet_agent_test.rb
|
129
|
+
- config/test.config.yml
|
128
130
|
homepage: http://rubygems.org/gems/spreadsheet_agent
|
129
131
|
licenses:
|
130
132
|
- MIT
|
@@ -155,3 +157,4 @@ test_files:
|
|
155
157
|
- test/spreadsheet_agent_db_test.rb
|
156
158
|
- test/spreadsheet_agent_runner_test.rb
|
157
159
|
- test/spreadsheet_agent_test.rb
|
160
|
+
has_rdoc:
|