nera 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.txt +6 -0
- data/lib/nera/nera_database.rb +26 -2
- data/lib/nera/nera_db_folders.rb +13 -0
- data/lib/nera/nera_job_layer_controller.rb +30 -26
- data/lib/nera/nera_job_records.rb +5 -0
- data/lib/nera/nera_parameter_layer_controller.rb +20 -15
- data/lib/nera/nera_parameter_records.rb +18 -6
- data/lib/nera/nera_remote_connector.rb +50 -20
- data/lib/nera/nera_run_layer_controller.rb +26 -16
- data/lib/nera/nera_run_records.rb +11 -2
- data/lib/nera/nera_simulator_layer_controller.rb +4 -10
- data/lib/nera/nera_simulator_records.rb +8 -0
- data/lib/nera.rb +1 -1
- data/test/test_nera_database.rb +63 -1
- data/test/test_nera_db_folders.rb +8 -0
- data/test/test_nera_job_layer_controller.rb +22 -0
- data/test/test_nera_job_records.rb +13 -0
- data/test/test_nera_parameter_layer_controller.rb +16 -0
- data/test/test_nera_parameter_records.rb +69 -7
- data/test/test_nera_remote_connector.rb +25 -0
- data/test/test_nera_run_layer_controller.rb +27 -0
- data/test/test_nera_run_records.rb +56 -2
- data/test/test_nera_simulator_records.rb +29 -0
- metadata +2 -2
data/History.txt
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
== 0.2.0
|
2
|
+
* 1 major enhancement:
|
3
|
+
* logger is implemented. Log informatios are stored in "db_name"/Tables/logfile.txt.
|
4
|
+
* 2 bug fixes
|
5
|
+
* updated_at records of tables have been correctly implemented.
|
6
|
+
|
1
7
|
== 0.1.2
|
2
8
|
* 1 major enhancement:
|
3
9
|
* constructor of ParamLayerController now accepts String type simulator name.
|
data/lib/nera/nera_database.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'fileutils'
|
2
2
|
require 'yaml'
|
3
|
-
#require 'filelock'
|
4
3
|
require 'pstore'
|
4
|
+
require 'observer'
|
5
5
|
require 'pp'
|
6
6
|
|
7
7
|
#* list of public methods
|
@@ -73,15 +73,24 @@ module NERA
|
|
73
73
|
# classes of values
|
74
74
|
VALUE_CLASSES = [ String, Symbol, Integer, Float, Date, DateTime, TrueClass, FalseClass, NilClass ]
|
75
75
|
|
76
|
+
# notify to updates to observer
|
77
|
+
include Observable
|
78
|
+
|
76
79
|
public
|
77
80
|
# ----------------------------------
|
78
|
-
def initialize( path_to_file )
|
81
|
+
def initialize( path_to_file, notify_observer_arg = nil )
|
79
82
|
@filename = path_to_file
|
80
83
|
@in_transaction = nil
|
84
|
+
@notify_observer_arg = notify_observer_arg
|
81
85
|
unless File.exist?(@filename)
|
82
86
|
raise "No such database : #{@filename}"
|
83
87
|
end
|
84
88
|
end
|
89
|
+
|
90
|
+
# ------------------------------------
|
91
|
+
def set_yaml_file( yml_path)
|
92
|
+
@yml_path = yml_path.to_s
|
93
|
+
end
|
85
94
|
|
86
95
|
# ------------------------------------
|
87
96
|
def transaction
|
@@ -94,6 +103,18 @@ module NERA
|
|
94
103
|
begin
|
95
104
|
@in_transaction = true
|
96
105
|
yield
|
106
|
+
if @notify_observer_arg
|
107
|
+
notify_observers( @notify_observer_arg)
|
108
|
+
else
|
109
|
+
notify_observers
|
110
|
+
end
|
111
|
+
if @yml_path
|
112
|
+
File.open( @yml_path,'w') do |io|
|
113
|
+
l = find_all do |r| true end
|
114
|
+
YAML.dump( l, io)
|
115
|
+
io.flush
|
116
|
+
end
|
117
|
+
end
|
97
118
|
ensure
|
98
119
|
@in_transaction = false
|
99
120
|
nil
|
@@ -140,6 +161,7 @@ module NERA
|
|
140
161
|
new_rec[:id] = new_id
|
141
162
|
@pstore[:max_id] = new_id
|
142
163
|
@pstore[new_id] = new_rec
|
164
|
+
changed
|
143
165
|
end
|
144
166
|
return new_id
|
145
167
|
end
|
@@ -235,6 +257,7 @@ module NERA
|
|
235
257
|
end
|
236
258
|
@pstore[id_to_update] = rec
|
237
259
|
flag = true
|
260
|
+
changed
|
238
261
|
end
|
239
262
|
return flag
|
240
263
|
end
|
@@ -267,6 +290,7 @@ module NERA
|
|
267
290
|
stat = nil
|
268
291
|
transaction do
|
269
292
|
stat = @pstore.delete(ids)
|
293
|
+
changed
|
270
294
|
end
|
271
295
|
return stat
|
272
296
|
else
|
data/lib/nera/nera_db_folders.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'fileutils'
|
2
2
|
require 'nera_simulator_records'
|
3
3
|
require 'nera_job_records'
|
4
|
+
require 'logger'
|
4
5
|
|
5
6
|
|
6
7
|
module NERA
|
@@ -8,6 +9,14 @@ module NERA
|
|
8
9
|
# This class contains the information of the folder structure
|
9
10
|
class DbFolders
|
10
11
|
|
12
|
+
# -------------------------------------
|
13
|
+
# instance variables ------------------
|
14
|
+
# -------------------------------------
|
15
|
+
@db_folder # path to db_folder
|
16
|
+
@logger # logger instance
|
17
|
+
|
18
|
+
attr_reader :logger
|
19
|
+
|
11
20
|
#--------------------------------------
|
12
21
|
#-class methods -----------------------
|
13
22
|
#--------------------------------------
|
@@ -26,6 +35,10 @@ module NERA
|
|
26
35
|
Dir.glob( "#{@db_folder}/Simulator_classes/*.rb").each do |lib|
|
27
36
|
require lib
|
28
37
|
end
|
38
|
+
|
39
|
+
# logger instance
|
40
|
+
@logger = Logger.new( "#{@db_folder}/Tables/logfile.txt", 8)
|
41
|
+
@logger.level = Logger::INFO
|
29
42
|
end
|
30
43
|
|
31
44
|
def valid_db?( fold)
|
@@ -22,6 +22,7 @@ module NERA
|
|
22
22
|
|
23
23
|
@db_folders = NERA::DbFolders.new( path_db_folder)
|
24
24
|
@job_records = NERA::JobRecords.new( @db_folders.path_to_jobs_table)
|
25
|
+
@job_records.set_yaml_file( @db_folders.path_to_jobs_yaml)
|
25
26
|
end
|
26
27
|
|
27
28
|
public
|
@@ -63,22 +64,29 @@ module NERA
|
|
63
64
|
d = @job_records.destroy( jid)
|
64
65
|
next unless d
|
65
66
|
sim_recs = NERA::SimulatorRecords.new( @db_folders.path_to_simulators_table)
|
66
|
-
|
67
|
+
sim_recs.set_yaml_file( @db_folders.path_to_simulators_yaml )
|
68
|
+
found = sim_recs.list.find do |item| item[:name] == d[:simulator] end
|
69
|
+
sim_id = found[:id]
|
70
|
+
klass = NERA::Simulator.inherited_simulators.find do |sim|
|
71
|
+
sim.to_s == d[:simulator]
|
72
|
+
end
|
67
73
|
raise RuntimeError, "must not happen" unless klass
|
68
|
-
|
74
|
+
p_tab_path = @db_folders.path_to_parameters_table( klass)
|
75
|
+
param_recs = NERA::ParameterRecords.new( p_tab_path, sim_recs, sim_id)
|
76
|
+
param_recs.set_yaml_file( @db_folders.path_to_parameters_yaml( klass) )
|
77
|
+
run_records = NERA::RunRecords.new( @db_folders.path_to_runs_table( klass, d[:parameter_id]), param_recs, d[:parameter_id] )
|
78
|
+
run_records.set_yaml_file( @db_folders.path_to_runs_yaml( klass, d[:parameter_id]) )
|
69
79
|
run_records.transaction {
|
70
80
|
a = run_records.destroy_job_id( jid)
|
71
81
|
raise RuntimeError, "must not happen" unless a
|
72
82
|
FileUtils.rm( @db_folders.path_to_job_script(jid) )
|
73
83
|
}
|
74
|
-
File.open( @db_folders.path_to_runs_yaml( klass, d[:parameter_id]), 'w') do |io|
|
75
|
-
YAML.dump( run_records.list_all, io)
|
76
|
-
io.flush
|
77
|
-
end
|
78
84
|
destroyed_jobids << jid
|
85
|
+
@db_folders.logger.info(self.class.to_s) {
|
86
|
+
"canceled a job (jobid:#{jid})"
|
87
|
+
}
|
79
88
|
end
|
80
89
|
}
|
81
|
-
dump_in_yaml
|
82
90
|
destroyed_jobids = nil if destroyed_jobids.size == 0
|
83
91
|
return destroyed_jobids
|
84
92
|
end
|
@@ -96,19 +104,18 @@ module NERA
|
|
96
104
|
fpath = @db_folders.path_to_include_layer + filename
|
97
105
|
return nil unless File.exist?(fpath)
|
98
106
|
jinfo, rinfo = expand_and_parse( fpath)
|
99
|
-
run_layer_path, run_recs
|
107
|
+
run_layer_path, run_recs = check_consistency( jinfo, rinfo)
|
100
108
|
run_recs.transaction {
|
101
109
|
folder_path = fpath.sub(/.tar.bz2$/,'/')
|
102
110
|
file_move( folder_path, run_layer_path, jinfo[:run_ids])
|
103
111
|
update_tables( run_recs, jinfo, rinfo)
|
104
112
|
}
|
105
113
|
remove_files( fpath)
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
114
|
+
@db_folders.logger.info(self.class.to_s) {
|
115
|
+
"included a file (#{filename})"
|
116
|
+
}
|
117
|
+
|
110
118
|
}
|
111
|
-
dump_in_yaml
|
112
119
|
return true
|
113
120
|
end
|
114
121
|
|
@@ -139,12 +146,17 @@ module NERA
|
|
139
146
|
# check consistency of the simulator
|
140
147
|
conflicting unless jinfo[:simulator] == found[:simulator]
|
141
148
|
sim_records = NERA::SimulatorRecords.new( @db_folders.path_to_simulators_table)
|
142
|
-
|
149
|
+
sim_records.set_yaml_file( @db_folders.path_to_simulators_yaml )
|
150
|
+
sim_class = NERA::Simulator.inherited_simulators.find do |sim|
|
151
|
+
sim.to_s == jinfo[:simulator]
|
152
|
+
end
|
143
153
|
conflicting unless jinfo[:simulator] == sim_class.to_s
|
144
154
|
|
145
155
|
# check consistency of the parameter
|
146
156
|
conflicting unless jinfo[:param_id] == found[:parameter_id]
|
147
|
-
|
157
|
+
f1 = sim_records.list.find do |item| item[:name] == jinfo[:simulator] end
|
158
|
+
param_records = NERA::ParameterRecords.new( @db_folders.path_to_parameters_table( sim_class) , sim_records, f1[:id] )
|
159
|
+
param_records.set_yaml_file( @db_folders.path_to_parameters_yaml( sim_class) )
|
148
160
|
param = param_records.find_by_id( jinfo[:param_id])
|
149
161
|
jinfo[:parameters].each_pair do |key, value|
|
150
162
|
conflicting unless param[key] == value
|
@@ -153,8 +165,8 @@ module NERA
|
|
153
165
|
# check consistency of the runs
|
154
166
|
conflicting unless jinfo[:run_ids][0] == found[:run_id]
|
155
167
|
conflicting unless jinfo[:run_ids].size == found[:number_of_runs]
|
156
|
-
run_recs = NERA::RunRecords.new( @db_folders.path_to_runs_table( sim_class, jinfo[:param_id]) )
|
157
|
-
|
168
|
+
run_recs = NERA::RunRecords.new( @db_folders.path_to_runs_table( sim_class, jinfo[:param_id]), param_records, jinfo[:param_id])
|
169
|
+
run_recs.set_yaml_file( @db_folders.path_to_runs_yaml( sim_class, jinfo[:param_id]) )
|
158
170
|
found_runs = run_recs.list_all_not_finished.find_all do |rec|
|
159
171
|
rec[:job_id] == jinfo[:job_id] and jinfo[:run_ids].include?(rec[:id])
|
160
172
|
end
|
@@ -164,7 +176,7 @@ module NERA
|
|
164
176
|
raise "Run info is not consistent with job info!" unless jinfo[:run_ids] == rinfo.keys.sort
|
165
177
|
|
166
178
|
run_layer_path = @db_folders.path_to_run_layer( sim_class, jinfo[:param_id])
|
167
|
-
return run_layer_path, run_recs
|
179
|
+
return run_layer_path, run_recs
|
168
180
|
end
|
169
181
|
|
170
182
|
def conflicting
|
@@ -223,16 +235,8 @@ module NERA
|
|
223
235
|
f = include(fname)
|
224
236
|
flag = nil unless f
|
225
237
|
end
|
226
|
-
dump_in_yaml
|
227
238
|
return flag
|
228
239
|
end
|
229
240
|
|
230
|
-
def dump_in_yaml
|
231
|
-
File.open( @db_folders.path_to_jobs_yaml, 'w') do |io|
|
232
|
-
YAML.dump( @job_records.list_all, io)
|
233
|
-
io.flush
|
234
|
-
end
|
235
|
-
end
|
236
|
-
|
237
241
|
end
|
238
242
|
end
|
@@ -25,6 +25,11 @@ module NERA
|
|
25
25
|
@db = NERA::Database.new( db_file)
|
26
26
|
end
|
27
27
|
|
28
|
+
# argument is the path to jobs.yml
|
29
|
+
def set_yaml_file( yaml_path)
|
30
|
+
@db.set_yaml_file( yaml_path)
|
31
|
+
end
|
32
|
+
|
28
33
|
# if file already exists, returns false
|
29
34
|
def self.create_table( db_file)
|
30
35
|
NERA::Database.create_table( db_file)
|
@@ -20,17 +20,21 @@ module NERA
|
|
20
20
|
raise ArgumentError unless path_db_folder.is_a?(String)
|
21
21
|
@db_folders = NERA::DbFolders.new( path_db_folder)
|
22
22
|
sim_records = NERA::SimulatorRecords.new( @db_folders.path_to_simulators_table)
|
23
|
+
sim_records.set_yaml_file( @db_folders.path_to_simulators_yaml )
|
24
|
+
|
23
25
|
found = sim_records.list.find do |rec|
|
24
26
|
rec[:name] == sim_name.to_s
|
25
27
|
end
|
26
28
|
raise "No such simulator." unless found
|
29
|
+
sim_id = found[:id]
|
27
30
|
@sim_class = NERA::Simulator.inherited_simulators.find do |sim|
|
28
31
|
sim.to_s == sim_name.to_s
|
29
32
|
end
|
30
33
|
raise "No such simulator." unless @sim_class
|
31
34
|
|
32
35
|
p_tab_path = @db_folders.path_to_parameters_table( @sim_class)
|
33
|
-
@param_records = NERA::ParameterRecords.new( p_tab_path,
|
36
|
+
@param_records = NERA::ParameterRecords.new( p_tab_path, sim_records, sim_id)
|
37
|
+
@param_records.set_yaml_file( @db_folders.path_to_parameters_yaml( @sim_class) )
|
34
38
|
end
|
35
39
|
|
36
40
|
def parameters_list_in_csv
|
@@ -73,22 +77,24 @@ module NERA
|
|
73
77
|
pid = @param_records.add( p_hash)
|
74
78
|
return nil unless pid
|
75
79
|
b = @db_folders.create_new_parameter_folder( @sim_class, pid)
|
76
|
-
raise "Creation of a new folder for #{
|
80
|
+
raise "Creation of a new folder for #{@sim_calss}, #{pid} failed." unless b
|
77
81
|
path = @db_folders.path_to_runs_table( @sim_class, pid)
|
78
82
|
c = NERA::RunRecords.create_table( path)
|
79
83
|
raise "File #{path} already exists." unless c
|
84
|
+
@db_folders.logger.info(self.class.to_s) {
|
85
|
+
"created a new parameter set (#{@sim_class.to_s}/#{pid})"
|
86
|
+
}
|
80
87
|
}
|
81
|
-
dump_in_yaml
|
82
88
|
return pid
|
83
89
|
end
|
84
90
|
|
85
91
|
def move_a_parameter_set_into_trashbox( id)
|
86
92
|
return nil unless @param_records.find_by_id(id)
|
87
|
-
run_records = NERA::RunRecords.new( @db_folders.path_to_runs_table( @sim_class, id) )
|
93
|
+
run_records = NERA::RunRecords.new( @db_folders.path_to_runs_table( @sim_class, id), @param_records, id )
|
88
94
|
return nil if run_records.list_all_not_finished.size > 0
|
89
95
|
|
90
96
|
job_records = NERA::JobRecords.new( @db_folders.path_to_jobs_table)
|
91
|
-
job_records.transaction {
|
97
|
+
job_records.transaction { # to lock the runs tables
|
92
98
|
@param_records.transaction {
|
93
99
|
flag = @param_records.update_to_state_trashbox(id)
|
94
100
|
return nil unless flag
|
@@ -97,9 +103,11 @@ module NERA
|
|
97
103
|
system(cmd)
|
98
104
|
raise "Command #{cmd} failed" unless $? == 0
|
99
105
|
FileUtils.rm_r(r_path)
|
106
|
+
@db_folders.logger.info(self.class.to_s) {
|
107
|
+
"moved a parameter set into the trashbox (#{@sim_class.to_s}/#{id})"
|
108
|
+
}
|
100
109
|
}
|
101
110
|
}
|
102
|
-
dump_in_yaml
|
103
111
|
return true
|
104
112
|
end
|
105
113
|
|
@@ -134,8 +142,10 @@ module NERA
|
|
134
142
|
system(cmd)
|
135
143
|
raise "Command #{cmd} failed" unless $? == 0
|
136
144
|
FileUtils.rm(r_path+".tar.bz2")
|
145
|
+
@db_folders.logger.info(self.class.to_s) {
|
146
|
+
"reverted a parameter set in the trashbox (#{@sim_class.to_s}/#{id})"
|
147
|
+
}
|
137
148
|
}
|
138
|
-
dump_in_yaml
|
139
149
|
return true
|
140
150
|
end
|
141
151
|
|
@@ -145,17 +155,12 @@ module NERA
|
|
145
155
|
return nil unless flag
|
146
156
|
r_path = @db_folders.path_to_run_layer( @sim_class, id).sub(/\/$/,'')
|
147
157
|
FileUtils.rm("#{r_path}.tar.bz2")
|
158
|
+
@db_folders.logger.info(self.class.to_s) {
|
159
|
+
"deleted a parameter set in the trashbox (#{@sim_class.to_s}/#{id})"
|
160
|
+
}
|
148
161
|
}
|
149
|
-
dump_in_yaml
|
150
162
|
return true
|
151
163
|
end
|
152
164
|
|
153
|
-
def dump_in_yaml
|
154
|
-
File.open( @db_folders.path_to_parameters_yaml( @sim_class), 'w') do |io|
|
155
|
-
YAML.dump( @param_records.list_all, io)
|
156
|
-
io.flush
|
157
|
-
end
|
158
|
-
end
|
159
|
-
|
160
165
|
end
|
161
166
|
end
|
@@ -21,13 +21,17 @@ module NERA
|
|
21
21
|
|
22
22
|
# simulator_specific keys
|
23
23
|
@sim_class
|
24
|
-
|
24
|
+
|
25
25
|
# argument is the path to database file
|
26
|
-
def initialize( db_file,
|
27
|
-
unless
|
28
|
-
raise ArgumentError, "
|
26
|
+
def initialize( db_file, sim_records, sim_id)
|
27
|
+
unless sim_records.is_a?( NERA::SimulatorRecords)
|
28
|
+
raise ArgumentError, "The second argument must be an instance of NERA::SimualtorRecords. #{sim_records.inspect}"
|
29
29
|
end
|
30
|
-
|
30
|
+
unless sim_id.is_a?(Integer)
|
31
|
+
raise ArgumentError, "sim_id must be an Integer. : #{sim_id.inspect}"
|
32
|
+
end
|
33
|
+
|
34
|
+
@sim_class = sim_records.get_class( sim_id)
|
31
35
|
@keys = ATTRIBUTES_PART.map do |k|
|
32
36
|
k[0]
|
33
37
|
end
|
@@ -41,7 +45,13 @@ module NERA
|
|
41
45
|
raise ArgumentError, "Invalid class. This class has a parameter : #{found}"
|
42
46
|
end
|
43
47
|
@keys += sim_keys
|
44
|
-
@db = NERA::Database.new( db_file)
|
48
|
+
@db = NERA::Database.new( db_file, sim_id)
|
49
|
+
@db.add_observer( sim_records)
|
50
|
+
end
|
51
|
+
|
52
|
+
# argument is the path to parameters.yml
|
53
|
+
def set_yaml_file( yaml_path)
|
54
|
+
@db.set_yaml_file( yaml_path)
|
45
55
|
end
|
46
56
|
|
47
57
|
# if file already exists, return false
|
@@ -169,6 +179,8 @@ module NERA
|
|
169
179
|
@db.update( found)
|
170
180
|
end
|
171
181
|
|
182
|
+
alias :update :touch
|
183
|
+
|
172
184
|
def destroy( id)
|
173
185
|
raise ArgumentError unless id.is_a?(Integer)
|
174
186
|
found = @db.find_by_id( id)
|
@@ -77,17 +77,21 @@ module NERA
|
|
77
77
|
Net::SSH.start( found[:host_url], found[:user], h) do |ssh|
|
78
78
|
str = ssh.exec!( cmd)
|
79
79
|
end
|
80
|
-
rescue Errno::ECONNREFUSED
|
80
|
+
rescue Errno::ECONNREFUSED => ex
|
81
81
|
$stderr.puts "failed"
|
82
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
82
83
|
return :connection_failed
|
83
|
-
rescue Errno::ENETUNREACH
|
84
|
+
rescue Errno::ENETUNREACH => ex
|
84
85
|
$stderr.puts "failed"
|
86
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
85
87
|
return :connection_failed
|
86
|
-
rescue
|
88
|
+
rescue SocketError => ex
|
87
89
|
$stderr.puts "failed"
|
90
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
88
91
|
return :connection_failed
|
89
|
-
rescue
|
92
|
+
rescue Net::SSH::Exception => ex
|
90
93
|
$stderr.puts "failed"
|
94
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
91
95
|
return :connection_failed
|
92
96
|
end
|
93
97
|
$stderr.puts "done"
|
@@ -131,20 +135,27 @@ module NERA
|
|
131
135
|
sftp.file.open(target_path, "w") do |f| f.puts contents end
|
132
136
|
ssh.exec!("chmod +x #{target_path}")
|
133
137
|
@job_records.update_to_state_copied( jid, host[:name])
|
138
|
+
@db_folders.logger.info(self.class.to_s) {
|
139
|
+
"transfered a file (#{File.basename(pjs)}) to #{host[:name]}"
|
140
|
+
}
|
134
141
|
end
|
135
142
|
end
|
136
143
|
end
|
137
|
-
rescue Errno::ECONNREFUSED
|
144
|
+
rescue Errno::ECONNREFUSED => ex
|
138
145
|
$stderr.puts "failed"
|
146
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
139
147
|
return :connection_failed
|
140
|
-
rescue Errno::ENETUNREACH
|
148
|
+
rescue Errno::ENETUNREACH => ex
|
141
149
|
$stderr.puts "failed"
|
150
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
142
151
|
return :connection_failed
|
143
|
-
rescue SocketError
|
152
|
+
rescue SocketError => ex
|
144
153
|
$stderr.puts "failed"
|
154
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
145
155
|
return :connection_failed
|
146
|
-
rescue Net::SSH::Exception
|
156
|
+
rescue Net::SSH::Exception => ex
|
147
157
|
$stderr.puts "failed"
|
158
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
148
159
|
return :connection_failed
|
149
160
|
end
|
150
161
|
$stderr.puts "done"
|
@@ -199,20 +210,27 @@ module NERA
|
|
199
210
|
@job_records.update_to_state_copied( jid, host[:name])
|
200
211
|
ssh.exec!("cd #{target_dir} && #{host[:submission_command]} #{target_path} &")
|
201
212
|
@job_records.update_to_state_submitted( jid, host[:name])
|
213
|
+
@db_folders.logger.info(self.class.to_s) {
|
214
|
+
"submitted a file (#{File.basename(pjs)}) to #{host[:name]}"
|
215
|
+
}
|
202
216
|
end
|
203
217
|
end
|
204
218
|
end
|
205
|
-
rescue Errno::ECONNREFUSED
|
219
|
+
rescue Errno::ECONNREFUSED => ex
|
206
220
|
$stderr.puts "failed"
|
221
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
207
222
|
return :connection_failed
|
208
|
-
rescue Errno::ENETUNREACH
|
223
|
+
rescue Errno::ENETUNREACH => ex
|
209
224
|
$stderr.puts "failed"
|
225
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
210
226
|
return :connection_failed
|
211
|
-
rescue SocketError
|
227
|
+
rescue SocketError => ex
|
212
228
|
$stderr.puts "failed"
|
229
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
213
230
|
return :connection_failed
|
214
|
-
rescue Net::SSH::Exception
|
231
|
+
rescue Net::SSH::Exception => ex
|
215
232
|
$stderr.puts "failed"
|
233
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
216
234
|
return :connection_failed
|
217
235
|
end
|
218
236
|
$stderr.puts "done"
|
@@ -235,17 +253,21 @@ module NERA
|
|
235
253
|
Net::SSH.start( host[:host_url], host[:user], h) do |ssh|
|
236
254
|
ls_str = ssh.exec!("ls #{host[:job_path]}")
|
237
255
|
end
|
238
|
-
rescue Errno::ECONNREFUSED
|
256
|
+
rescue Errno::ECONNREFUSED => ex
|
239
257
|
$stderr.puts "failed"
|
258
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
240
259
|
return :connection_failed
|
241
|
-
rescue Errno::ENETUNREACH
|
260
|
+
rescue Errno::ENETUNREACH => ex
|
242
261
|
$stderr.puts "failed"
|
262
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
243
263
|
return :connection_failed
|
244
|
-
rescue SocketError
|
264
|
+
rescue SocketError => ex
|
245
265
|
$stderr.puts "failed"
|
266
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
246
267
|
return :connection_failed
|
247
|
-
rescue Net::SSH::Exception
|
268
|
+
rescue Net::SSH::Exception => ex
|
248
269
|
$stderr.puts "failed"
|
270
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
249
271
|
return :connection_failed
|
250
272
|
end
|
251
273
|
$stderr.puts "done"
|
@@ -312,20 +334,28 @@ module NERA
|
|
312
334
|
$stderr.print " deleting #{sh_file}.....\t"
|
313
335
|
sftp.remove!(target_dir + '/' + sh_file)
|
314
336
|
$stderr.puts "done"
|
337
|
+
|
338
|
+
@db_folders.logger.info(self.class.to_s) {
|
339
|
+
"downloaded a data file (#{d_file}) from #{host[:name]}"
|
340
|
+
}
|
315
341
|
end
|
316
342
|
end
|
317
343
|
end
|
318
|
-
rescue Errno::ECONNREFUSED
|
344
|
+
rescue Errno::ECONNREFUSED => ex
|
319
345
|
$stderr.puts "failed"
|
346
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
320
347
|
return :connection_failed
|
321
|
-
rescue Errno::ENETUNREACH
|
348
|
+
rescue Errno::ENETUNREACH => ex
|
322
349
|
$stderr.puts "failed"
|
350
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
323
351
|
return :connection_failed
|
324
|
-
rescue SocketError
|
352
|
+
rescue SocketError => ex
|
325
353
|
$stderr.puts "failed"
|
354
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
326
355
|
return :connection_failed
|
327
|
-
rescue Net::SSH::Exception
|
356
|
+
rescue Net::SSH::Exception => ex
|
328
357
|
$stderr.puts "failed"
|
358
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
329
359
|
return :connection_failed
|
330
360
|
end
|
331
361
|
$stderr.puts "done"
|
@@ -25,10 +25,13 @@ module NERA
|
|
25
25
|
|
26
26
|
@db_folders = NERA::DbFolders.new( path_db_folder)
|
27
27
|
sim_records = NERA::SimulatorRecords.new( @db_folders.path_to_simulators_table)
|
28
|
+
sim_records.set_yaml_file( @db_folders.path_to_simulators_yaml )
|
29
|
+
|
28
30
|
found = sim_records.list.find do |rec|
|
29
31
|
rec[:name] == sim_name.to_s
|
30
32
|
end
|
31
33
|
raise "No such simulator : #{sim_name.to_s}" unless found
|
34
|
+
sim_id = found[:id]
|
32
35
|
sim_class = NERA::Simulator.inherited_simulators.find do |sim|
|
33
36
|
sim.to_s == sim_name.to_s
|
34
37
|
end
|
@@ -36,7 +39,8 @@ module NERA
|
|
36
39
|
|
37
40
|
@sim_instance = sim_class.new
|
38
41
|
@param_id = param_id
|
39
|
-
param_records = NERA::ParameterRecords.new( @db_folders.path_to_parameters_table( sim_class),
|
42
|
+
param_records = NERA::ParameterRecords.new( @db_folders.path_to_parameters_table( sim_class), sim_records, sim_id)
|
43
|
+
param_records.set_yaml_file( @db_folders.path_to_parameters_yaml( sim_class) )
|
40
44
|
found = param_records.find_by_id( @param_id)
|
41
45
|
raise "No such parameter : #{param_id}." unless found
|
42
46
|
@sim_instance.class::Parameters.each do |pa|
|
@@ -48,7 +52,8 @@ module NERA
|
|
48
52
|
end
|
49
53
|
end
|
50
54
|
|
51
|
-
@run_records = NERA::RunRecords.new( @db_folders.path_to_runs_table( sim_class, @param_id) )
|
55
|
+
@run_records = NERA::RunRecords.new( @db_folders.path_to_runs_table( sim_class, @param_id), param_records, @param_id )
|
56
|
+
@run_records.set_yaml_file( @db_folders.path_to_runs_yaml( @sim_instance.class, @param_id) )
|
52
57
|
end
|
53
58
|
|
54
59
|
def path_to_run_layer
|
@@ -80,6 +85,7 @@ module NERA
|
|
80
85
|
|
81
86
|
job_ids = []
|
82
87
|
job_recs = NERA::JobRecords.new( @db_folders.path_to_jobs_table)
|
88
|
+
job_recs.set_yaml_file( @db_folders.path_to_jobs_yaml)
|
83
89
|
js = NERA::JobScript.new( @db_folders)
|
84
90
|
job_recs.transaction {
|
85
91
|
@run_records.transaction {
|
@@ -89,13 +95,16 @@ module NERA
|
|
89
95
|
jid = job_recs.add( @sim_instance.class, @param_id, rid, num_runs_per_job)
|
90
96
|
@run_records.set_job_id( rid, num_runs_per_job, jid)
|
91
97
|
sim_records = NERA::SimulatorRecords.new( @db_folders.path_to_simulators_table)
|
98
|
+
sim_records.set_yaml_file( @db_folders.path_to_simulators_yaml )
|
92
99
|
found = sim_records.list.find do |rec| rec[:name] == @sim_instance.class.to_s end
|
93
100
|
path_script = js.create_script( jid, found[:id], @param_id, @sim_instance, run_stat)
|
94
101
|
job_ids << jid
|
102
|
+
@db_folders.logger.info(self.class.to_s) {
|
103
|
+
"created a job (#{@sim_instance.class.to_s}/#{@param_id}/#{rid}, #{num_runs_per_job} runs, jobid:#{jid})"
|
104
|
+
}
|
95
105
|
end
|
96
106
|
}
|
97
107
|
}
|
98
|
-
dump_in_yaml( job_recs)
|
99
108
|
return job_ids
|
100
109
|
end
|
101
110
|
|
@@ -132,6 +141,7 @@ module NERA
|
|
132
141
|
end
|
133
142
|
|
134
143
|
job_recs = NERA::JobRecords.new( @db_folders.path_to_jobs_table)
|
144
|
+
job_recs.set_yaml_file( @db_folders.path_to_jobs_yaml)
|
135
145
|
destroyed_jobids = []
|
136
146
|
job_recs.transaction {
|
137
147
|
@run_records.transaction {
|
@@ -142,10 +152,12 @@ module NERA
|
|
142
152
|
raise "must not happen" unless a
|
143
153
|
FileUtils.rm( @db_folders.path_to_job_script(jid) )
|
144
154
|
destroyed_jobids << jid
|
155
|
+
@db_folders.logger.info(self.class.to_s) {
|
156
|
+
"canceled a job (jobid:#{jid})"
|
157
|
+
}
|
145
158
|
end
|
146
159
|
}
|
147
160
|
}
|
148
|
-
dump_in_yaml( job_recs)
|
149
161
|
destroyed_jobids = nil if destroyed_jobids.size == 0
|
150
162
|
return destroyed_jobids
|
151
163
|
end
|
@@ -172,6 +184,10 @@ module NERA
|
|
172
184
|
FileUtils.cd( @db_folders.path_to_run_layer(@sim_instance.class, @param_id) ) {
|
173
185
|
@sim_instance.__send__( method_str)
|
174
186
|
}
|
187
|
+
@db_folders.logger.info(self.class.to_s) {
|
188
|
+
"#{method_str} (#{@sim_instance.class.to_s}/#{@param_id})"
|
189
|
+
}
|
190
|
+
|
175
191
|
end
|
176
192
|
|
177
193
|
def analyze_all
|
@@ -180,24 +196,18 @@ module NERA
|
|
180
196
|
end
|
181
197
|
return nil unless found
|
182
198
|
|
199
|
+
found.each do |meth|
|
200
|
+
analyze( meth)
|
201
|
+
end
|
202
|
+
end
|
203
|
+
=begin
|
183
204
|
FileUtils.cd( @db_folders.path_to_run_layer(@sim_instance.class, @param_id) ) {
|
184
205
|
found.each do |meth|
|
185
206
|
@sim_instance.__send__( meth)
|
186
207
|
end
|
187
208
|
}
|
188
209
|
end
|
189
|
-
|
190
|
-
private
|
191
|
-
def dump_in_yaml( job_recs)
|
192
|
-
File.open( @db_folders.path_to_runs_yaml( @sim_instance.class, @param_id), 'w') do |io|
|
193
|
-
YAML.dump( @run_records.list_all, io)
|
194
|
-
io.flush
|
195
|
-
end
|
196
|
-
File.open( @db_folders.path_to_jobs_yaml, 'w') do |io|
|
197
|
-
YAML.dump( job_recs.list_all, io)
|
198
|
-
io.flush
|
199
|
-
end
|
200
|
-
end
|
210
|
+
=end
|
201
211
|
|
202
212
|
end
|
203
213
|
end
|