content_server 1.2.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bin/backup_server +3 -10
- data/bin/content_server +3 -10
- data/bin/testing_memory +3 -11
- data/bin/testing_server +3 -10
- data/lib/content_data/content_data.rb +9 -16
- data/lib/content_server/backup_server.rb +10 -9
- data/lib/content_server/content_receiver.rb +6 -6
- data/lib/content_server/content_server.rb +10 -10
- data/lib/content_server/file_streamer.rb +15 -13
- data/lib/content_server/queue_copy.rb +25 -23
- data/lib/content_server/queue_indexer.rb +11 -23
- data/lib/content_server/remote_content.rb +6 -6
- data/lib/content_server/server.rb +8 -7
- data/lib/content_server/version.rb +1 -1
- data/lib/file_copy/copy.rb +10 -9
- data/lib/file_monitoring/file_monitoring.rb +51 -18
- data/lib/file_monitoring/monitor_path.rb +68 -33
- data/lib/log.rb +40 -4
- data/lib/networking/tcp.rb +15 -15
- data/lib/params.rb +3 -9
- metadata +2 -2
data/bin/backup_server
CHANGED
@@ -20,8 +20,6 @@ ContentServer.init_globals
|
|
20
20
|
|
21
21
|
Thread.abort_on_exception = true # TODO (genadyp) should be treated globally? by param for example.
|
22
22
|
|
23
|
-
retries = 0 # number of retries to run a server
|
24
|
-
|
25
23
|
begin
|
26
24
|
RunInBackground.run { ContentServer.run_backup_server }
|
27
25
|
rescue SystemExit, SignalException => exception
|
@@ -31,11 +29,6 @@ rescue SystemExit, SignalException => exception
|
|
31
29
|
exit
|
32
30
|
rescue Exception => exception
|
33
31
|
ContentServer.handle_program_termination(exception)
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
Log.debug1("Exiting...")
|
38
|
-
Log.flush
|
39
|
-
end
|
40
|
-
retries -= 1
|
41
|
-
end while retries >= 0
|
32
|
+
Log.flush
|
33
|
+
exit
|
34
|
+
end
|
data/bin/content_server
CHANGED
@@ -17,8 +17,6 @@ ContentServer.init_globals
|
|
17
17
|
|
18
18
|
Thread.abort_on_exception = true # TODO (genadyp) should be treated globally? by param for example.
|
19
19
|
|
20
|
-
retries = 0 # number of retries to run a server
|
21
|
-
|
22
20
|
begin
|
23
21
|
RunInBackground.run { ContentServer.run_content_server }
|
24
22
|
rescue SystemExit, SignalException => exception
|
@@ -28,11 +26,6 @@ rescue SystemExit, SignalException => exception
|
|
28
26
|
exit
|
29
27
|
rescue Exception => exception
|
30
28
|
ContentServer.handle_program_termination(exception)
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
Log.debug1("Exiting...")
|
35
|
-
Log.flush
|
36
|
-
end
|
37
|
-
retries -= 1
|
38
|
-
end while retries >= 0
|
29
|
+
Log.flush
|
30
|
+
exit
|
31
|
+
end
|
data/bin/testing_memory
CHANGED
@@ -24,8 +24,6 @@ ContentServer.init_globals
|
|
24
24
|
|
25
25
|
Thread.abort_on_exception = true
|
26
26
|
|
27
|
-
retries = 0 # number of retries to run a server
|
28
|
-
|
29
27
|
begin
|
30
28
|
case Params['server_to_test']
|
31
29
|
when 'content'
|
@@ -35,7 +33,6 @@ begin
|
|
35
33
|
else
|
36
34
|
raise ArgumentError.new "Incorrect server_to_test parameter value: #{Params['server_to_test']}"
|
37
35
|
end
|
38
|
-
retries -=1
|
39
36
|
|
40
37
|
rescue SystemExit, SignalException => exc
|
41
38
|
# TODO (genadyp) do we need to trap signals by types?
|
@@ -50,11 +47,6 @@ rescue Exception => exc
|
|
50
47
|
"#{exc.backtrace.join("\n")}")
|
51
48
|
Log.error("Exception happened in #{Params['service_name']} server: #{exc.class}:#{exc.message}\nBacktrace:\n" +
|
52
49
|
"#{exc.backtrace.join("\n")}")
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
Log.debug1("Exiting...")
|
57
|
-
Log.flush
|
58
|
-
end
|
59
|
-
retries -= 1
|
60
|
-
end while retries >= 0
|
50
|
+
Log.flush
|
51
|
+
exit
|
52
|
+
end
|
data/bin/testing_server
CHANGED
@@ -22,8 +22,6 @@ ContentServer.init_globals
|
|
22
22
|
|
23
23
|
Thread.abort_on_exception = true
|
24
24
|
|
25
|
-
retries = 0 # number of retries to run a server
|
26
|
-
|
27
25
|
begin
|
28
26
|
case Params['server_to_test']
|
29
27
|
when 'content'
|
@@ -47,11 +45,6 @@ rescue Exception => exc
|
|
47
45
|
"#{exc.backtrace.join("\n")}")
|
48
46
|
Log.error("Exception happened in #{Params['service_name']} server: #{exc.class}:#{exc.message}\nBacktrace:\n" +
|
49
47
|
"#{exc.backtrace.join("\n")}")
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
Log.debug1("Exiting...")
|
54
|
-
Log.flush
|
55
|
-
end
|
56
|
-
retries -= 1
|
57
|
-
end while retries >= 0
|
48
|
+
Log.flush
|
49
|
+
exit
|
50
|
+
end
|
@@ -144,7 +144,7 @@ module ContentData
|
|
144
144
|
modification_time]
|
145
145
|
else
|
146
146
|
if size != content_info[0]
|
147
|
-
Log.warning
|
147
|
+
Log.warning('File size different from content size while same checksum')
|
148
148
|
Log.warning("instance location:server:'#{location[0]}' path:'#{location[1]}'")
|
149
149
|
Log.warning("instance mod time:'#{modification_time}'")
|
150
150
|
end
|
@@ -168,13 +168,6 @@ module ContentData
|
|
168
168
|
@instances_info.has_key?([server, path])
|
169
169
|
end
|
170
170
|
|
171
|
-
def stats_by_location(location)
|
172
|
-
checksum = @instances_info[location]
|
173
|
-
content_info = @contents_info[checksum]
|
174
|
-
return nil if content_info.nil?
|
175
|
-
return [content_info[0], content_info[1][location]]
|
176
|
-
end
|
177
|
-
|
178
171
|
# removes an instance record both in @instances_info and @instances_info.
|
179
172
|
# input params: server & path - are the instance unique key (called location)
|
180
173
|
# removes also the content, if content becomes empty after removing the instance
|
@@ -260,11 +253,11 @@ module ContentData
|
|
260
253
|
file = File.open(filename, 'w')
|
261
254
|
file.write("#{@contents_info.length}\n")
|
262
255
|
each_content { |checksum, size, content_mod_time|
|
263
|
-
file.write("#{checksum},#{size},#{content_mod_time}\n")
|
256
|
+
file.write("#{checksum},#{size},#{Time.at(content_mod_time)}\n")
|
264
257
|
}
|
265
258
|
file.write("#{@instances_info.length}\n")
|
266
259
|
each_instance { |checksum, size, _, instance_mod_time, server, path|
|
267
|
-
file.write("#{checksum},#{size},#{server},#{path},#{instance_mod_time}\n")
|
260
|
+
file.write("#{checksum},#{size},#{server},#{path},#{Time.at(instance_mod_time)}\n")
|
268
261
|
}
|
269
262
|
file.close
|
270
263
|
end
|
@@ -278,7 +271,7 @@ module ContentData
|
|
278
271
|
i += 1
|
279
272
|
number_of_instances.times {
|
280
273
|
if lines[i].nil?
|
281
|
-
Log.warning
|
274
|
+
Log.warning("line ##{i} is nil !!!, Backing filename: #{filename} to #{filename}.bad")
|
282
275
|
FileUtils.cp(filename, "#{filename}.bad")
|
283
276
|
Log.warning("Lines:\n#{lines[i].join("\n")}")
|
284
277
|
else
|
@@ -297,7 +290,7 @@ module ContentData
|
|
297
290
|
parameters[1].to_i,
|
298
291
|
parameters[2],
|
299
292
|
parameters[3],
|
300
|
-
parameters[4].to_i)
|
293
|
+
DateTime.parse(parameters[4]).to_time.to_i)
|
301
294
|
end
|
302
295
|
i += 1
|
303
296
|
}
|
@@ -397,19 +390,19 @@ module ContentData
|
|
397
390
|
if File.size(path) != size
|
398
391
|
is_valid = false
|
399
392
|
err_msg = "#{path} size #{File.size(path)} differs from indexed size #{size}"
|
400
|
-
Log.warning
|
393
|
+
Log.warning(err_msg)
|
401
394
|
end
|
402
395
|
#if ContentData.format_time(File.mtime(path)) != instance.modification_time
|
403
396
|
if File.mtime(path).to_i != instance_mtime
|
404
397
|
is_valid = false
|
405
398
|
err_msg = "#{path} modification time #{File.mtime(path).to_i} differs from " \
|
406
399
|
+ "indexed #{instance_mtime}"
|
407
|
-
Log.warning
|
400
|
+
Log.warning(err_msg)
|
408
401
|
end
|
409
402
|
else
|
410
403
|
is_valid = false
|
411
404
|
err_msg = "Indexed file #{path} doesn't exist"
|
412
|
-
Log.warning
|
405
|
+
Log.warning(err_msg)
|
413
406
|
end
|
414
407
|
is_valid
|
415
408
|
end
|
@@ -430,7 +423,7 @@ module ContentData
|
|
430
423
|
true
|
431
424
|
else
|
432
425
|
err_msg = "#{path} checksum #{current_checksum} differs from indexed #{instance_checksum}"
|
433
|
-
Log.warning
|
426
|
+
Log.warning(err_msg)
|
434
427
|
false
|
435
428
|
end
|
436
429
|
else
|
@@ -39,19 +39,19 @@ module ContentServer
|
|
39
39
|
$tmp_content_data_file = File.join(Params['tmp_path'], 'backup.data')
|
40
40
|
|
41
41
|
if Params['enable_monitoring']
|
42
|
-
Log.info("Initializing monitoring of process params on port
|
42
|
+
Log.info("Initializing monitoring of process params on port:%s", Params['process_monitoring_web_port'])
|
43
43
|
$process_vars.set('server_name', 'backup_server')
|
44
44
|
end
|
45
45
|
|
46
46
|
# # # # # # # # # # # #
|
47
47
|
# Initialize/start monitoring and destination folder
|
48
48
|
Params['backup_destination_folder'][0]['path']=File.expand_path(Params['backup_destination_folder'][0]['path'])
|
49
|
-
Log.info("backup_destination_folder is
|
49
|
+
Log.info("backup_destination_folder is:%s", Params['backup_destination_folder'][0]['path'])
|
50
50
|
#adding destination folder to monitoring paths
|
51
51
|
Params['monitoring_paths'] << Params['backup_destination_folder'][0]
|
52
52
|
Log.info('Start monitoring following directories:')
|
53
53
|
Params['monitoring_paths'].each { |path|
|
54
|
-
Log.info(" Path:'
|
54
|
+
Log.info(" Path:'%s'", path['path'])
|
55
55
|
}
|
56
56
|
|
57
57
|
# initial global local content data object
|
@@ -60,12 +60,14 @@ module ContentServer
|
|
60
60
|
|
61
61
|
# Read here for initial content data that exist from previous system run
|
62
62
|
content_data_path = Params['local_content_data_path']
|
63
|
+
last_content_data_id = nil
|
63
64
|
if File.exists?(content_data_path) and !File.directory?(content_data_path)
|
64
|
-
Log.info("reading initial content data that exist from previous system run from file
|
65
|
+
Log.info("reading initial content data that exist from previous system run from file:%s", content_data_path)
|
65
66
|
$local_content_data.from_file(content_data_path)
|
67
|
+
last_content_data_id = $local_content_data.unique_id
|
66
68
|
else
|
67
69
|
if File.directory?(content_data_path)
|
68
|
-
raise("Param:'local_content_data_path':'
|
70
|
+
raise("Param:'local_content_data_path':'%s' cannot be a directory name", Params['local_content_data_path'])
|
69
71
|
end
|
70
72
|
# create directory if needed
|
71
73
|
dir = File.dirname(Params['local_content_data_path'])
|
@@ -93,7 +95,6 @@ module ContentServer
|
|
93
95
|
Log.debug1('Init thread: flush local content data to file')
|
94
96
|
all_threads << Thread.new do
|
95
97
|
FileUtils.mkdir_p(Params['tmp_path']) unless File.directory?(Params['tmp_path'])
|
96
|
-
last_content_data_id = nil
|
97
98
|
loop{
|
98
99
|
sleep(Params['data_flush_delay'])
|
99
100
|
Log.info('Start flush local content data to file.')
|
@@ -135,9 +136,9 @@ module ContentServer
|
|
135
136
|
$remote_content_data_lock.synchronize{
|
136
137
|
diff = ContentData.remove($local_content_data, $remote_content_data)
|
137
138
|
unless diff.nil? || diff.empty?
|
138
|
-
Log.debug2("Backup content:\n
|
139
|
-
Log.debug2("Remote content:\n
|
140
|
-
Log.debug2("Missing contents:\n
|
139
|
+
Log.debug2("Backup content:\n%s", $local_content_data)
|
140
|
+
Log.debug2("Remote content:\n%s", $remote_content_data)
|
141
|
+
Log.debug2("Missing contents:\n%s", diff)
|
141
142
|
Log.info('Start sync check. Backup and remote contents need a sync, requesting copy files:')
|
142
143
|
file_copy_client.request_copy(diff)
|
143
144
|
else
|
@@ -14,15 +14,15 @@ module ContentServer
|
|
14
14
|
Socket.tcp_server_loop(@port) do |sock, client_addrinfo|
|
15
15
|
while size_of_data = sock.read(4)
|
16
16
|
size_of_data = size_of_data.unpack("l")[0]
|
17
|
-
Log.debug2("Size of data:
|
17
|
+
Log.debug2("Size of data: %s", size_of_data)
|
18
18
|
data = sock.read(size_of_data)
|
19
19
|
unmarshaled_data = Marshal.load(data)
|
20
20
|
@queue.push unmarshaled_data
|
21
|
-
Log.debug2("Socket closed?
|
21
|
+
Log.debug2("Socket closed? %s.", sock.closed?)
|
22
22
|
break if sock.closed?
|
23
|
-
Log.debug2
|
23
|
+
Log.debug2('Waiting on sock.read')
|
24
24
|
end
|
25
|
-
Log.debug2
|
25
|
+
Log.debug2('Exited, socket closed or read returned nil.')
|
26
26
|
end
|
27
27
|
end
|
28
28
|
end
|
@@ -36,14 +36,14 @@ module ContentServer
|
|
36
36
|
end
|
37
37
|
|
38
38
|
def open_socket
|
39
|
-
Log.debug1("Connecting to content server
|
39
|
+
Log.debug1("Connecting to content server %s:%s.", @host, @port)
|
40
40
|
@tcp_socket = TCPSocket.new(@host, @port)
|
41
41
|
end
|
42
42
|
|
43
43
|
def send_content_data content_data
|
44
44
|
open_socket if @tcp_socket.closed?
|
45
45
|
marshal_data = Marshal.dump(content_data)
|
46
|
-
Log.debug2("Marshaled size:
|
46
|
+
Log.debug2("Marshaled size: %s.", marshal_data.length)
|
47
47
|
data_size = [marshal_data.length].pack("l")
|
48
48
|
if data_size.nil? || marshal_data.nil?
|
49
49
|
Log.debug2('Send data is nil!!!!!!!!')
|
@@ -32,7 +32,7 @@ module ContentServer
|
|
32
32
|
$tmp_content_data_file = File.join(Params['tmp_path'], 'content.data')
|
33
33
|
|
34
34
|
if Params['enable_monitoring']
|
35
|
-
Log.info("Initializing monitoring of process params on port
|
35
|
+
Log.info("Initializing monitoring of process params on port:%s", Params['process_monitoring_web_port'])
|
36
36
|
$process_vars.set('server_name', 'content_server')
|
37
37
|
end
|
38
38
|
|
@@ -40,9 +40,8 @@ module ContentServer
|
|
40
40
|
# Initialize/Start monitoring
|
41
41
|
Log.info('Start monitoring following directories:')
|
42
42
|
Params['monitoring_paths'].each {|path|
|
43
|
-
Log.info(" Path:'
|
43
|
+
Log.info(" Path:'%s'", path['path'])
|
44
44
|
}
|
45
|
-
monitoring_events = Queue.new
|
46
45
|
|
47
46
|
# initial global local content data object
|
48
47
|
$local_content_data_lock = Mutex.new
|
@@ -50,19 +49,21 @@ module ContentServer
|
|
50
49
|
|
51
50
|
# Read here for initial content data that exist from previous system run
|
52
51
|
content_data_path = Params['local_content_data_path']
|
52
|
+
last_content_data_id = nil
|
53
53
|
if File.exists?(content_data_path) and !File.directory?(content_data_path)
|
54
|
-
Log.info("reading initial content data that exist from previous system run from file
|
54
|
+
Log.info("reading initial content data that exist from previous system run from file:%s", content_data_path)
|
55
55
|
$local_content_data.from_file(content_data_path)
|
56
|
+
last_content_data_id = $local_content_data.unique_id
|
56
57
|
else
|
57
58
|
if File.directory?(content_data_path)
|
58
|
-
raise("Param:'local_content_data_path':'
|
59
|
+
raise("Param:'local_content_data_path':'%s'cannot be a directory name", Params['local_content_data_path'])
|
59
60
|
end
|
60
61
|
# create directory if needed
|
61
62
|
dir = File.dirname(Params['local_content_data_path'])
|
62
63
|
FileUtils.mkdir_p(dir) unless File.exists?(dir)
|
63
64
|
end
|
64
65
|
|
65
|
-
Log.info(
|
66
|
+
Log.info('Init monitoring')
|
66
67
|
monitoring_events = Queue.new
|
67
68
|
fm = FileMonitoring::FileMonitoring.new()
|
68
69
|
fm.set_event_queue(monitoring_events)
|
@@ -83,11 +84,10 @@ module ContentServer
|
|
83
84
|
Log.debug1('Init thread: flush local content data to file')
|
84
85
|
all_threads << Thread.new do
|
85
86
|
FileUtils.mkdir_p(Params['tmp_path']) unless File.directory?(Params['tmp_path'])
|
86
|
-
last_content_data_id = nil
|
87
87
|
loop{
|
88
88
|
sleep(Params['data_flush_delay'])
|
89
89
|
Log.info('Start flush local content data to file.')
|
90
|
-
$testing_memory_log.info('Start flush content data to file') if $
|
90
|
+
$testing_memory_log.info('Start flush content data to file') if $testing_memory_active
|
91
91
|
written_to_file = false
|
92
92
|
$local_content_data_lock.synchronize{
|
93
93
|
local_content_data_unique_id = $local_content_data.unique_id
|
@@ -96,11 +96,11 @@ module ContentServer
|
|
96
96
|
$local_content_data.to_file($tmp_content_data_file)
|
97
97
|
written_to_file = true
|
98
98
|
else
|
99
|
-
Log.
|
99
|
+
Log.info('no need to flush. content data has not changed')
|
100
100
|
end
|
101
101
|
}
|
102
102
|
File.rename($tmp_content_data_file, Params['local_content_data_path']) if written_to_file
|
103
|
-
$testing_memory_log.info("End flush content data to file") if $
|
103
|
+
$testing_memory_log.info("End flush content data to file") if $testing_memory_active
|
104
104
|
}
|
105
105
|
end
|
106
106
|
|
@@ -24,12 +24,12 @@ module ContentServer
|
|
24
24
|
|
25
25
|
def self.close_delete_stream(checksum, streams_hash)
|
26
26
|
if streams_hash.key?(checksum)
|
27
|
-
Log.debug1("close_delete_stream
|
27
|
+
Log.debug1("close_delete_stream %s", streams_hash[checksum].path)
|
28
28
|
begin
|
29
29
|
streams_hash[checksum].file.close()
|
30
30
|
rescue IOError => e
|
31
31
|
Log.warning("While closing stream, could not close file #{streams_hash[checksum].path}." \
|
32
|
-
"
|
32
|
+
" IOError msg:#{e.to_s}")
|
33
33
|
end
|
34
34
|
streams_hash.delete(checksum)
|
35
35
|
$process_vars.set('Streams size', streams_hash.size)
|
@@ -105,7 +105,7 @@ module ContentServer
|
|
105
105
|
checksum = content
|
106
106
|
if @streams.key?(checksum)
|
107
107
|
offset = @streams[checksum].file.pos
|
108
|
-
Log.debug1("Sending chunk for
|
108
|
+
Log.debug1("Sending chunk for %s, offset %s.",checksum, offset)
|
109
109
|
chunk = @streams[checksum].file.read(Params['streaming_chunk_size'])
|
110
110
|
if chunk.nil?
|
111
111
|
# No more to read, send end of file.
|
@@ -116,7 +116,7 @@ module ContentServer
|
|
116
116
|
@send_chunk_clb.call(checksum, offset, @streams[checksum].size, chunk, chunk_checksum)
|
117
117
|
end
|
118
118
|
else
|
119
|
-
Log.debug1("No
|
119
|
+
Log.debug1("No stream found for copy chunk checksum %s.", checksum)
|
120
120
|
end
|
121
121
|
end
|
122
122
|
|
@@ -129,7 +129,7 @@ module ContentServer
|
|
129
129
|
if offset > 0
|
130
130
|
file.seek(offset)
|
131
131
|
end
|
132
|
-
Log.debug1("File streamer:
|
132
|
+
Log.debug1("File streamer: %s.", file)
|
133
133
|
@streams[checksum] = Stream.new(checksum, path, file, file.size)
|
134
134
|
$process_vars.set('Streams size', @streams.size)
|
135
135
|
rescue IOError, Errno::ENOENT => e
|
@@ -223,13 +223,15 @@ module ContentServer
|
|
223
223
|
def handle_new_chunk(file_checksum, offset, content)
|
224
224
|
if offset == @streams[file_checksum].file.pos
|
225
225
|
FileReceiver.write_string_to_file(content, @streams[file_checksum].file)
|
226
|
-
|
227
|
-
|
228
|
-
|
226
|
+
if Params['log_debug_level'] >= 1 # added the condition here to avoid calculations
|
227
|
+
Log.debug1("Written already %s bytes, out of %s (%s%%)",
|
228
|
+
@streams[file_checksum].file.pos, @streams[file_checksum].size,
|
229
|
+
100.0*@streams[file_checksum].file.size/@streams[file_checksum].size)
|
230
|
+
end
|
229
231
|
return true
|
230
232
|
else
|
231
233
|
# Offset is wrong, send reset/resume copy from correct offset.
|
232
|
-
Log.warning("Received chunk with incorrect offset #{offset}, should " \
|
234
|
+
Log.warning("Received chunk with incorrect offset #{offset}, should " + \
|
233
235
|
"be #{@streams[file_checksum].file.pos}, file_checksum:#{file_checksum}")
|
234
236
|
@reset_copy.call(file_checksum, @streams[file_checksum].file.pos) unless @reset_copy.nil?
|
235
237
|
return false
|
@@ -246,7 +248,7 @@ module ContentServer
|
|
246
248
|
# Make the directory if does not exists.
|
247
249
|
path = FileReceiver.destination_filename(Params['backup_destination_folder'][0]['path'],
|
248
250
|
file_checksum)
|
249
|
-
Log.debug1("Moving tmp file
|
251
|
+
Log.debug1("Moving tmp file %s to %s", @streams[file_checksum].path, path)
|
250
252
|
file_dir = File.dirname(path)
|
251
253
|
FileUtils.makedirs(file_dir) unless File.directory?(file_dir)
|
252
254
|
# Move tmp file to permanent location.
|
@@ -259,7 +261,7 @@ module ContentServer
|
|
259
261
|
Log.debug1(message)
|
260
262
|
begin
|
261
263
|
File.rename(tmp_file_path, path)
|
262
|
-
Log.debug1("End move tmp file to permanent location
|
264
|
+
Log.debug1("End move tmp file to permanent location %s.", path)
|
263
265
|
@file_done_clb.call(local_file_checksum, path) unless @file_done_clb.nil?
|
264
266
|
rescue Exception => e
|
265
267
|
Log.warning("Could not move tmp file to permanent path #{path}." +
|
@@ -268,7 +270,7 @@ module ContentServer
|
|
268
270
|
else
|
269
271
|
begin
|
270
272
|
Log.error(message)
|
271
|
-
Log.debug1("Deleting tmp file:
|
273
|
+
Log.debug1("Deleting tmp file: %s", tmp_file_path)
|
272
274
|
File.delete(tmp_file_path)
|
273
275
|
rescue Exception => e
|
274
276
|
Log.warning("Could not delete tmp file from tmp path #{tmp_file_path}." +
|
@@ -282,7 +284,7 @@ module ContentServer
|
|
282
284
|
|
283
285
|
def self.write_string_to_file(str, file)
|
284
286
|
bytes_to_write = str.bytesize
|
285
|
-
Log.debug1("writing to file:
|
287
|
+
Log.debug1("writing to file: %s, %s bytes.", file, bytes_to_write)
|
286
288
|
while bytes_to_write > 0
|
287
289
|
bytes_to_write -= file.write(str)
|
288
290
|
end
|
@@ -36,7 +36,7 @@ module ContentServer
|
|
36
36
|
# Add content to copy process. If already in copy process or waiting for copy then skip.
|
37
37
|
# If no open places for copy then put in waiting list
|
38
38
|
def add_content(checksum, path)
|
39
|
-
Log.debug2("Try to add content
|
39
|
+
Log.debug2("Try to add content:%s to copy waiting list", checksum)
|
40
40
|
@keeper.synchronize{
|
41
41
|
# if content is being copied or waiting then skip it
|
42
42
|
if !@contents_under_copy[checksum]
|
@@ -48,16 +48,16 @@ module ContentServer
|
|
48
48
|
$process_vars.set('Copy File Queue Size', @copy_input_queue.size)
|
49
49
|
else
|
50
50
|
# no place in copy streams. Add to waiting list
|
51
|
-
Log.debug2("add content
|
51
|
+
Log.debug2("add content:%s to copy waiting list", checksum)
|
52
52
|
@contents_to_copy[checksum] = true # replace with a set
|
53
53
|
@contents_to_copy_queue.push([checksum, path])
|
54
54
|
$process_vars.set('contents to copy queue', @contents_to_copy_queue.size)
|
55
55
|
end
|
56
56
|
else
|
57
|
-
Log.debug2("content
|
57
|
+
Log.debug2("content:%s already in waiting list. skipping.", checksum)
|
58
58
|
end
|
59
59
|
else
|
60
|
-
Log.debug2("content
|
60
|
+
Log.debug2("content:%s is being copied. skipping.", checksum)
|
61
61
|
end
|
62
62
|
}
|
63
63
|
end
|
@@ -68,22 +68,22 @@ module ContentServer
|
|
68
68
|
if content_record
|
69
69
|
if !content_record[1]
|
70
70
|
path = content_record[0]
|
71
|
-
Log.debug1("Streaming to backup server. content:
|
71
|
+
Log.debug1("Streaming to backup server. content: %s path:%s.", checksum, path)
|
72
72
|
@file_streamer.start_streaming(checksum, path)
|
73
73
|
# updating Ack
|
74
74
|
content_record[1] = true
|
75
75
|
else
|
76
|
-
Log.warning("File already received ack:
|
76
|
+
Log.warning("File already received ack: %s", checksum)
|
77
77
|
end
|
78
78
|
else
|
79
|
-
Log.warning("File was aborted or copied:
|
79
|
+
Log.warning("File was aborted or copied: %s", checksum)
|
80
80
|
end
|
81
81
|
}
|
82
82
|
end
|
83
83
|
|
84
84
|
def remove_content(checksum)
|
85
85
|
@keeper.synchronize{
|
86
|
-
Log.debug3("removing checksum
|
86
|
+
Log.debug3("removing checksum:%s from contents under copy", checksum)
|
87
87
|
@contents_under_copy.delete(checksum)
|
88
88
|
$process_vars.set('contents under copy', @contents_under_copy.size)
|
89
89
|
#1 place is became available. Put another file in copy process if waiting
|
@@ -146,7 +146,7 @@ module ContentServer
|
|
146
146
|
# resend the ack request.
|
147
147
|
@copy_prepare = {}
|
148
148
|
@file_streamer = FileStreamer.new(method(:send_chunk))
|
149
|
-
Log.debug3("initialize FileCopyServer on port
|
149
|
+
Log.debug3("initialize FileCopyServer on port:%s", port)
|
150
150
|
@file_copy_manager = FileCopyManager.new(@copy_input_queue, @file_streamer)
|
151
151
|
end
|
152
152
|
|
@@ -157,7 +157,7 @@ module ContentServer
|
|
157
157
|
|
158
158
|
def receive_message(addr_info, message)
|
159
159
|
# Add ack message to copy queue.
|
160
|
-
Log.debug2("Content server Copy message received:
|
160
|
+
Log.debug2("Content server Copy message received: %s", message)
|
161
161
|
@copy_input_queue.push(message)
|
162
162
|
$process_vars.set('Copy File Queue Size', @copy_input_queue.size)
|
163
163
|
end
|
@@ -170,10 +170,10 @@ module ContentServer
|
|
170
170
|
Log.debug1 'Waiting on copy files events.'
|
171
171
|
(message_type, message_content) = @copy_input_queue.pop
|
172
172
|
$process_vars.set('Copy File Queue Size', @copy_input_queue.size)
|
173
|
-
Log.debug1("Content copy message
|
173
|
+
Log.debug1("Content copy message:Type:%s content:%s", message_type, message_content)
|
174
174
|
|
175
175
|
if message_type == :SEND_ACK_MESSAGE
|
176
|
-
Log.debug1("Sending ack for:
|
176
|
+
Log.debug1("Sending ack for: %s", message_content)
|
177
177
|
@backup_tcp.send_obj([:ACK_MESSAGE, [message_content, Time.now.to_i]])
|
178
178
|
elsif message_type == :COPY_MESSAGE
|
179
179
|
message_content.each_instance { |checksum, size, content_mod_time, instance_mod_time, server, path|
|
@@ -184,7 +184,7 @@ module ContentServer
|
|
184
184
|
# The timestamp is of local content server! not backup server!
|
185
185
|
timestamp, ack, checksum = message_content
|
186
186
|
Log.debug1("Ack (#{ack}) received for content: #{checksum}, timestamp: #{timestamp} " \
|
187
|
-
"now: #{Time.now.to_i}")
|
187
|
+
"now: #{Time.now.to_i}") if Params['log_debug_level'] >= 1 # adding to avoid Time.now
|
188
188
|
|
189
189
|
# Copy file if ack (does not exists on backup and not too much time passed)
|
190
190
|
if ack
|
@@ -192,7 +192,7 @@ module ContentServer
|
|
192
192
|
@file_copy_manager.receive_ack(checksum)
|
193
193
|
else
|
194
194
|
Log.debug1("Ack timed out span: #{Time.now.to_i - timestamp} > " \
|
195
|
-
|
195
|
+
"timeout: #{Params['ack_timeout']}") if Params['log_debug_level'] >= 1 # adding to avoid Time.now
|
196
196
|
# remove only content under copy
|
197
197
|
@file_copy_manager.remove_content(checksum)
|
198
198
|
end
|
@@ -207,8 +207,9 @@ module ContentServer
|
|
207
207
|
elsif message_type == :COPY_CHUNK
|
208
208
|
# We open the message here for printing info and deleting copy_prepare!
|
209
209
|
file_checksum, offset, file_size, content, content_checksum = message_content
|
210
|
-
Log.debug1("Send chunk for file
|
211
|
-
"filesize:
|
210
|
+
Log.debug1("Send chunk for file %s, offset: %s " \
|
211
|
+
"filesize: %s, checksum: %s",
|
212
|
+
file_checksum, offset, file_size, content_checksum)
|
212
213
|
# Blocking send.
|
213
214
|
@backup_tcp.send_obj([:COPY_CHUNK, message_content])
|
214
215
|
if content.nil? and content_checksum.nil?
|
@@ -216,13 +217,14 @@ module ContentServer
|
|
216
217
|
@file_copy_manager.remove_content(file_checksum)
|
217
218
|
end
|
218
219
|
elsif message_type == :ABORT_COPY
|
219
|
-
Log.debug1("Aborting file copy:
|
220
|
+
Log.debug1("Aborting file copy: %s", message_content)
|
220
221
|
@file_streamer.abort_streaming(message_content)
|
221
222
|
# remove only content under copy
|
222
223
|
@file_copy_manager.remove_content(message_content)
|
223
224
|
elsif message_type == :RESET_RESUME_COPY
|
224
225
|
(file_checksum, new_offset) = message_content
|
225
|
-
Log.debug1("Resetting/Resuming file (
|
226
|
+
Log.debug1("Resetting/Resuming file (%s) copy to %s",
|
227
|
+
file_checksum, new_offset)
|
226
228
|
@file_streamer.reset_streaming(file_checksum, new_offset)
|
227
229
|
else
|
228
230
|
Log.error("Copy event not supported: #{message_type}")
|
@@ -248,7 +250,7 @@ module ContentServer
|
|
248
250
|
end
|
249
251
|
end
|
250
252
|
@local_thread.abort_on_exception = true
|
251
|
-
Log.debug3("initialize FileCopyClient host
|
253
|
+
Log.debug3("initialize FileCopyClient host:%s port:%s", host, port)
|
252
254
|
end
|
253
255
|
|
254
256
|
def threads
|
@@ -271,7 +273,7 @@ module ContentServer
|
|
271
273
|
|
272
274
|
def done_copy(local_file_checksum, local_path)
|
273
275
|
$process_vars.inc('num_files_received')
|
274
|
-
Log.debug1("Done copy file:
|
276
|
+
Log.debug1("Done copy file: path %s, checksum %s", local_path, local_file_checksum)
|
275
277
|
end
|
276
278
|
|
277
279
|
def handle_message(message)
|
@@ -284,10 +286,10 @@ module ContentServer
|
|
284
286
|
# of ack. Note that it is being executed from the class thread only!
|
285
287
|
def handle(message)
|
286
288
|
message_type, message_content = message
|
287
|
-
Log.debug1("backup copy message: Type
|
289
|
+
Log.debug1("backup copy message: Type %s. message: %s", message_type, message_content)
|
288
290
|
if message_type == :SEND_COPY_MESSAGE
|
289
291
|
bytes_written = @tcp_client.send_obj([:COPY_MESSAGE, message_content])
|
290
|
-
Log.debug2("Sending copy message succeeded? bytes_written:
|
292
|
+
Log.debug2("Sending copy message succeeded? bytes_written: %s.", bytes_written)
|
291
293
|
elsif message_type == :COPY_CHUNK
|
292
294
|
if @file_receiver.receive_chunk(*message_content)
|
293
295
|
file_checksum, offset, file_size, content, content_checksum = message_content
|
@@ -301,7 +303,7 @@ module ContentServer
|
|
301
303
|
# check if checksum exists in final destination
|
302
304
|
dest_path = FileReceiver.destination_filename(Params['backup_destination_folder'][0]['path'], checksum)
|
303
305
|
need_to_copy = !File.exists?(dest_path)
|
304
|
-
Log.debug1("Returning ack for content:'
|
306
|
+
Log.debug1("Returning ack for content:'%s' timestamp:'%s' Ack:'%s'", checksum, timestamp, need_to_copy)
|
305
307
|
@tcp_client.send_obj([:ACK_MESSAGE, [timestamp,
|
306
308
|
need_to_copy,
|
307
309
|
checksum]])
|