content_server 0.0.10 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bin/backup_server +26 -14
- data/bin/content_server +26 -24
- data/lib/content_server.rb +10 -186
- data/lib/content_server/backup_server.rb +120 -0
- data/lib/content_server/content_receiver.rb +44 -45
- data/lib/content_server/content_server.rb +89 -0
- data/lib/content_server/file_streamer.rb +225 -220
- data/lib/content_server/queue_copy.rb +190 -182
- data/lib/content_server/queue_indexer.rb +91 -92
- data/lib/content_server/remote_content.rb +70 -72
- data/lib/content_server/version.rb +1 -3
- data/spec/content_server/content_server_spec.rb +10 -12
- data/spec/content_server/file_streamer_spec.rb +50 -52
- metadata +246 -33
@@ -5,199 +5,207 @@ require 'file_indexing/index_agent'
|
|
5
5
|
require 'log'
|
6
6
|
require 'networking/tcp'
|
7
7
|
|
8
|
-
module BBFS
|
9
|
-
module ContentServer
|
10
|
-
Params.integer('ack_timeout', 5, 'Timeout of ack from backup server in seconds.')
|
11
|
-
|
12
|
-
# Copy message types.
|
13
|
-
:ACK_MESSAGE
|
14
|
-
:COPY_MESSAGE
|
15
|
-
:SEND_COPY_MESSAGE
|
16
|
-
:COPY_CHUNK
|
17
|
-
:COPY_CHUNK_FROM_REMOTE
|
18
|
-
:ABORT_COPY # Asks the sender to abort file copy.
|
19
|
-
:RESET_RESUME_COPY # Sends the stream sender to resend chunk or resume from different offset.
|
20
|
-
|
21
|
-
# Simple copier, gets inputs events (files to copy), requests ack from backup to copy
|
22
|
-
# then copies one file.
|
23
|
-
class FileCopyServer
|
24
|
-
def initialize(copy_input_queue, port)
|
25
|
-
# Local simple tcp connection.
|
26
|
-
@backup_tcp = Networking::TCPServer.new(port, method(:receive_message))
|
27
|
-
@copy_input_queue = copy_input_queue
|
28
|
-
# Stores for each checksum, the file source path.
|
29
|
-
# TODO(kolman): If there are items in copy_prepare which timeout (don't get ack),
|
30
|
-
# resend the ack request.
|
31
|
-
@copy_prepare = {}
|
32
|
-
@file_streamer = FileStreamer.new(method(:send_chunk))
|
33
|
-
end
|
34
|
-
|
35
|
-
def send_chunk(*arg)
|
36
|
-
@copy_input_queue.push([:COPY_CHUNK, arg])
|
37
|
-
end
|
38
8
|
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
9
|
+
module ContentServer
|
10
|
+
Params.integer('ack_timeout', 5, 'Timeout of ack from backup server in seconds.')
|
11
|
+
|
12
|
+
# Copy message types.
|
13
|
+
:ACK_MESSAGE
|
14
|
+
:COPY_MESSAGE
|
15
|
+
:SEND_COPY_MESSAGE
|
16
|
+
:COPY_CHUNK
|
17
|
+
:COPY_CHUNK_FROM_REMOTE
|
18
|
+
:ABORT_COPY # Asks the sender to abort file copy.
|
19
|
+
:RESET_RESUME_COPY # Sends the stream sender to resend chunk or resume from different offset.
|
20
|
+
|
21
|
+
# Simple copier, gets inputs events (files to copy), requests ack from backup to copy
|
22
|
+
# then copies one file.
|
23
|
+
class FileCopyServer
|
24
|
+
def initialize(copy_input_queue, port)
|
25
|
+
# Local simple tcp connection.
|
26
|
+
@backup_tcp = Networking::TCPServer.new(port, method(:receive_message))
|
27
|
+
@copy_input_queue = copy_input_queue
|
28
|
+
# Stores for each checksum, the file source path.
|
29
|
+
# TODO(kolman): If there are items in copy_prepare which timeout (don't get ack),
|
30
|
+
# resend the ack request.
|
31
|
+
@copy_prepare = {}
|
32
|
+
@file_streamer = FileStreamer.new(method(:send_chunk))
|
33
|
+
Log.debug2("initialize FileCopyServer on port:#{port}")
|
34
|
+
end
|
35
|
+
|
36
|
+
def send_chunk(*arg)
|
37
|
+
@copy_input_queue.push([:COPY_CHUNK, arg])
|
38
|
+
end
|
39
|
+
|
40
|
+
def receive_message(addr_info, message)
|
41
|
+
# Add ack message to copy queue.
|
42
|
+
Log.info("Master Copy Server message received: #{message}")
|
43
|
+
@copy_input_queue.push(message)
|
44
|
+
end
|
45
|
+
|
46
|
+
def run()
|
47
|
+
threads = []
|
48
|
+
threads << @backup_tcp.tcp_thread if @backup_tcp != nil
|
49
|
+
threads << Thread.new do
|
50
|
+
while true do
|
51
|
+
Log.info 'Waiting on copy files events.'
|
52
|
+
message_type, message_content = @copy_input_queue.pop
|
53
|
+
|
54
|
+
if message_type == :COPY_MESSAGE
|
55
|
+
Log.info "Copy file event: #{message_content}"
|
56
|
+
# Prepare source,dest map for copy.
|
57
|
+
message_content.instances.each { |key, instance|
|
58
|
+
# If not already sending.
|
59
|
+
if !@copy_prepare.key?(instance.checksum) || !@copy_prepare[instance.checksum][1]
|
60
|
+
@copy_prepare[instance.checksum] = [instance.full_path, false]
|
61
|
+
Log.info("Sending ack for: #{instance.checksum}")
|
62
|
+
@backup_tcp.send_obj([:ACK_MESSAGE, [instance.checksum, Time.now.to_i]])
|
63
|
+
end
|
64
|
+
}
|
65
|
+
elsif message_type == :ACK_MESSAGE
|
66
|
+
# Received ack from backup, copy file if all is good.
|
67
|
+
# The timestamp is of local content server! not backup server!
|
68
|
+
timestamp, ack, checksum = message_content
|
44
69
|
|
45
|
-
|
46
|
-
threads = []
|
47
|
-
threads << @backup_tcp.tcp_thread if @backup_tcp != nil
|
48
|
-
threads << Thread.new do
|
49
|
-
while true do
|
50
|
-
Log.info 'Waiting on copy files events.'
|
51
|
-
message_type, message_content = @copy_input_queue.pop
|
52
|
-
|
53
|
-
if message_type == :COPY_MESSAGE
|
54
|
-
Log.info "Copy file event: #{message_content}"
|
55
|
-
# Prepare source,dest map for copy.
|
56
|
-
message_content.instances.each { |key, instance|
|
57
|
-
# If not already sending.
|
58
|
-
if !@copy_prepare.key?(instance.checksum) || !@copy_prepare[instance.checksum][1]
|
59
|
-
@copy_prepare[instance.checksum] = [instance.full_path, false]
|
60
|
-
Log.info("Sending ack for: #{instance.checksum}")
|
61
|
-
@backup_tcp.send_obj([:ACK_MESSAGE, [instance.checksum, Time.now.to_i]])
|
62
|
-
end
|
63
|
-
}
|
64
|
-
elsif message_type == :ACK_MESSAGE
|
65
|
-
# Received ack from backup, copy file if all is good.
|
66
|
-
# The timestamp is of local content server! not backup server!
|
67
|
-
timestamp, ack, checksum = message_content
|
68
|
-
|
69
|
-
Log.info("Ack (#{ack}) received for: #{checksum}, timestamp: #{timestamp} " \
|
70
|
+
Log.info("Ack (#{ack}) received for: #{checksum}, timestamp: #{timestamp} " \
|
70
71
|
"now: #{Time.now.to_i}")
|
71
72
|
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
else
|
77
|
-
path = @copy_prepare[checksum][0]
|
78
|
-
Log.info "Streaming file: #{checksum} #{path}."
|
79
|
-
@file_streamer.start_streaming(checksum, path)
|
80
|
-
# Ack received, setting prepare to true
|
81
|
-
@copy_prepare[checksum][1] = true
|
82
|
-
end
|
73
|
+
# Copy file if ack (does not exists on backup and not too much time passed)
|
74
|
+
if ack && (Time.now.to_i - timestamp < Params['ack_timeout'])
|
75
|
+
if !@copy_prepare.key?(checksum) || @copy_prepare[checksum][1]
|
76
|
+
Log.warning("File was aborted, copied, or started copy just now: #{checksum}")
|
83
77
|
else
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
@file_streamer.copy_another_chuck(checksum)
|
90
|
-
elsif message_type == :COPY_CHUNK
|
91
|
-
# We open the message here for printing info and deleting copy_prepare!
|
92
|
-
file_checksum, offset, file_size, content, content_checksum = message_content
|
93
|
-
Log.info("Send chunk for file #{file_checksum}, offset: #{offset} " \
|
94
|
-
"filesize: #{file_size}.")
|
95
|
-
# Blocking send.
|
96
|
-
@backup_tcp.send_obj([:COPY_CHUNK, message_content])
|
97
|
-
if content.nil? and content_checksum.nil?
|
98
|
-
@copy_prepare.delete(file_checksum)
|
78
|
+
path = @copy_prepare[checksum][0]
|
79
|
+
Log.info "Streaming file: #{checksum} #{path}."
|
80
|
+
@file_streamer.start_streaming(checksum, path)
|
81
|
+
# Ack received, setting prepare to true
|
82
|
+
@copy_prepare[checksum][1] = true
|
99
83
|
end
|
100
|
-
elsif message_type == :ABORT_COPY
|
101
|
-
Log.info("Aborting file copy: #{message_content}")
|
102
|
-
if @copy_prepare.key?(message_content)
|
103
|
-
Log.info("Aborting: #{@copy_prepare[message_content][0]}")
|
104
|
-
@copy_prepare.delete(message_content)
|
105
|
-
end
|
106
|
-
@file_streamer.abort_streaming(message_content)
|
107
|
-
elsif message_type == :RESET_RESUME_COPY
|
108
|
-
file_checksum, new_offset = message_content
|
109
|
-
Log.info("Resetting/Resuming file (#{file_checksum}) copy to #{new_offset}")
|
110
|
-
@file_streamer.reset_streaming(file_checksum, new_offset)
|
111
84
|
else
|
112
|
-
Log.
|
113
|
-
|
114
|
-
|
115
|
-
|
85
|
+
Log.debug1("Ack timed out span: #{Time.now.to_i - timestamp} > " \
|
86
|
+
"timeout: #{Params['ack_timeout']}")
|
87
|
+
end
|
88
|
+
elsif message_type == :COPY_CHUNK_FROM_REMOTE
|
89
|
+
checksum = message_content
|
90
|
+
@file_streamer.copy_another_chuck(checksum)
|
91
|
+
elsif message_type == :COPY_CHUNK
|
92
|
+
# We open the message here for printing info and deleting copy_prepare!
|
93
|
+
file_checksum, offset, file_size, content, content_checksum = message_content
|
94
|
+
Log.debug1("Send chunk for file #{file_checksum}, offset: #{offset} " \
|
95
|
+
"filesize: #{file_size}.")
|
96
|
+
# Blocking send.
|
97
|
+
@backup_tcp.send_obj([:COPY_CHUNK, message_content])
|
98
|
+
if content.nil? and content_checksum.nil?
|
99
|
+
@copy_prepare.delete(file_checksum)
|
100
|
+
end
|
101
|
+
elsif message_type == :ABORT_COPY
|
102
|
+
Log.info("Aborting file copy: #{message_content}")
|
103
|
+
if @copy_prepare.key?(message_content)
|
104
|
+
Log.info("Aborting: #{@copy_prepare[message_content][0]}")
|
105
|
+
@copy_prepare.delete(message_content)
|
106
|
+
end
|
107
|
+
@file_streamer.abort_streaming(message_content)
|
108
|
+
elsif message_type == :RESET_RESUME_COPY
|
109
|
+
file_checksum, new_offset = message_content
|
110
|
+
Log.info("Resetting/Resuming file (#{file_checksum}) copy to #{new_offset}")
|
111
|
+
@file_streamer.reset_streaming(file_checksum, new_offset)
|
112
|
+
else
|
113
|
+
Log.error("Copy event not supported: #{message_type}")
|
114
|
+
end # handle messages here
|
116
115
|
end
|
116
|
+
Log.error("Should not reach here, loop should continue.")
|
117
117
|
end
|
118
|
-
end
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
118
|
+
end
|
119
|
+
end # class QueueCopy
|
120
|
+
|
121
|
+
class FileCopyClient
|
122
|
+
def initialize(host, port, dynamic_content_data, process_variables)
|
123
|
+
@local_queue = Queue.new
|
124
|
+
@dynamic_content_data = dynamic_content_data
|
125
|
+
@tcp_client = Networking::TCPClient.new(host, port, method(:handle_message))
|
126
|
+
@file_receiver = FileReceiver.new(method(:done_copy),
|
127
|
+
method(:abort_copy),
|
128
|
+
method(:reset_copy))
|
129
|
+
@local_thread = Thread.new do
|
130
|
+
loop do
|
131
|
+
handle(@local_queue.pop)
|
132
132
|
end
|
133
|
-
@local_thread.abort_on_exception = true
|
134
|
-
end
|
135
|
-
|
136
|
-
def threads
|
137
|
-
ret = [@local_thread]
|
138
|
-
ret << @tcp_server.tcp_thread if @tcp_server != nil
|
139
|
-
return ret
|
140
|
-
end
|
141
|
-
|
142
|
-
def request_copy(content_data)
|
143
|
-
handle_message([:SEND_COPY_MESSAGE, content_data])
|
144
133
|
end
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
134
|
+
@local_thread.abort_on_exception = true
|
135
|
+
@process_variables = process_variables
|
136
|
+
Log.debug2("initialize FileCopyClient host:#{host} port:#{port}")
|
137
|
+
end
|
138
|
+
|
139
|
+
def threads
|
140
|
+
ret = [@local_thread]
|
141
|
+
ret << @tcp_client.tcp_thread if @tcp_client != nil
|
142
|
+
return ret
|
143
|
+
end
|
144
|
+
|
145
|
+
def request_copy(content_data)
|
146
|
+
handle_message([:SEND_COPY_MESSAGE, content_data])
|
147
|
+
end
|
148
|
+
|
149
|
+
def abort_copy(checksum)
|
150
|
+
handle_message([:ABORT_COPY, checksum])
|
151
|
+
end
|
152
|
+
|
153
|
+
def reset_copy(checksum, new_offset)
|
154
|
+
handle_message([:RESET_RESUME_COPY, [checksum, new_offset]])
|
155
|
+
end
|
156
|
+
|
157
|
+
def done_copy(local_file_checksum, local_path)
|
158
|
+
add_process_variables_info()
|
159
|
+
Log.info("Done copy file: #{local_path}, #{local_file_checksum}")
|
160
|
+
end
|
161
|
+
|
162
|
+
def add_process_variables_info()
|
163
|
+
@process_variables.inc('num_files_received')
|
164
|
+
end
|
165
|
+
|
166
|
+
def handle_message(message)
|
167
|
+
Log.debug2('QueueFileReceiver handle message')
|
168
|
+
@local_queue.push(message)
|
169
|
+
end
|
170
|
+
|
171
|
+
# This is a function which receives the messages (file or ack) and return answer in case
|
172
|
+
# of ack. Note that it is being executed from the class thread only!
|
173
|
+
def handle(message)
|
174
|
+
message_type, message_content = message
|
175
|
+
if message_type == :SEND_COPY_MESSAGE
|
176
|
+
Log.debug1("Requesting file (content data) to copy.")
|
177
|
+
Log.debug3("File requested: #{message_content.to_s}")
|
178
|
+
bytes_written = @tcp_client.send_obj([:COPY_MESSAGE, message_content])
|
179
|
+
Log.debug1("Sending copy message succeeded? bytes_written: #{bytes_written}.")
|
180
|
+
elsif message_type == :COPY_CHUNK
|
181
|
+
Log.debug1('Chunk received.')
|
182
|
+
if @file_receiver.receive_chunk(*message_content)
|
183
|
+
file_checksum, offset, file_size, content, content_checksum = message_content
|
184
|
+
@tcp_client.send_obj([:COPY_CHUNK_FROM_REMOTE, file_checksum])
|
192
185
|
end
|
186
|
+
elsif message_type == :ACK_MESSAGE
|
187
|
+
checksum, timestamp = message_content
|
188
|
+
# Here we should check file existence
|
189
|
+
Log.debug1("Returning ack for: #{checksum}, timestamp: #{timestamp}")
|
190
|
+
Log.debug1("Ack: #{!@dynamic_content_data.exists?(checksum)}")
|
191
|
+
@tcp_client.send_obj([:ACK_MESSAGE, [timestamp,
|
192
|
+
!@dynamic_content_data.exists?(checksum),
|
193
|
+
checksum]])
|
194
|
+
elsif message_type == :ABORT_COPY
|
195
|
+
@tcp_client.send_obj([:ABORT_COPY, message_content])
|
196
|
+
elsif message_type == :RESET_RESUME_COPY
|
197
|
+
@tcp_client.send_obj([:RESET_RESUME_COPY, message_content])
|
198
|
+
else
|
199
|
+
Log.error("Unexpected message type: #{message_type}")
|
193
200
|
end
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
end
|
202
|
-
end
|
201
|
+
end
|
202
|
+
|
203
|
+
# Creates destination filename for backup server, input is base folder and sha1.
|
204
|
+
# for example: folder:/mnt/hd1/bbbackup, sha1:d0be2dc421be4fcd0172e5afceea3970e2f3d940
|
205
|
+
# dest filename: /mnt/hd1/bbbackup/d0/be/2d/d0be2dc421be4fcd0172e5afceea3970e2f3d940
|
206
|
+
def self.destination_filename(folder, sha1)
|
207
|
+
File.join(folder, sha1[0,2], sha1[2,2], sha1)
|
208
|
+
end
|
209
|
+
end # class QueueFileReceiver
|
203
210
|
end
|
211
|
+
|
@@ -2,109 +2,108 @@ require 'file_indexing/index_agent'
|
|
2
2
|
require 'file_indexing/indexer_patterns'
|
3
3
|
require 'log'
|
4
4
|
|
5
|
-
module
|
6
|
-
module ContentServer
|
5
|
+
module ContentServer
|
7
6
|
|
8
|
-
|
9
|
-
|
10
|
-
|
7
|
+
# Simple indexer, gets inputs events (files to index) and outputs
|
8
|
+
# content data updates into output queue.
|
9
|
+
class QueueIndexer
|
11
10
|
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
11
|
+
def initialize(input_queue, output_queue, content_data_path)
|
12
|
+
@input_queue = input_queue
|
13
|
+
@output_queue = output_queue
|
14
|
+
@content_data_path = content_data_path
|
15
|
+
end
|
17
16
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
end
|
17
|
+
def run
|
18
|
+
server_content_data = ContentData::ContentData.new
|
19
|
+
# Shallow check content data files.
|
20
|
+
tmp_content_data = ContentData::ContentData.new
|
21
|
+
tmp_content_data.from_file(@content_data_path) if File.exists?(@content_data_path)
|
22
|
+
tmp_content_data.instances.each_value do |instance|
|
23
|
+
# Skipp instances (files) which did not pass the shallow check.
|
24
|
+
Log.info('Shallow checking content data:')
|
25
|
+
if shallow_check(instance)
|
26
|
+
Log.info("exists: #{instance.full_path}")
|
27
|
+
server_content_data.add_content(tmp_content_data.contents[instance.checksum])
|
28
|
+
server_content_data.add_instance(instance)
|
29
|
+
else
|
30
|
+
Log.info("changed: #{instance.full_path}")
|
31
|
+
# Add non existing and changed files to index queue.
|
32
|
+
@input_queue.push([FileMonitoring::FileStatEnum::STABLE, instance.full_path])
|
35
33
|
end
|
34
|
+
end
|
36
35
|
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
36
|
+
# Start indexing on demand and write changes to queue
|
37
|
+
thread = Thread.new do
|
38
|
+
while true do
|
39
|
+
Log.info 'Waiting on index input queue.'
|
40
|
+
state, is_dir, path = @input_queue.pop
|
41
|
+
Log.info "event: #{state}, #{is_dir}, #{path}."
|
43
42
|
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
43
|
+
# index files and add to copy queue
|
44
|
+
# delete directory with it's sub files
|
45
|
+
# delete file
|
46
|
+
if state == FileMonitoring::FileStatEnum::STABLE && !is_dir
|
47
|
+
Log.info "Indexing content #{path}."
|
48
|
+
index_agent = FileIndexing::IndexAgent.new
|
49
|
+
indexer_patterns = FileIndexing::IndexerPatterns.new
|
50
|
+
indexer_patterns.add_pattern(path)
|
51
|
+
index_agent.index(indexer_patterns, server_content_data)
|
52
|
+
Log.info("Failed files: #{index_agent.failed_files.to_a.join(',')}.") \
|
54
53
|
if !index_agent.failed_files.empty?
|
55
|
-
|
56
|
-
|
57
|
-
|
54
|
+
Log.info("indexed content #{index_agent.indexed_content}.")
|
55
|
+
server_content_data.merge index_agent.indexed_content
|
56
|
+
elsif ((state == FileMonitoring::FileStatEnum::NON_EXISTING ||
|
58
57
|
state == FileMonitoring::FileStatEnum::CHANGED) && !is_dir)
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
end
|
58
|
+
# If file content changed, we should remove old instance.
|
59
|
+
key = FileIndexing::IndexAgent.global_path(path)
|
60
|
+
# Check if deleted file exists at content data.
|
61
|
+
Log.info("Instance to remove: #{key}")
|
62
|
+
if server_content_data.instances.key?(key)
|
63
|
+
instance_to_remove = server_content_data.instances[key]
|
64
|
+
# Remove file from content data only if it does not pass the shallow check, i.e.,
|
65
|
+
# content has changed/removed.
|
66
|
+
if !shallow_check(instance_to_remove)
|
67
|
+
content_to_remove = server_content_data.contents[instance_to_remove.checksum]
|
68
|
+
# Remove the deleted instance.
|
69
|
+
content_data_to_remove = ContentData::ContentData.new
|
70
|
+
content_data_to_remove.add_content(content_to_remove)
|
71
|
+
content_data_to_remove.add_instance(instance_to_remove)
|
72
|
+
# Remove the file.
|
73
|
+
server_content_data = ContentData::ContentData.remove_instances(
|
74
|
+
content_data_to_remove, server_content_data)
|
77
75
|
end
|
78
|
-
elsif state == FileMonitoring::FileStatEnum::NON_EXISTING && is_dir
|
79
|
-
Log.info("NonExisting/Changed: #{path}")
|
80
|
-
# Remove directory but only when non-existing.
|
81
|
-
Log.info("Directory to remove: #{path}")
|
82
|
-
global_dir = FileIndexing::IndexAgent.global_path(path)
|
83
|
-
server_content_data = ContentData::ContentData.remove_directory(
|
84
|
-
server_content_data, global_dir)
|
85
|
-
else
|
86
|
-
Log.info("This case should not be handled: #{state}, #{is_dir}, #{path}.")
|
87
76
|
end
|
88
|
-
|
89
|
-
Log.info
|
90
|
-
|
77
|
+
elsif state == FileMonitoring::FileStatEnum::NON_EXISTING && is_dir
|
78
|
+
Log.info("NonExisting/Changed: #{path}")
|
79
|
+
# Remove directory but only when non-existing.
|
80
|
+
Log.info("Directory to remove: #{path}")
|
81
|
+
global_dir = FileIndexing::IndexAgent.global_path(path)
|
82
|
+
server_content_data = ContentData::ContentData.remove_directory(
|
83
|
+
server_content_data, global_dir)
|
84
|
+
else
|
85
|
+
Log.info("This case should not be handled: #{state}, #{is_dir}, #{path}.")
|
86
|
+
end
|
87
|
+
# TODO(kolman): Don't write to file each change?
|
88
|
+
Log.info "Writing server content data to #{@content_data_path}."
|
89
|
+
server_content_data.to_file(@content_data_path)
|
91
90
|
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
91
|
+
Log.info 'Adding server content data to queue.'
|
92
|
+
@output_queue.push(ContentData::ContentData.new(server_content_data))
|
93
|
+
end # while true do
|
94
|
+
end # Thread.new do
|
95
|
+
thread
|
96
|
+
end # def run
|
98
97
|
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
98
|
+
# Check file existence, check it's size and modification date.
|
99
|
+
# If something wrong reindex the file and update content data.
|
100
|
+
def shallow_check(instance)
|
101
|
+
shallow_instance = FileIndexing::IndexAgent.create_shallow_instance(instance.full_path)
|
102
|
+
return false unless shallow_instance
|
103
|
+
return (shallow_instance.size == instance.size &&
|
104
|
+
shallow_instance.modification_time == instance.modification_time)
|
105
|
+
end
|
107
106
|
|
108
|
-
|
109
|
-
end
|
107
|
+
end # class QueueIndexer
|
110
108
|
end
|
109
|
+
|