content_server 1.1.0 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/backup_server +8 -20
- data/bin/content_server +8 -20
- data/bin/testing_memory +60 -0
- data/bin/testing_server +57 -0
- data/ext/run_in_background/mkrf_conf.rb +34 -0
- data/lib/content_data/content_data.rb +613 -0
- data/lib/content_data/version.rb +3 -0
- data/lib/content_data.rb +6 -0
- data/lib/content_server/backup_server.rb +65 -86
- data/lib/content_server/content_server.rb +47 -77
- data/lib/content_server/file_streamer.rb +27 -33
- data/lib/content_server/queue_copy.rb +154 -49
- data/lib/content_server/queue_indexer.rb +19 -11
- data/lib/content_server/remote_content.rb +41 -23
- data/lib/content_server/server.rb +91 -0
- data/lib/content_server/version.rb +1 -1
- data/lib/content_server.rb +0 -15
- data/lib/email/email.rb +87 -0
- data/lib/email/version.rb +3 -0
- data/lib/email.rb +4 -0
- data/lib/file_copy/copy.rb +68 -0
- data/lib/file_copy/version.rb +4 -0
- data/lib/file_copy.rb +4 -0
- data/lib/file_indexing/index_agent.rb +170 -0
- data/lib/file_indexing/indexer_patterns.rb +72 -0
- data/lib/file_indexing/version.rb +3 -0
- data/lib/file_indexing.rb +9 -0
- data/lib/file_monitoring/file_monitoring.rb +105 -0
- data/lib/file_monitoring/monitor_path.rb +304 -0
- data/lib/file_monitoring/version.rb +3 -0
- data/lib/file_monitoring.rb +29 -0
- data/lib/file_utils/file_generator/README +97 -0
- data/lib/file_utils/file_generator/file_generator.rb +156 -0
- data/lib/file_utils/file_utils.rb +260 -0
- data/lib/file_utils/version.rb +3 -0
- data/lib/file_utils.rb +4 -0
- data/lib/log/version.rb +3 -0
- data/lib/log.rb +188 -0
- data/lib/networking/tcp.rb +213 -0
- data/lib/networking/version.rb +3 -0
- data/lib/networking.rb +4 -0
- data/lib/params/version.rb +3 -0
- data/lib/params.rb +419 -0
- data/lib/process_monitoring/monitoring.rb +85 -0
- data/lib/process_monitoring/monitoring_info.rb +79 -0
- data/lib/process_monitoring/send_email.rb +40 -0
- data/lib/process_monitoring/thread_safe_hash.rb +77 -0
- data/lib/process_monitoring/version.rb +3 -0
- data/lib/process_monitoring.rb +6 -0
- data/lib/run_in_background/version.rb +3 -0
- data/lib/run_in_background.rb +432 -0
- data/lib/testing_memory/testing_memory.rb +187 -0
- data/lib/testing_server/testing_server.rb +236 -0
- data/lib/testing_server/version.rb +3 -0
- data/lib/testing_server.rb +12 -0
- data/lib/validations/index_validations.rb +106 -0
- data/lib/validations/version.rb +3 -0
- data/lib/validations.rb +4 -0
- data/spec/content_data/validations_spec.rb +113 -0
- data/spec/file_copy/copy_spec.rb +54 -0
- data/spec/file_indexing/index_agent_spec.rb +53 -0
- data/spec/networking/tcp_spec.rb +95 -0
- data/spec/validations/index_validations_spec.rb +77 -0
- data/test/content_data/content_data_test.rb +290 -0
- data/test/file_generator/file_generator_spec.rb +84 -0
- data/test/file_indexing/index_agent_test/New.txt +0 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/bin/libexslt.dll +0 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/bin/libxslt.dll +0 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/bin/xsltproc.exe +0 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libexslt/exslt.h +102 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libexslt/exsltconfig.h +73 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libexslt/exsltexports.h +140 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libexslt/libexslt.h +29 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/attributes.h +38 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/documents.h +93 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/extensions.h +262 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/extra.h +80 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/functions.h +78 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/imports.h +75 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/keys.h +53 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/libxslt.h +30 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/namespaces.h +68 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/numbersInternals.h +69 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/pattern.h +81 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/preproc.h +43 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/security.h +104 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/templates.h +77 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/transform.h +207 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/trio.h +216 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/triodef.h +220 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/variables.h +91 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/win32config.h +101 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/xslt.h +103 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/xsltInternals.h +1967 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/xsltconfig.h +172 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/xsltexports.h +142 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/xsltlocale.h +57 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/xsltutils.h +309 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/xsltwin32config.h +105 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/lib/libexslt.lib +0 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/lib/libexslt_a.lib +0 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/lib/libxslt.lib +0 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/lib/libxslt_a.lib +0 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/readme.txt +22 -0
- data/test/file_indexing/index_agent_test/patterns.input +3 -0
- data/test/file_indexing/index_agent_test.rb +51 -0
- data/test/file_monitoring/file_monitoring_test/conf.yml +4 -0
- data/test/file_monitoring/file_monitoring_test/conf_win32.yml +5 -0
- data/test/file_monitoring/file_monitoring_test/log +56 -0
- data/test/file_monitoring/file_monitoring_test.rb +0 -0
- data/test/file_monitoring/monitor_path_test/dir1000/test_file.1000 +1000 -0
- data/test/file_monitoring/monitor_path_test/dir1000/test_file.1000.0 +1000 -0
- data/test/file_monitoring/monitor_path_test/dir1000/test_file.1000.1 +1000 -0
- data/test/file_monitoring/monitor_path_test/dir1500/test_file.1500 +1500 -0
- data/test/file_monitoring/monitor_path_test/dir1500/test_file.1500.0 +1500 -0
- data/test/file_monitoring/monitor_path_test/dir1500/test_file.1500.1 +1500 -0
- data/test/file_monitoring/monitor_path_test/test_file.500 +500 -0
- data/test/file_monitoring/monitor_path_test/test_file.500.0 +500 -0
- data/test/file_monitoring/monitor_path_test/test_file.500.1 +500 -0
- data/test/file_monitoring/monitor_path_test.rb +153 -0
- data/test/file_utils/fileutil_mksymlink_test/dir1000/dir1500/test_file.1500 +1500 -0
- data/test/file_utils/fileutil_mksymlink_test/dir1000/dir1500/test_file.1500.0 +1500 -0
- data/test/file_utils/fileutil_mksymlink_test/dir1000/dir1500/test_file.1500.1 +1500 -0
- data/test/file_utils/fileutil_mksymlink_test/dir1000/test_file.1000 +1000 -0
- data/test/file_utils/fileutil_mksymlink_test/dir1000/test_file.1000.0 +1000 -0
- data/test/file_utils/fileutil_mksymlink_test/dir1000/test_file.1000.1 +1000 -0
- data/test/file_utils/fileutil_mksymlink_test/test_file.500 +500 -0
- data/test/file_utils/fileutil_mksymlink_test/test_file.500.0 +500 -0
- data/test/file_utils/fileutil_mksymlink_test/test_file.500.1 +500 -0
- data/test/file_utils/fileutil_mksymlink_test.rb +125 -0
- data/test/file_utils/time_modification_test.rb +132 -0
- data/test/params/params_spec.rb +280 -0
- data/test/params/params_test.rb +43 -0
- data/test/run_in_background/run_in_background_test.rb +122 -0
- data/test/run_in_background/test_app +57 -0
- metadata +272 -132
- data/lib/content_server/globals.rb +0 -10
@@ -0,0 +1,260 @@
|
|
1
|
+
# Author:
|
2
|
+
# originator: Kolman Vornovizky (kolmanv@gmail.com)
|
3
|
+
# update: Alexey Nemytov (alexeyn66@gmail.com) 31/07/2012
|
4
|
+
# This file is the consist utility functionality for BBFS
|
5
|
+
# Such as automatic file generation, symlink, etc.
|
6
|
+
# Run from bbfs> ruby -Ilib bin/file_utils --command=generate_files --log_write_to_console=true --log_debug_level=3
|
7
|
+
|
8
|
+
require 'content_data'
|
9
|
+
require 'file_indexing'
|
10
|
+
require 'file_utils'
|
11
|
+
require 'file_utils/file_generator/file_generator'
|
12
|
+
|
13
|
+
require 'log'
|
14
|
+
require 'params'
|
15
|
+
|
16
|
+
module FileUtils
|
17
|
+
Params.string 'command', nil ,'path'
|
18
|
+
Params.string 'ref_cd', nil ,'reference path'
|
19
|
+
Params.string 'base_cd', nil ,'base path'
|
20
|
+
Params.string 'dest', nil ,'destination path'
|
21
|
+
Params.string 'dest_server', nil ,'server name'
|
22
|
+
Params.string 'patterns', nil ,'patterns path'
|
23
|
+
Params.string 'cd', nil ,'path'
|
24
|
+
Params.string 'cd_a', nil ,'a path'
|
25
|
+
Params.string 'cd_b', nil ,'b path'
|
26
|
+
Params.string 'cd_out', nil ,'output path'
|
27
|
+
Params.string 'cd_in', nil ,'input path'
|
28
|
+
Params.string 'exist_cd', nil, 'exist_path'
|
29
|
+
#Params.string 'config_file', 'path' ,'configuration file path'
|
30
|
+
class FileUtils
|
31
|
+
def FileUtils.run
|
32
|
+
if Params['command'] == 'mksymlink'
|
33
|
+
if Params['ref_cd'].nil?
|
34
|
+
Log.error ("--ref_cd is not set")
|
35
|
+
return
|
36
|
+
end
|
37
|
+
ref_cd = ContentData.new()
|
38
|
+
ref_cd.from_file(Params['ref_cd'])
|
39
|
+
if ref_cd.nil?
|
40
|
+
Log.error ("Error loading content data ref_cd=%s" % Params['ref_cd'])
|
41
|
+
return
|
42
|
+
end
|
43
|
+
|
44
|
+
if Params['base_cd'].nil?
|
45
|
+
Log.error ("--base_cd is not set")
|
46
|
+
return
|
47
|
+
end
|
48
|
+
base_cd = ContentData.new()
|
49
|
+
base_cd.from_file(Params['base_cd'])
|
50
|
+
if base_cd.nil?
|
51
|
+
Log.error ("Error loading content data base_cd=%s" % Params['base_cd'])
|
52
|
+
return
|
53
|
+
end
|
54
|
+
|
55
|
+
if Params['dest'].nil?
|
56
|
+
Log.error ("--dest is not set")
|
57
|
+
return
|
58
|
+
end
|
59
|
+
|
60
|
+
not_found = nil
|
61
|
+
begin
|
62
|
+
not_found = FileUtil.mksymlink(ref_cd, base_cd, Params['dest'])
|
63
|
+
rescue NotImplementedError
|
64
|
+
Log.error ("symlinks are unimplemented on this machine")
|
65
|
+
return nil
|
66
|
+
end
|
67
|
+
return not_found
|
68
|
+
elsif (Params['command'] == "merge" or
|
69
|
+
Params['command'] == "intersect" or
|
70
|
+
Params['command'] == "minus")
|
71
|
+
|
72
|
+
if Params['cd_a'].nil?
|
73
|
+
Log.error ("--cd_a is not set")
|
74
|
+
return
|
75
|
+
end
|
76
|
+
cd_a = ContentData.new()
|
77
|
+
cd_a.from_file(Params['cd_a'])
|
78
|
+
if cd_a.nil?
|
79
|
+
Log.error ("Error loading content data cd_a=%s" % Params['cd_a'])
|
80
|
+
return
|
81
|
+
end
|
82
|
+
|
83
|
+
if Params['cd_b'].nil?
|
84
|
+
Log.error ("--cd_b is not set")
|
85
|
+
return
|
86
|
+
end
|
87
|
+
cd_b = ContentData.new()
|
88
|
+
cd_b.from_file(Params['cd_b'])
|
89
|
+
if cd_b.nil?
|
90
|
+
Log.error ("Error loading content data cd_b=%s" % Params['cd_b'])
|
91
|
+
return
|
92
|
+
end
|
93
|
+
|
94
|
+
if Params['cd_b'].nil?
|
95
|
+
Log.error ("--dest is not set")
|
96
|
+
return
|
97
|
+
end
|
98
|
+
|
99
|
+
output = FileUtil.contet_data_command(Params['command'], cd_a, cd_b, Params['dest'])
|
100
|
+
|
101
|
+
elsif Params['command'] == 'unify_time'
|
102
|
+
if Params['cd'].nil?
|
103
|
+
Log.error ("--cd is not set")
|
104
|
+
return
|
105
|
+
end
|
106
|
+
cd = ContentData.new()
|
107
|
+
cd.from_file(Params['cd'])
|
108
|
+
if cd.nil?
|
109
|
+
Log.error ("Error loading content data cd=%s" % Params['cd'])
|
110
|
+
return
|
111
|
+
end
|
112
|
+
output = unify_time(cd)
|
113
|
+
# indexer
|
114
|
+
elsif Params['command'] == 'indexer'
|
115
|
+
if Params['patterns'].nil?
|
116
|
+
Log.error ("--patterns is not set")
|
117
|
+
return
|
118
|
+
end
|
119
|
+
|
120
|
+
patterns = FileIndexing::IndexerPatterns.new
|
121
|
+
Params['patterns'].split(':').each { |pattern|
|
122
|
+
Log.debug1 "Pattern: #{pattern}"
|
123
|
+
patterns.add_pattern File.expand_path(pattern)
|
124
|
+
}
|
125
|
+
|
126
|
+
unless patterns.size > 0
|
127
|
+
Log.error ("Error loading patterns=%s (empty file)" % Params['patterns'])
|
128
|
+
return
|
129
|
+
end
|
130
|
+
|
131
|
+
exist_cd = nil
|
132
|
+
if not Params['exist_cd'].nil?
|
133
|
+
exist_cd = ContentData::ContentData.new()
|
134
|
+
exist_cd.from_file(Params['exist_cd'])
|
135
|
+
if exist_cd.nil?
|
136
|
+
Log.error ("Error loading content data exist_cd=%s" % Params['exist_cd'])
|
137
|
+
return
|
138
|
+
end
|
139
|
+
end
|
140
|
+
indexer = FileIndexing::IndexAgent.new
|
141
|
+
indexer.index(patterns, exist_cd)
|
142
|
+
Log.debug1 indexer.indexed_content.to_s
|
143
|
+
# crawler
|
144
|
+
elsif Params['command'] == 'crawler'
|
145
|
+
if Params['conf_file'].nil?
|
146
|
+
Log.error ("--conf_file is not set")
|
147
|
+
return
|
148
|
+
end
|
149
|
+
if not File.exists?(Params['conf_file'])
|
150
|
+
Log.error ("config file doesn't exist conf_file=%s" % Params['conf_file'])
|
151
|
+
return
|
152
|
+
end
|
153
|
+
|
154
|
+
if Params['cd_out'].nil?
|
155
|
+
time = Tme.now.utc
|
156
|
+
Params['cd_out'] = "crawler.out.#{time.strftime('%Y/%m/%d_%H-%M-%S')}"
|
157
|
+
end
|
158
|
+
unless (Params['cd_in'].nil?)
|
159
|
+
if not File.exists?(Params['conf_file'])
|
160
|
+
Log.error ("input data file doesn't exist cd_in=%s" % Params['cd_in'])
|
161
|
+
return
|
162
|
+
end
|
163
|
+
end
|
164
|
+
|
165
|
+
conf = Configuration.new(Params['conf_file'])
|
166
|
+
threads = Array.new
|
167
|
+
conf.server_conf_vec.each do |server|
|
168
|
+
threads.push(Thread.new { Crawler.new(server, Params['cd_out'], Params['cd_in']) })
|
169
|
+
end
|
170
|
+
|
171
|
+
threads.each { |a| a.join }
|
172
|
+
join_servers_results(conf.server_conf_vec, Params['cd_out'])
|
173
|
+
elsif Params['command'] == 'generate_files'
|
174
|
+
fg = FileGenerator::FileGenerator.new()
|
175
|
+
fg.run()
|
176
|
+
end
|
177
|
+
end
|
178
|
+
|
179
|
+
def self.contet_data_command(command, cd_a, cd_b, dest_path)
|
180
|
+
dest = nil
|
181
|
+
if command == "merge"
|
182
|
+
dest = ContentData.merge(cd_a, cd_b)
|
183
|
+
elsif command == "intersect"
|
184
|
+
dest = ContentData.intersect(cd_a, cd_b)
|
185
|
+
elsif command == "minus"
|
186
|
+
dest = ContentData.remove(cd_b, cd_a)
|
187
|
+
end
|
188
|
+
if dest
|
189
|
+
dest.to_file(dest_path)
|
190
|
+
end
|
191
|
+
end
|
192
|
+
|
193
|
+
# Unify modification/first_appearance time according to input DB
|
194
|
+
# Input: ContentData
|
195
|
+
# Output: ContentData with unified times
|
196
|
+
# Files modification time attribute physically updated
|
197
|
+
# Assumption: Files in the input db are from this device.
|
198
|
+
# There is no check what device they are belong - only the check of server name.
|
199
|
+
# It is a user responsibility (meanwhile) to provide a correct input
|
200
|
+
# If file has a modification time attribute that doesn't present in the input db,
|
201
|
+
# then the assumption is that this file wasnt indexized and it will not be treated
|
202
|
+
# (e.i. we do nothing with it)
|
203
|
+
def self.unify_time(content_data)
|
204
|
+
orig_content_data = ContentData::ContentData.new(content_data)
|
205
|
+
content_data.unify_time
|
206
|
+
content_data.each_instance { |checksum, size, content_mod_time, unified_inst_mod_time, server, path|
|
207
|
+
location = [server, path]
|
208
|
+
orig_inst_mod_time = orig_content_data.get_instance_mod_time(checksum, location)
|
209
|
+
file_mtime, file_size = File.open(path) { |f| [f.mtime, f.size] }
|
210
|
+
Log.debug1 "file:#{path} file_mtime:#{file_mtime}."
|
211
|
+
Log.debug1 "update mtime:#{unified_inst_mod_time}"
|
212
|
+
Log.debug1 "original instance mtime:#{orig_inst_mod_time}."
|
213
|
+
Log.debug1 "unify instance mtime:#{unified_inst_mod_time}."
|
214
|
+
Log.debug1 "Comparison: Real file = unified? #{file_mtime.to_i == unified_inst_mod_time}"
|
215
|
+
if (file_mtime.to_i == orig_inst_mod_time) \
|
216
|
+
and file_size == size \
|
217
|
+
and (file_mtime.to_i != unified_inst_mod_time)
|
218
|
+
Log.debug1 ("Comparison results: File actual time is same as instance time before unification. Need to modify file time")
|
219
|
+
File.utime(File.atime(path), unified_inst_mod_time, path)
|
220
|
+
file_mtime = File.open(path) { |f| f.mtime }
|
221
|
+
Log.debug1 "new file mtime:#{file_mtime}."
|
222
|
+
Log.debug1 "new file mtime in seconds:#{file_mtime.to_i}."
|
223
|
+
end
|
224
|
+
}
|
225
|
+
content_data
|
226
|
+
end
|
227
|
+
|
228
|
+
# Creates directory structure and symlinks with ref_cd structure to the base_cd files and dest as a root dir
|
229
|
+
# Parameters: ref_cd [ContentData]
|
230
|
+
# base_cd [ContentData]
|
231
|
+
# dest [String]
|
232
|
+
# Output: ContentData object consists of contents/instances from ref_cd that have no target in base_cd
|
233
|
+
|
234
|
+
def self.mksymlink(ref_cd, base_cd, dest)
|
235
|
+
# symlinks are not implemented in Windows
|
236
|
+
raise NotImplementedError.new if (RUBY_PLATFORM =~ /mingw/ or RUBY_PLATFORM =~ /ms/ or RUBY_PLATFORM =~ /win/)
|
237
|
+
|
238
|
+
not_found = {}
|
239
|
+
not_found_cd = ContentData::ContentData.new
|
240
|
+
warnings = []
|
241
|
+
dest.chop! if (dest.end_with?("/") or dest.end_with?("\\"))
|
242
|
+
|
243
|
+
ref_cd.each_instance { |checksum, size, content_mod_time, inst_mod_time, server, path|
|
244
|
+
if base_cd.content_exists(checksum)
|
245
|
+
symlink_path = dest + path
|
246
|
+
::FileUtils.mkdir_p(File.dirname(symlink_path)) unless (Dir.exists?(File.dirname(symlink_path)))
|
247
|
+
File.symlink(path, symlink_path)
|
248
|
+
else
|
249
|
+
# add instance to not_found cd
|
250
|
+
not_found_cd.add_instance(checksum, size, server, path, inst_mod_time)
|
251
|
+
warnings << "Warning: base content does not contains:'%s'" % checksum
|
252
|
+
end
|
253
|
+
}
|
254
|
+
Log.warning (warnings)
|
255
|
+
not_found_cd
|
256
|
+
end
|
257
|
+
end
|
258
|
+
|
259
|
+
end
|
260
|
+
|
data/lib/file_utils.rb
ADDED
data/lib/log/version.rb
ADDED
data/lib/log.rb
ADDED
@@ -0,0 +1,188 @@
|
|
1
|
+
# Author: Yaron Dror (yaron.dror.bb@gmail.com)
|
2
|
+
# Description: The file contains the code which implements the 'Log' module
|
3
|
+
# Run: Add to 'require' list. Execute Log.init
|
4
|
+
|
5
|
+
#require 'email'
|
6
|
+
require 'log4r'
|
7
|
+
require 'log4r/outputter/emailoutputter'
|
8
|
+
|
9
|
+
require 'params'
|
10
|
+
|
11
|
+
# Module: Log.
|
12
|
+
# Abstruct: The Log is used to log info\warning\error\debug messages
|
13
|
+
# This module is actually a wrapper to log4r gem and serves
|
14
|
+
# as a central code to use the log utility.
|
15
|
+
module Log
|
16
|
+
|
17
|
+
#Auxiliary method to retrieve the executable name
|
18
|
+
def Log.executable_name
|
19
|
+
/([a-zA-Z0-9\-_\.]+):\d+/ =~ caller[caller.size-1]
|
20
|
+
return $1
|
21
|
+
end
|
22
|
+
|
23
|
+
# Global params
|
24
|
+
Params.integer('log_debug_level', 0 , \
|
25
|
+
'Verbosity of logging. 0 will log only INFO messages. Other value, will log all DEBUG messages as well.')
|
26
|
+
Params.boolean('log_flush_each_message', true, \
|
27
|
+
'If true then File and Console outputters will be flushed for each message.')
|
28
|
+
Params.boolean('log_write_to_file', true , \
|
29
|
+
'If true then the logger will write the messages to a file.')
|
30
|
+
Params.path('log_file_name', "~/.bbfs/log/#{Log.executable_name}.log4r" , \
|
31
|
+
'log file name: ~/.bbfs/log/<executable_name>.log')
|
32
|
+
Params.integer('log_rotation_size',1000000 , \
|
33
|
+
'max log file size. when reaching this size, a new file is created for rest of log')
|
34
|
+
Params.boolean('log_write_to_console', false , \
|
35
|
+
'If true then the logger will write the messages to the console.')
|
36
|
+
Params.boolean('log_write_to_email', false , \
|
37
|
+
'If true then the logger will write the error and fatal messages to email.')
|
38
|
+
Params.string('from_email', 'bbfsdev@gmail.com', 'From gmail address for update.')
|
39
|
+
Params.string('from_email_password', '', 'From gmail password.')
|
40
|
+
Params.string('to_email', 'bbfsdev@gmail.com', 'Destination email for updates.')
|
41
|
+
|
42
|
+
#Should be called from executable right after params handling.
|
43
|
+
# Init Log level and set output to file,stdout and email according to configuration params.
|
44
|
+
def Log.init
|
45
|
+
@log4r = Log4r::Logger.new 'BBFS log'
|
46
|
+
@log4r.trace = true
|
47
|
+
|
48
|
+
#levels setup
|
49
|
+
log4r_level = Log4r::DEBUG
|
50
|
+
log4r_level = Log4r::INFO if 0 == Params['log_debug_level']
|
51
|
+
|
52
|
+
#formatters
|
53
|
+
formatter = Log4r::PatternFormatter.new(:pattern => "[%l] [%d] [%m]")
|
54
|
+
|
55
|
+
#stdout setup
|
56
|
+
if Params['log_write_to_console']
|
57
|
+
stdout_outputter = Log4r::Outputter.stdout
|
58
|
+
stdout_outputter.formatter = formatter
|
59
|
+
stdout_outputter.level = log4r_level
|
60
|
+
@log4r.outputters << stdout_outputter
|
61
|
+
end
|
62
|
+
|
63
|
+
#file setup
|
64
|
+
if Params['log_write_to_file']
|
65
|
+
if File.exist?(Params['log_file_name'])
|
66
|
+
File.delete(Params['log_file_name'])
|
67
|
+
else
|
68
|
+
dir_name = File.dirname(Params['log_file_name'])
|
69
|
+
FileUtils.mkdir_p(dir_name) unless File.directory?(dir_name)
|
70
|
+
end
|
71
|
+
file_config = {
|
72
|
+
"filename" => Params['log_file_name'],
|
73
|
+
"maxsize" => Params['log_rotation_size'],
|
74
|
+
"trunc" => true
|
75
|
+
}
|
76
|
+
file_outputter = Log4r::RollingFileOutputter.new("file_log", file_config)
|
77
|
+
file_outputter.level = log4r_level
|
78
|
+
file_outputter.formatter = formatter
|
79
|
+
@log4r.outputters << file_outputter
|
80
|
+
end
|
81
|
+
|
82
|
+
#email setup
|
83
|
+
if Params['log_write_to_email']
|
84
|
+
email_outputter = Log4r::EmailOutputter.new('email_log',
|
85
|
+
:server => 'smtp.gmail.com',
|
86
|
+
:port => 587,
|
87
|
+
:subject => "Error happened at server:'#{Params['local_server_name']}' run by #{ENV['USER']}. Service_name is #{Params['service_name']}",
|
88
|
+
:acct => Params['from_email'],
|
89
|
+
:from => Params['from_email'],
|
90
|
+
:passwd => Params['from_email_password'],
|
91
|
+
:to => Params['to_email'],
|
92
|
+
:immediate_at => 'FATAL,ERROR',
|
93
|
+
:authtype => :plain,
|
94
|
+
:tls => true,
|
95
|
+
:formatfirst => true,
|
96
|
+
:buffsize => 9999,
|
97
|
+
)
|
98
|
+
email_outputter.level = Log4r::ERROR
|
99
|
+
email_outputter.formatter = formatter
|
100
|
+
@log4r.outputters << email_outputter
|
101
|
+
end
|
102
|
+
|
103
|
+
# Write init message and user parameters
|
104
|
+
@log4r.info('BBFS Log initialized.') # log first data
|
105
|
+
|
106
|
+
# print params to console
|
107
|
+
if Params['print_params_to_stdout']
|
108
|
+
Params.get_init_info_messages().each { |msg|
|
109
|
+
puts(msg)
|
110
|
+
}
|
111
|
+
else
|
112
|
+
Log.info("Not printing executable parameters to console since param:'print_params_to_stdout' is false")
|
113
|
+
end
|
114
|
+
#print params to log info and warning
|
115
|
+
Params.get_init_info_messages().each { |msg|
|
116
|
+
Log.info(msg)
|
117
|
+
}
|
118
|
+
Params.get_init_warning_messages().each { |msg|
|
119
|
+
Log.warning(msg)
|
120
|
+
}
|
121
|
+
end
|
122
|
+
|
123
|
+
# Auxiliary method to add the calling method to the message
|
124
|
+
def Log.msg_with_caller(msg)
|
125
|
+
/([a-zA-Z0-9\-_\.]+:\d+)/ =~ caller[1]
|
126
|
+
$1 + ':' + msg
|
127
|
+
end
|
128
|
+
|
129
|
+
# Log warning massages
|
130
|
+
def Log.warning(msg)
|
131
|
+
Log.init if @log4r.nil?
|
132
|
+
@log4r.warn(msg_with_caller(msg))
|
133
|
+
Log.flush if Params['log_flush_each_message']
|
134
|
+
end
|
135
|
+
|
136
|
+
# Log error massages
|
137
|
+
def Log.error(msg)
|
138
|
+
Log.init if @log4r.nil?
|
139
|
+
@log4r.error(msg_with_caller(msg))
|
140
|
+
Log.flush if Params['log_flush_each_message']
|
141
|
+
end
|
142
|
+
|
143
|
+
# Log info massages
|
144
|
+
def Log.info(msg)
|
145
|
+
Log.init if @log4r.nil?
|
146
|
+
@log4r.info(msg_with_caller(msg))
|
147
|
+
Log.flush if Params['log_flush_each_message']
|
148
|
+
end
|
149
|
+
|
150
|
+
# Log debug level 1 massages
|
151
|
+
def Log.debug1(msg)
|
152
|
+
if Params['log_debug_level'] >= 1
|
153
|
+
Log.init if @log4r.nil?
|
154
|
+
@log4r.debug(msg_with_caller(msg))
|
155
|
+
Log.flush if Params['log_flush_each_message']
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
# Log debug level 2 massages
|
160
|
+
def Log.debug2(msg)
|
161
|
+
if Params['log_debug_level'] >= 2
|
162
|
+
Log.init if @log4r.nil?
|
163
|
+
@log4r.debug(msg_with_caller(msg))
|
164
|
+
Log.flush if Params['log_flush_each_message']
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
# Log debug level 3 massages
|
169
|
+
def Log.debug3(msg)
|
170
|
+
if Params['log_debug_level'] >= 3
|
171
|
+
Log.init if @log4r.nil?
|
172
|
+
@log4r.debug(msg_with_caller(msg))
|
173
|
+
Log.flush if Params['log_flush_each_message']
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
# Flush email log
|
178
|
+
def Log.flush()
|
179
|
+
return if @log4r.nil?
|
180
|
+
@log4r.outputters.each { |o|
|
181
|
+
# Not flushing to email since this will cause empty emails to be sent
|
182
|
+
# Email is already configured to immediately send mail on ERROR|FATAL messages.
|
183
|
+
o.flush unless o.is_a?(Log4r::EmailOutputter)
|
184
|
+
}
|
185
|
+
end
|
186
|
+
|
187
|
+
private_class_method(:msg_with_caller)
|
188
|
+
end
|
@@ -0,0 +1,213 @@
|
|
1
|
+
require 'log'
|
2
|
+
require 'socket'
|
3
|
+
|
4
|
+
module Networking
|
5
|
+
|
6
|
+
Params.integer('client_retry_delay', 60, 'Number of seconds before trying to reconnect.')
|
7
|
+
|
8
|
+
# TODO(kolman): To get robustness, just use try catch + return 0 bytes on write +
|
9
|
+
# return false on status on read.
|
10
|
+
def Networking.write_to_stream(stream, obj)
|
11
|
+
Log.debug3('Writing to stream.')
|
12
|
+
marshal_data = Marshal.dump(obj)
|
13
|
+
Log.debug2("Writing data size: #{marshal_data.length}")
|
14
|
+
data_size = [marshal_data.length].pack("l")
|
15
|
+
if data_size.nil? || marshal_data.nil?
|
16
|
+
Log.debug2 'Send data size is nil!'
|
17
|
+
end
|
18
|
+
begin
|
19
|
+
bytes_written = stream.write data_size
|
20
|
+
bytes_written += stream.write marshal_data
|
21
|
+
return bytes_written
|
22
|
+
rescue Exception => e
|
23
|
+
Log.warning("Could not write tcp/ip stream, #{e.to_s}")
|
24
|
+
begin
|
25
|
+
stream.close()
|
26
|
+
rescue IOError => e
|
27
|
+
Log.warning("Could not close stream, #{e.to_s}.")
|
28
|
+
end
|
29
|
+
return 0
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
# Returns pair [status(true/false), obj]
|
34
|
+
def Networking.read_from_stream(stream)
|
35
|
+
Log.debug3('Read from stream.')
|
36
|
+
begin
|
37
|
+
return [false, nil] unless size_of_data = stream.read(4)
|
38
|
+
size_of_data = size_of_data.unpack("l")[0]
|
39
|
+
Log.debug2("Reading data size:#{size_of_data}")
|
40
|
+
data = stream.read(size_of_data)
|
41
|
+
rescue Exception => e
|
42
|
+
Log.warning("Could not read tcp/ip stream, #{e.to_s}.")
|
43
|
+
begin
|
44
|
+
stream.close()
|
45
|
+
rescue IOError => e
|
46
|
+
Log.warning("Could not close stream, #{e.to_s}.")
|
47
|
+
end
|
48
|
+
return [false, nil]
|
49
|
+
end
|
50
|
+
|
51
|
+
unmarshalled_data = Marshal.load(data)
|
52
|
+
Log.debug2('Read good.')
|
53
|
+
return [true, unmarshalled_data]
|
54
|
+
end
|
55
|
+
|
56
|
+
# Limit on number of concurrent connections?
|
57
|
+
# The TCP server is not responsible for reconnection.
|
58
|
+
class TCPServer
|
59
|
+
attr_reader :tcp_thread
|
60
|
+
|
61
|
+
def initialize(port, obj_clb, new_clb=nil, closed_clb=nil, max_parallel=1)
|
62
|
+
@port = port
|
63
|
+
@obj_clb = obj_clb
|
64
|
+
@new_clb = new_clb
|
65
|
+
@closed_clb = closed_clb
|
66
|
+
# Max parallel connections is not implemented yet.
|
67
|
+
@max_parallel = max_parallel
|
68
|
+
@sockets = {}
|
69
|
+
@tcp_thread = run_server
|
70
|
+
@tcp_thread.abort_on_exception = true
|
71
|
+
end
|
72
|
+
|
73
|
+
def send_obj(obj, addr_info=nil)
|
74
|
+
Log.debug3("send_obj with addr_info:#{addr_info}")
|
75
|
+
unless addr_info.nil?
|
76
|
+
if @sockets.key?(addr_info)
|
77
|
+
Networking.write_to_stream(@sockets[addr_info], obj)
|
78
|
+
else
|
79
|
+
Log.warning("Could not find client socket: #{addr_info}")
|
80
|
+
return 0
|
81
|
+
end
|
82
|
+
else
|
83
|
+
out = {}
|
84
|
+
@sockets.each { |key, sock| out[key] = Networking.write_to_stream(sock, obj) }
|
85
|
+
return out
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
private
|
90
|
+
# Creates new thread/pool
|
91
|
+
def run_server
|
92
|
+
Log.debug3('run_server')
|
93
|
+
return Thread.new do
|
94
|
+
loop {
|
95
|
+
begin
|
96
|
+
Socket.tcp_server_loop(@port) do |sock, addr_info|
|
97
|
+
Log.debug2("----- #{@port} -----")
|
98
|
+
Log.debug2("tcp_server_loop... #{sock} #{addr_info.inspect}")
|
99
|
+
@sockets[addr_info] = sock
|
100
|
+
@new_clb.call(addr_info) if @new_clb != nil
|
101
|
+
loop do
|
102
|
+
# Blocking read.
|
103
|
+
Log.debug2('read_from_stream')
|
104
|
+
stream_ok, obj = Networking.read_from_stream(sock)
|
105
|
+
Log.debug2("Server returned from read: #{stream_ok}")
|
106
|
+
@obj_clb.call(addr_info, obj) if @obj_clb != nil && stream_ok
|
107
|
+
break if !stream_ok
|
108
|
+
end
|
109
|
+
Log.warning("Connection broken, #{addr_info.inspect}")
|
110
|
+
begin
|
111
|
+
@sockets[addr_info].close() unless @sockets[addr_info].nil?
|
112
|
+
rescue IOError => e
|
113
|
+
Log.warning("Could not close socket, #{e.to_s}.")
|
114
|
+
end
|
115
|
+
@sockets.delete(addr_info)
|
116
|
+
@closed_clb.call(addr_info) if @closed_clb != nil
|
117
|
+
end
|
118
|
+
# !!! note this break is needed for tests only !!!! server loop should never end.
|
119
|
+
# and if it ends, it will fail, and rescue will work.
|
120
|
+
break
|
121
|
+
rescue IOError => e
|
122
|
+
Log.warning("Connection broken during tcp_server_loop. Restarting server loop " \
|
123
|
+
"port:#{port}.")
|
124
|
+
end
|
125
|
+
}
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end # TCPServer
|
129
|
+
|
130
|
+
class TCPClient
|
131
|
+
attr_reader :tcp_thread
|
132
|
+
|
133
|
+
def initialize(host, port, obj_clb=nil, reconnected_clb=nil)
|
134
|
+
@host = host
|
135
|
+
@port = port
|
136
|
+
@tcp_socket = nil
|
137
|
+
@obj_clb = obj_clb
|
138
|
+
@reconnected_clb = reconnected_clb
|
139
|
+
Log.debug3("Start TCPClient initialize with @obj_clb: #{@obj_clb}")
|
140
|
+
if @obj_clb != nil
|
141
|
+
@tcp_thread = start_reading
|
142
|
+
@tcp_thread.abort_on_exception = true
|
143
|
+
end
|
144
|
+
# Variable to signal when remote server is ready.
|
145
|
+
@remote_server_available = ConditionVariable.new
|
146
|
+
@remote_server_available_mutex = Mutex.new
|
147
|
+
open_socket unless socket_good?
|
148
|
+
Log.debug3("End TCPClient initialize.")
|
149
|
+
end
|
150
|
+
|
151
|
+
def send_obj(obj)
|
152
|
+
Log.debug1('send_obj')
|
153
|
+
open_socket unless socket_good?
|
154
|
+
Log.debug1('after open')
|
155
|
+
if !socket_good?
|
156
|
+
Log.warning('Socket not opened for writing, skipping send.')
|
157
|
+
return false
|
158
|
+
end
|
159
|
+
Log.debug2("writing... socket port: #{@tcp_socket.peeraddr}")
|
160
|
+
bytes_written = Networking.write_to_stream(@tcp_socket, obj)
|
161
|
+
return bytes_written
|
162
|
+
end
|
163
|
+
|
164
|
+
# This function may be executed only from one thread!!! or in synchronized manner.
|
165
|
+
# private
|
166
|
+
def open_socket
|
167
|
+
Log.debug1("Connecting to content server #{@host}:#{@port}.")
|
168
|
+
begin
|
169
|
+
@tcp_socket = TCPSocket.new(@host, @port)
|
170
|
+
rescue Errno::ECONNREFUSED
|
171
|
+
Log.warning('Connection refused')
|
172
|
+
end
|
173
|
+
Log.debug3("Reconnect clb: '#{@reconnected_clb.nil? ? 'nil' : @reconnected_clb}'")
|
174
|
+
if socket_good?
|
175
|
+
@remote_server_available_mutex.synchronize {
|
176
|
+
@remote_server_available.signal
|
177
|
+
}
|
178
|
+
@reconnected_clb.call if @reconnected_clb != nil && socket_good?
|
179
|
+
end
|
180
|
+
end
|
181
|
+
|
182
|
+
private
|
183
|
+
def socket_good?
|
184
|
+
Log.debug3("socket_good? #{@tcp_socket != nil && !@tcp_socket.closed?}")
|
185
|
+
return @tcp_socket != nil && !@tcp_socket.closed?
|
186
|
+
end
|
187
|
+
|
188
|
+
private
|
189
|
+
def start_reading
|
190
|
+
return Thread.new do
|
191
|
+
loop do
|
192
|
+
Log.debug3('Start blocking read (TCPClient).')
|
193
|
+
# Blocking read.
|
194
|
+
if !socket_good?
|
195
|
+
Log.warning("Socket not good, waiting for reconnection with " \
|
196
|
+
"#{Params['client_retry_delay']} seconds timeout.")
|
197
|
+
@remote_server_available_mutex.synchronize {
|
198
|
+
@remote_server_available.wait(@remote_server_available_mutex,
|
199
|
+
Params['client_retry_delay'])
|
200
|
+
}
|
201
|
+
else
|
202
|
+
read_ok, obj = Networking.read_from_stream(@tcp_socket)
|
203
|
+
Log.debug3("Client returned from read: #{read_ok}")
|
204
|
+
# Handle case when socket is closed in middle.
|
205
|
+
# In that case we should not call obj_clb.
|
206
|
+
@obj_clb.call(obj) if (read_ok && @obj_clb != nil)
|
207
|
+
end
|
208
|
+
end
|
209
|
+
end
|
210
|
+
end
|
211
|
+
end # TCPClient
|
212
|
+
end
|
213
|
+
|
data/lib/networking.rb
ADDED