content_server 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bin/backup_server +8 -20
- data/bin/content_server +8 -20
- data/bin/testing_memory +60 -0
- data/bin/testing_server +57 -0
- data/ext/run_in_background/mkrf_conf.rb +34 -0
- data/lib/content_data/content_data.rb +613 -0
- data/lib/content_data/version.rb +3 -0
- data/lib/content_data.rb +6 -0
- data/lib/content_server/backup_server.rb +65 -86
- data/lib/content_server/content_server.rb +47 -77
- data/lib/content_server/file_streamer.rb +27 -33
- data/lib/content_server/queue_copy.rb +154 -49
- data/lib/content_server/queue_indexer.rb +19 -11
- data/lib/content_server/remote_content.rb +41 -23
- data/lib/content_server/server.rb +91 -0
- data/lib/content_server/version.rb +1 -1
- data/lib/content_server.rb +0 -15
- data/lib/email/email.rb +87 -0
- data/lib/email/version.rb +3 -0
- data/lib/email.rb +4 -0
- data/lib/file_copy/copy.rb +68 -0
- data/lib/file_copy/version.rb +4 -0
- data/lib/file_copy.rb +4 -0
- data/lib/file_indexing/index_agent.rb +170 -0
- data/lib/file_indexing/indexer_patterns.rb +72 -0
- data/lib/file_indexing/version.rb +3 -0
- data/lib/file_indexing.rb +9 -0
- data/lib/file_monitoring/file_monitoring.rb +105 -0
- data/lib/file_monitoring/monitor_path.rb +304 -0
- data/lib/file_monitoring/version.rb +3 -0
- data/lib/file_monitoring.rb +29 -0
- data/lib/file_utils/file_generator/README +97 -0
- data/lib/file_utils/file_generator/file_generator.rb +156 -0
- data/lib/file_utils/file_utils.rb +260 -0
- data/lib/file_utils/version.rb +3 -0
- data/lib/file_utils.rb +4 -0
- data/lib/log/version.rb +3 -0
- data/lib/log.rb +188 -0
- data/lib/networking/tcp.rb +213 -0
- data/lib/networking/version.rb +3 -0
- data/lib/networking.rb +4 -0
- data/lib/params/version.rb +3 -0
- data/lib/params.rb +419 -0
- data/lib/process_monitoring/monitoring.rb +85 -0
- data/lib/process_monitoring/monitoring_info.rb +79 -0
- data/lib/process_monitoring/send_email.rb +40 -0
- data/lib/process_monitoring/thread_safe_hash.rb +77 -0
- data/lib/process_monitoring/version.rb +3 -0
- data/lib/process_monitoring.rb +6 -0
- data/lib/run_in_background/version.rb +3 -0
- data/lib/run_in_background.rb +432 -0
- data/lib/testing_memory/testing_memory.rb +187 -0
- data/lib/testing_server/testing_server.rb +236 -0
- data/lib/testing_server/version.rb +3 -0
- data/lib/testing_server.rb +12 -0
- data/lib/validations/index_validations.rb +106 -0
- data/lib/validations/version.rb +3 -0
- data/lib/validations.rb +4 -0
- data/spec/content_data/validations_spec.rb +113 -0
- data/spec/file_copy/copy_spec.rb +54 -0
- data/spec/file_indexing/index_agent_spec.rb +53 -0
- data/spec/networking/tcp_spec.rb +95 -0
- data/spec/validations/index_validations_spec.rb +77 -0
- data/test/content_data/content_data_test.rb +290 -0
- data/test/file_generator/file_generator_spec.rb +84 -0
- data/test/file_indexing/index_agent_test/New.txt +0 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/bin/libexslt.dll +0 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/bin/libxslt.dll +0 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/bin/xsltproc.exe +0 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libexslt/exslt.h +102 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libexslt/exsltconfig.h +73 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libexslt/exsltexports.h +140 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libexslt/libexslt.h +29 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/attributes.h +38 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/documents.h +93 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/extensions.h +262 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/extra.h +80 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/functions.h +78 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/imports.h +75 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/keys.h +53 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/libxslt.h +30 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/namespaces.h +68 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/numbersInternals.h +69 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/pattern.h +81 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/preproc.h +43 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/security.h +104 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/templates.h +77 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/transform.h +207 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/trio.h +216 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/triodef.h +220 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/variables.h +91 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/win32config.h +101 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/xslt.h +103 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/xsltInternals.h +1967 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/xsltconfig.h +172 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/xsltexports.h +142 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/xsltlocale.h +57 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/xsltutils.h +309 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/xsltwin32config.h +105 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/lib/libexslt.lib +0 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/lib/libexslt_a.lib +0 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/lib/libxslt.lib +0 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/lib/libxslt_a.lib +0 -0
- data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/readme.txt +22 -0
- data/test/file_indexing/index_agent_test/patterns.input +3 -0
- data/test/file_indexing/index_agent_test.rb +51 -0
- data/test/file_monitoring/file_monitoring_test/conf.yml +4 -0
- data/test/file_monitoring/file_monitoring_test/conf_win32.yml +5 -0
- data/test/file_monitoring/file_monitoring_test/log +56 -0
- data/test/file_monitoring/file_monitoring_test.rb +0 -0
- data/test/file_monitoring/monitor_path_test/dir1000/test_file.1000 +1000 -0
- data/test/file_monitoring/monitor_path_test/dir1000/test_file.1000.0 +1000 -0
- data/test/file_monitoring/monitor_path_test/dir1000/test_file.1000.1 +1000 -0
- data/test/file_monitoring/monitor_path_test/dir1500/test_file.1500 +1500 -0
- data/test/file_monitoring/monitor_path_test/dir1500/test_file.1500.0 +1500 -0
- data/test/file_monitoring/monitor_path_test/dir1500/test_file.1500.1 +1500 -0
- data/test/file_monitoring/monitor_path_test/test_file.500 +500 -0
- data/test/file_monitoring/monitor_path_test/test_file.500.0 +500 -0
- data/test/file_monitoring/monitor_path_test/test_file.500.1 +500 -0
- data/test/file_monitoring/monitor_path_test.rb +153 -0
- data/test/file_utils/fileutil_mksymlink_test/dir1000/dir1500/test_file.1500 +1500 -0
- data/test/file_utils/fileutil_mksymlink_test/dir1000/dir1500/test_file.1500.0 +1500 -0
- data/test/file_utils/fileutil_mksymlink_test/dir1000/dir1500/test_file.1500.1 +1500 -0
- data/test/file_utils/fileutil_mksymlink_test/dir1000/test_file.1000 +1000 -0
- data/test/file_utils/fileutil_mksymlink_test/dir1000/test_file.1000.0 +1000 -0
- data/test/file_utils/fileutil_mksymlink_test/dir1000/test_file.1000.1 +1000 -0
- data/test/file_utils/fileutil_mksymlink_test/test_file.500 +500 -0
- data/test/file_utils/fileutil_mksymlink_test/test_file.500.0 +500 -0
- data/test/file_utils/fileutil_mksymlink_test/test_file.500.1 +500 -0
- data/test/file_utils/fileutil_mksymlink_test.rb +125 -0
- data/test/file_utils/time_modification_test.rb +132 -0
- data/test/params/params_spec.rb +280 -0
- data/test/params/params_test.rb +43 -0
- data/test/run_in_background/run_in_background_test.rb +122 -0
- data/test/run_in_background/test_app +57 -0
- metadata +272 -132
- data/lib/content_server/globals.rb +0 -10
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
# Author:
|
|
2
|
+
# originator: Kolman Vornovizky (kolmanv@gmail.com)
|
|
3
|
+
# update: Alexey Nemytov (alexeyn66@gmail.com) 31/07/2012
|
|
4
|
+
# This file is the consist utility functionality for BBFS
|
|
5
|
+
# Such as automatic file generation, symlink, etc.
|
|
6
|
+
# Run from bbfs> ruby -Ilib bin/file_utils --command=generate_files --log_write_to_console=true --log_debug_level=3
|
|
7
|
+
|
|
8
|
+
require 'content_data'
|
|
9
|
+
require 'file_indexing'
|
|
10
|
+
require 'file_utils'
|
|
11
|
+
require 'file_utils/file_generator/file_generator'
|
|
12
|
+
|
|
13
|
+
require 'log'
|
|
14
|
+
require 'params'
|
|
15
|
+
|
|
16
|
+
module FileUtils
|
|
17
|
+
Params.string 'command', nil ,'path'
|
|
18
|
+
Params.string 'ref_cd', nil ,'reference path'
|
|
19
|
+
Params.string 'base_cd', nil ,'base path'
|
|
20
|
+
Params.string 'dest', nil ,'destination path'
|
|
21
|
+
Params.string 'dest_server', nil ,'server name'
|
|
22
|
+
Params.string 'patterns', nil ,'patterns path'
|
|
23
|
+
Params.string 'cd', nil ,'path'
|
|
24
|
+
Params.string 'cd_a', nil ,'a path'
|
|
25
|
+
Params.string 'cd_b', nil ,'b path'
|
|
26
|
+
Params.string 'cd_out', nil ,'output path'
|
|
27
|
+
Params.string 'cd_in', nil ,'input path'
|
|
28
|
+
Params.string 'exist_cd', nil, 'exist_path'
|
|
29
|
+
#Params.string 'config_file', 'path' ,'configuration file path'
|
|
30
|
+
class FileUtils
|
|
31
|
+
def FileUtils.run
|
|
32
|
+
if Params['command'] == 'mksymlink'
|
|
33
|
+
if Params['ref_cd'].nil?
|
|
34
|
+
Log.error ("--ref_cd is not set")
|
|
35
|
+
return
|
|
36
|
+
end
|
|
37
|
+
ref_cd = ContentData.new()
|
|
38
|
+
ref_cd.from_file(Params['ref_cd'])
|
|
39
|
+
if ref_cd.nil?
|
|
40
|
+
Log.error ("Error loading content data ref_cd=%s" % Params['ref_cd'])
|
|
41
|
+
return
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
if Params['base_cd'].nil?
|
|
45
|
+
Log.error ("--base_cd is not set")
|
|
46
|
+
return
|
|
47
|
+
end
|
|
48
|
+
base_cd = ContentData.new()
|
|
49
|
+
base_cd.from_file(Params['base_cd'])
|
|
50
|
+
if base_cd.nil?
|
|
51
|
+
Log.error ("Error loading content data base_cd=%s" % Params['base_cd'])
|
|
52
|
+
return
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
if Params['dest'].nil?
|
|
56
|
+
Log.error ("--dest is not set")
|
|
57
|
+
return
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
not_found = nil
|
|
61
|
+
begin
|
|
62
|
+
not_found = FileUtil.mksymlink(ref_cd, base_cd, Params['dest'])
|
|
63
|
+
rescue NotImplementedError
|
|
64
|
+
Log.error ("symlinks are unimplemented on this machine")
|
|
65
|
+
return nil
|
|
66
|
+
end
|
|
67
|
+
return not_found
|
|
68
|
+
elsif (Params['command'] == "merge" or
|
|
69
|
+
Params['command'] == "intersect" or
|
|
70
|
+
Params['command'] == "minus")
|
|
71
|
+
|
|
72
|
+
if Params['cd_a'].nil?
|
|
73
|
+
Log.error ("--cd_a is not set")
|
|
74
|
+
return
|
|
75
|
+
end
|
|
76
|
+
cd_a = ContentData.new()
|
|
77
|
+
cd_a.from_file(Params['cd_a'])
|
|
78
|
+
if cd_a.nil?
|
|
79
|
+
Log.error ("Error loading content data cd_a=%s" % Params['cd_a'])
|
|
80
|
+
return
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
if Params['cd_b'].nil?
|
|
84
|
+
Log.error ("--cd_b is not set")
|
|
85
|
+
return
|
|
86
|
+
end
|
|
87
|
+
cd_b = ContentData.new()
|
|
88
|
+
cd_b.from_file(Params['cd_b'])
|
|
89
|
+
if cd_b.nil?
|
|
90
|
+
Log.error ("Error loading content data cd_b=%s" % Params['cd_b'])
|
|
91
|
+
return
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
if Params['cd_b'].nil?
|
|
95
|
+
Log.error ("--dest is not set")
|
|
96
|
+
return
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
output = FileUtil.contet_data_command(Params['command'], cd_a, cd_b, Params['dest'])
|
|
100
|
+
|
|
101
|
+
elsif Params['command'] == 'unify_time'
|
|
102
|
+
if Params['cd'].nil?
|
|
103
|
+
Log.error ("--cd is not set")
|
|
104
|
+
return
|
|
105
|
+
end
|
|
106
|
+
cd = ContentData.new()
|
|
107
|
+
cd.from_file(Params['cd'])
|
|
108
|
+
if cd.nil?
|
|
109
|
+
Log.error ("Error loading content data cd=%s" % Params['cd'])
|
|
110
|
+
return
|
|
111
|
+
end
|
|
112
|
+
output = unify_time(cd)
|
|
113
|
+
# indexer
|
|
114
|
+
elsif Params['command'] == 'indexer'
|
|
115
|
+
if Params['patterns'].nil?
|
|
116
|
+
Log.error ("--patterns is not set")
|
|
117
|
+
return
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
patterns = FileIndexing::IndexerPatterns.new
|
|
121
|
+
Params['patterns'].split(':').each { |pattern|
|
|
122
|
+
Log.debug1 "Pattern: #{pattern}"
|
|
123
|
+
patterns.add_pattern File.expand_path(pattern)
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
unless patterns.size > 0
|
|
127
|
+
Log.error ("Error loading patterns=%s (empty file)" % Params['patterns'])
|
|
128
|
+
return
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
exist_cd = nil
|
|
132
|
+
if not Params['exist_cd'].nil?
|
|
133
|
+
exist_cd = ContentData::ContentData.new()
|
|
134
|
+
exist_cd.from_file(Params['exist_cd'])
|
|
135
|
+
if exist_cd.nil?
|
|
136
|
+
Log.error ("Error loading content data exist_cd=%s" % Params['exist_cd'])
|
|
137
|
+
return
|
|
138
|
+
end
|
|
139
|
+
end
|
|
140
|
+
indexer = FileIndexing::IndexAgent.new
|
|
141
|
+
indexer.index(patterns, exist_cd)
|
|
142
|
+
Log.debug1 indexer.indexed_content.to_s
|
|
143
|
+
# crawler
|
|
144
|
+
elsif Params['command'] == 'crawler'
|
|
145
|
+
if Params['conf_file'].nil?
|
|
146
|
+
Log.error ("--conf_file is not set")
|
|
147
|
+
return
|
|
148
|
+
end
|
|
149
|
+
if not File.exists?(Params['conf_file'])
|
|
150
|
+
Log.error ("config file doesn't exist conf_file=%s" % Params['conf_file'])
|
|
151
|
+
return
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
if Params['cd_out'].nil?
|
|
155
|
+
time = Tme.now.utc
|
|
156
|
+
Params['cd_out'] = "crawler.out.#{time.strftime('%Y/%m/%d_%H-%M-%S')}"
|
|
157
|
+
end
|
|
158
|
+
unless (Params['cd_in'].nil?)
|
|
159
|
+
if not File.exists?(Params['conf_file'])
|
|
160
|
+
Log.error ("input data file doesn't exist cd_in=%s" % Params['cd_in'])
|
|
161
|
+
return
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
conf = Configuration.new(Params['conf_file'])
|
|
166
|
+
threads = Array.new
|
|
167
|
+
conf.server_conf_vec.each do |server|
|
|
168
|
+
threads.push(Thread.new { Crawler.new(server, Params['cd_out'], Params['cd_in']) })
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
threads.each { |a| a.join }
|
|
172
|
+
join_servers_results(conf.server_conf_vec, Params['cd_out'])
|
|
173
|
+
elsif Params['command'] == 'generate_files'
|
|
174
|
+
fg = FileGenerator::FileGenerator.new()
|
|
175
|
+
fg.run()
|
|
176
|
+
end
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
def self.contet_data_command(command, cd_a, cd_b, dest_path)
|
|
180
|
+
dest = nil
|
|
181
|
+
if command == "merge"
|
|
182
|
+
dest = ContentData.merge(cd_a, cd_b)
|
|
183
|
+
elsif command == "intersect"
|
|
184
|
+
dest = ContentData.intersect(cd_a, cd_b)
|
|
185
|
+
elsif command == "minus"
|
|
186
|
+
dest = ContentData.remove(cd_b, cd_a)
|
|
187
|
+
end
|
|
188
|
+
if dest
|
|
189
|
+
dest.to_file(dest_path)
|
|
190
|
+
end
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
# Unify modification/first_appearance time according to input DB
|
|
194
|
+
# Input: ContentData
|
|
195
|
+
# Output: ContentData with unified times
|
|
196
|
+
# Files modification time attribute physically updated
|
|
197
|
+
# Assumption: Files in the input db are from this device.
|
|
198
|
+
# There is no check what device they are belong - only the check of server name.
|
|
199
|
+
# It is a user responsibility (meanwhile) to provide a correct input
|
|
200
|
+
# If file has a modification time attribute that doesn't present in the input db,
|
|
201
|
+
# then the assumption is that this file wasnt indexized and it will not be treated
|
|
202
|
+
# (e.i. we do nothing with it)
|
|
203
|
+
def self.unify_time(content_data)
|
|
204
|
+
orig_content_data = ContentData::ContentData.new(content_data)
|
|
205
|
+
content_data.unify_time
|
|
206
|
+
content_data.each_instance { |checksum, size, content_mod_time, unified_inst_mod_time, server, path|
|
|
207
|
+
location = [server, path]
|
|
208
|
+
orig_inst_mod_time = orig_content_data.get_instance_mod_time(checksum, location)
|
|
209
|
+
file_mtime, file_size = File.open(path) { |f| [f.mtime, f.size] }
|
|
210
|
+
Log.debug1 "file:#{path} file_mtime:#{file_mtime}."
|
|
211
|
+
Log.debug1 "update mtime:#{unified_inst_mod_time}"
|
|
212
|
+
Log.debug1 "original instance mtime:#{orig_inst_mod_time}."
|
|
213
|
+
Log.debug1 "unify instance mtime:#{unified_inst_mod_time}."
|
|
214
|
+
Log.debug1 "Comparison: Real file = unified? #{file_mtime.to_i == unified_inst_mod_time}"
|
|
215
|
+
if (file_mtime.to_i == orig_inst_mod_time) \
|
|
216
|
+
and file_size == size \
|
|
217
|
+
and (file_mtime.to_i != unified_inst_mod_time)
|
|
218
|
+
Log.debug1 ("Comparison results: File actual time is same as instance time before unification. Need to modify file time")
|
|
219
|
+
File.utime(File.atime(path), unified_inst_mod_time, path)
|
|
220
|
+
file_mtime = File.open(path) { |f| f.mtime }
|
|
221
|
+
Log.debug1 "new file mtime:#{file_mtime}."
|
|
222
|
+
Log.debug1 "new file mtime in seconds:#{file_mtime.to_i}."
|
|
223
|
+
end
|
|
224
|
+
}
|
|
225
|
+
content_data
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
# Creates directory structure and symlinks with ref_cd structure to the base_cd files and dest as a root dir
|
|
229
|
+
# Parameters: ref_cd [ContentData]
|
|
230
|
+
# base_cd [ContentData]
|
|
231
|
+
# dest [String]
|
|
232
|
+
# Output: ContentData object consists of contents/instances from ref_cd that have no target in base_cd
|
|
233
|
+
|
|
234
|
+
def self.mksymlink(ref_cd, base_cd, dest)
|
|
235
|
+
# symlinks are not implemented in Windows
|
|
236
|
+
raise NotImplementedError.new if (RUBY_PLATFORM =~ /mingw/ or RUBY_PLATFORM =~ /ms/ or RUBY_PLATFORM =~ /win/)
|
|
237
|
+
|
|
238
|
+
not_found = {}
|
|
239
|
+
not_found_cd = ContentData::ContentData.new
|
|
240
|
+
warnings = []
|
|
241
|
+
dest.chop! if (dest.end_with?("/") or dest.end_with?("\\"))
|
|
242
|
+
|
|
243
|
+
ref_cd.each_instance { |checksum, size, content_mod_time, inst_mod_time, server, path|
|
|
244
|
+
if base_cd.content_exists(checksum)
|
|
245
|
+
symlink_path = dest + path
|
|
246
|
+
::FileUtils.mkdir_p(File.dirname(symlink_path)) unless (Dir.exists?(File.dirname(symlink_path)))
|
|
247
|
+
File.symlink(path, symlink_path)
|
|
248
|
+
else
|
|
249
|
+
# add instance to not_found cd
|
|
250
|
+
not_found_cd.add_instance(checksum, size, server, path, inst_mod_time)
|
|
251
|
+
warnings << "Warning: base content does not contains:'%s'" % checksum
|
|
252
|
+
end
|
|
253
|
+
}
|
|
254
|
+
Log.warning (warnings)
|
|
255
|
+
not_found_cd
|
|
256
|
+
end
|
|
257
|
+
end
|
|
258
|
+
|
|
259
|
+
end
|
|
260
|
+
|
data/lib/file_utils.rb
ADDED
data/lib/log/version.rb
ADDED
data/lib/log.rb
ADDED
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
# Author: Yaron Dror (yaron.dror.bb@gmail.com)
|
|
2
|
+
# Description: The file contains the code which implements the 'Log' module
|
|
3
|
+
# Run: Add to 'require' list. Execute Log.init
|
|
4
|
+
|
|
5
|
+
#require 'email'
|
|
6
|
+
require 'log4r'
|
|
7
|
+
require 'log4r/outputter/emailoutputter'
|
|
8
|
+
|
|
9
|
+
require 'params'
|
|
10
|
+
|
|
11
|
+
# Module: Log.
|
|
12
|
+
# Abstruct: The Log is used to log info\warning\error\debug messages
|
|
13
|
+
# This module is actually a wrapper to log4r gem and serves
|
|
14
|
+
# as a central code to use the log utility.
|
|
15
|
+
module Log
|
|
16
|
+
|
|
17
|
+
#Auxiliary method to retrieve the executable name
|
|
18
|
+
def Log.executable_name
|
|
19
|
+
/([a-zA-Z0-9\-_\.]+):\d+/ =~ caller[caller.size-1]
|
|
20
|
+
return $1
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# Global params
|
|
24
|
+
Params.integer('log_debug_level', 0 , \
|
|
25
|
+
'Verbosity of logging. 0 will log only INFO messages. Other value, will log all DEBUG messages as well.')
|
|
26
|
+
Params.boolean('log_flush_each_message', true, \
|
|
27
|
+
'If true then File and Console outputters will be flushed for each message.')
|
|
28
|
+
Params.boolean('log_write_to_file', true , \
|
|
29
|
+
'If true then the logger will write the messages to a file.')
|
|
30
|
+
Params.path('log_file_name', "~/.bbfs/log/#{Log.executable_name}.log4r" , \
|
|
31
|
+
'log file name: ~/.bbfs/log/<executable_name>.log')
|
|
32
|
+
Params.integer('log_rotation_size',1000000 , \
|
|
33
|
+
'max log file size. when reaching this size, a new file is created for rest of log')
|
|
34
|
+
Params.boolean('log_write_to_console', false , \
|
|
35
|
+
'If true then the logger will write the messages to the console.')
|
|
36
|
+
Params.boolean('log_write_to_email', false , \
|
|
37
|
+
'If true then the logger will write the error and fatal messages to email.')
|
|
38
|
+
Params.string('from_email', 'bbfsdev@gmail.com', 'From gmail address for update.')
|
|
39
|
+
Params.string('from_email_password', '', 'From gmail password.')
|
|
40
|
+
Params.string('to_email', 'bbfsdev@gmail.com', 'Destination email for updates.')
|
|
41
|
+
|
|
42
|
+
#Should be called from executable right after params handling.
|
|
43
|
+
# Init Log level and set output to file,stdout and email according to configuration params.
|
|
44
|
+
def Log.init
|
|
45
|
+
@log4r = Log4r::Logger.new 'BBFS log'
|
|
46
|
+
@log4r.trace = true
|
|
47
|
+
|
|
48
|
+
#levels setup
|
|
49
|
+
log4r_level = Log4r::DEBUG
|
|
50
|
+
log4r_level = Log4r::INFO if 0 == Params['log_debug_level']
|
|
51
|
+
|
|
52
|
+
#formatters
|
|
53
|
+
formatter = Log4r::PatternFormatter.new(:pattern => "[%l] [%d] [%m]")
|
|
54
|
+
|
|
55
|
+
#stdout setup
|
|
56
|
+
if Params['log_write_to_console']
|
|
57
|
+
stdout_outputter = Log4r::Outputter.stdout
|
|
58
|
+
stdout_outputter.formatter = formatter
|
|
59
|
+
stdout_outputter.level = log4r_level
|
|
60
|
+
@log4r.outputters << stdout_outputter
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
#file setup
|
|
64
|
+
if Params['log_write_to_file']
|
|
65
|
+
if File.exist?(Params['log_file_name'])
|
|
66
|
+
File.delete(Params['log_file_name'])
|
|
67
|
+
else
|
|
68
|
+
dir_name = File.dirname(Params['log_file_name'])
|
|
69
|
+
FileUtils.mkdir_p(dir_name) unless File.directory?(dir_name)
|
|
70
|
+
end
|
|
71
|
+
file_config = {
|
|
72
|
+
"filename" => Params['log_file_name'],
|
|
73
|
+
"maxsize" => Params['log_rotation_size'],
|
|
74
|
+
"trunc" => true
|
|
75
|
+
}
|
|
76
|
+
file_outputter = Log4r::RollingFileOutputter.new("file_log", file_config)
|
|
77
|
+
file_outputter.level = log4r_level
|
|
78
|
+
file_outputter.formatter = formatter
|
|
79
|
+
@log4r.outputters << file_outputter
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
#email setup
|
|
83
|
+
if Params['log_write_to_email']
|
|
84
|
+
email_outputter = Log4r::EmailOutputter.new('email_log',
|
|
85
|
+
:server => 'smtp.gmail.com',
|
|
86
|
+
:port => 587,
|
|
87
|
+
:subject => "Error happened at server:'#{Params['local_server_name']}' run by #{ENV['USER']}. Service_name is #{Params['service_name']}",
|
|
88
|
+
:acct => Params['from_email'],
|
|
89
|
+
:from => Params['from_email'],
|
|
90
|
+
:passwd => Params['from_email_password'],
|
|
91
|
+
:to => Params['to_email'],
|
|
92
|
+
:immediate_at => 'FATAL,ERROR',
|
|
93
|
+
:authtype => :plain,
|
|
94
|
+
:tls => true,
|
|
95
|
+
:formatfirst => true,
|
|
96
|
+
:buffsize => 9999,
|
|
97
|
+
)
|
|
98
|
+
email_outputter.level = Log4r::ERROR
|
|
99
|
+
email_outputter.formatter = formatter
|
|
100
|
+
@log4r.outputters << email_outputter
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Write init message and user parameters
|
|
104
|
+
@log4r.info('BBFS Log initialized.') # log first data
|
|
105
|
+
|
|
106
|
+
# print params to console
|
|
107
|
+
if Params['print_params_to_stdout']
|
|
108
|
+
Params.get_init_info_messages().each { |msg|
|
|
109
|
+
puts(msg)
|
|
110
|
+
}
|
|
111
|
+
else
|
|
112
|
+
Log.info("Not printing executable parameters to console since param:'print_params_to_stdout' is false")
|
|
113
|
+
end
|
|
114
|
+
#print params to log info and warning
|
|
115
|
+
Params.get_init_info_messages().each { |msg|
|
|
116
|
+
Log.info(msg)
|
|
117
|
+
}
|
|
118
|
+
Params.get_init_warning_messages().each { |msg|
|
|
119
|
+
Log.warning(msg)
|
|
120
|
+
}
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# Auxiliary method to add the calling method to the message
|
|
124
|
+
def Log.msg_with_caller(msg)
|
|
125
|
+
/([a-zA-Z0-9\-_\.]+:\d+)/ =~ caller[1]
|
|
126
|
+
$1 + ':' + msg
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# Log warning massages
|
|
130
|
+
def Log.warning(msg)
|
|
131
|
+
Log.init if @log4r.nil?
|
|
132
|
+
@log4r.warn(msg_with_caller(msg))
|
|
133
|
+
Log.flush if Params['log_flush_each_message']
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# Log error massages
|
|
137
|
+
def Log.error(msg)
|
|
138
|
+
Log.init if @log4r.nil?
|
|
139
|
+
@log4r.error(msg_with_caller(msg))
|
|
140
|
+
Log.flush if Params['log_flush_each_message']
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# Log info massages
|
|
144
|
+
def Log.info(msg)
|
|
145
|
+
Log.init if @log4r.nil?
|
|
146
|
+
@log4r.info(msg_with_caller(msg))
|
|
147
|
+
Log.flush if Params['log_flush_each_message']
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
# Log debug level 1 massages
|
|
151
|
+
def Log.debug1(msg)
|
|
152
|
+
if Params['log_debug_level'] >= 1
|
|
153
|
+
Log.init if @log4r.nil?
|
|
154
|
+
@log4r.debug(msg_with_caller(msg))
|
|
155
|
+
Log.flush if Params['log_flush_each_message']
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# Log debug level 2 massages
|
|
160
|
+
def Log.debug2(msg)
|
|
161
|
+
if Params['log_debug_level'] >= 2
|
|
162
|
+
Log.init if @log4r.nil?
|
|
163
|
+
@log4r.debug(msg_with_caller(msg))
|
|
164
|
+
Log.flush if Params['log_flush_each_message']
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
# Log debug level 3 massages
|
|
169
|
+
def Log.debug3(msg)
|
|
170
|
+
if Params['log_debug_level'] >= 3
|
|
171
|
+
Log.init if @log4r.nil?
|
|
172
|
+
@log4r.debug(msg_with_caller(msg))
|
|
173
|
+
Log.flush if Params['log_flush_each_message']
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
# Flush email log
|
|
178
|
+
def Log.flush()
|
|
179
|
+
return if @log4r.nil?
|
|
180
|
+
@log4r.outputters.each { |o|
|
|
181
|
+
# Not flushing to email since this will cause empty emails to be sent
|
|
182
|
+
# Email is already configured to immediately send mail on ERROR|FATAL messages.
|
|
183
|
+
o.flush unless o.is_a?(Log4r::EmailOutputter)
|
|
184
|
+
}
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
private_class_method(:msg_with_caller)
|
|
188
|
+
end
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
require 'log'
|
|
2
|
+
require 'socket'
|
|
3
|
+
|
|
4
|
+
module Networking
|
|
5
|
+
|
|
6
|
+
Params.integer('client_retry_delay', 60, 'Number of seconds before trying to reconnect.')
|
|
7
|
+
|
|
8
|
+
# TODO(kolman): To get robustness, just use try catch + return 0 bytes on write +
|
|
9
|
+
# return false on status on read.
|
|
10
|
+
def Networking.write_to_stream(stream, obj)
|
|
11
|
+
Log.debug3('Writing to stream.')
|
|
12
|
+
marshal_data = Marshal.dump(obj)
|
|
13
|
+
Log.debug2("Writing data size: #{marshal_data.length}")
|
|
14
|
+
data_size = [marshal_data.length].pack("l")
|
|
15
|
+
if data_size.nil? || marshal_data.nil?
|
|
16
|
+
Log.debug2 'Send data size is nil!'
|
|
17
|
+
end
|
|
18
|
+
begin
|
|
19
|
+
bytes_written = stream.write data_size
|
|
20
|
+
bytes_written += stream.write marshal_data
|
|
21
|
+
return bytes_written
|
|
22
|
+
rescue Exception => e
|
|
23
|
+
Log.warning("Could not write tcp/ip stream, #{e.to_s}")
|
|
24
|
+
begin
|
|
25
|
+
stream.close()
|
|
26
|
+
rescue IOError => e
|
|
27
|
+
Log.warning("Could not close stream, #{e.to_s}.")
|
|
28
|
+
end
|
|
29
|
+
return 0
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Returns pair [status(true/false), obj]
|
|
34
|
+
def Networking.read_from_stream(stream)
|
|
35
|
+
Log.debug3('Read from stream.')
|
|
36
|
+
begin
|
|
37
|
+
return [false, nil] unless size_of_data = stream.read(4)
|
|
38
|
+
size_of_data = size_of_data.unpack("l")[0]
|
|
39
|
+
Log.debug2("Reading data size:#{size_of_data}")
|
|
40
|
+
data = stream.read(size_of_data)
|
|
41
|
+
rescue Exception => e
|
|
42
|
+
Log.warning("Could not read tcp/ip stream, #{e.to_s}.")
|
|
43
|
+
begin
|
|
44
|
+
stream.close()
|
|
45
|
+
rescue IOError => e
|
|
46
|
+
Log.warning("Could not close stream, #{e.to_s}.")
|
|
47
|
+
end
|
|
48
|
+
return [false, nil]
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
unmarshalled_data = Marshal.load(data)
|
|
52
|
+
Log.debug2('Read good.')
|
|
53
|
+
return [true, unmarshalled_data]
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Limit on number of concurrent connections?
|
|
57
|
+
# The TCP server is not responsible for reconnection.
|
|
58
|
+
class TCPServer
|
|
59
|
+
attr_reader :tcp_thread
|
|
60
|
+
|
|
61
|
+
def initialize(port, obj_clb, new_clb=nil, closed_clb=nil, max_parallel=1)
|
|
62
|
+
@port = port
|
|
63
|
+
@obj_clb = obj_clb
|
|
64
|
+
@new_clb = new_clb
|
|
65
|
+
@closed_clb = closed_clb
|
|
66
|
+
# Max parallel connections is not implemented yet.
|
|
67
|
+
@max_parallel = max_parallel
|
|
68
|
+
@sockets = {}
|
|
69
|
+
@tcp_thread = run_server
|
|
70
|
+
@tcp_thread.abort_on_exception = true
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def send_obj(obj, addr_info=nil)
|
|
74
|
+
Log.debug3("send_obj with addr_info:#{addr_info}")
|
|
75
|
+
unless addr_info.nil?
|
|
76
|
+
if @sockets.key?(addr_info)
|
|
77
|
+
Networking.write_to_stream(@sockets[addr_info], obj)
|
|
78
|
+
else
|
|
79
|
+
Log.warning("Could not find client socket: #{addr_info}")
|
|
80
|
+
return 0
|
|
81
|
+
end
|
|
82
|
+
else
|
|
83
|
+
out = {}
|
|
84
|
+
@sockets.each { |key, sock| out[key] = Networking.write_to_stream(sock, obj) }
|
|
85
|
+
return out
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
private
|
|
90
|
+
# Creates new thread/pool
|
|
91
|
+
def run_server
|
|
92
|
+
Log.debug3('run_server')
|
|
93
|
+
return Thread.new do
|
|
94
|
+
loop {
|
|
95
|
+
begin
|
|
96
|
+
Socket.tcp_server_loop(@port) do |sock, addr_info|
|
|
97
|
+
Log.debug2("----- #{@port} -----")
|
|
98
|
+
Log.debug2("tcp_server_loop... #{sock} #{addr_info.inspect}")
|
|
99
|
+
@sockets[addr_info] = sock
|
|
100
|
+
@new_clb.call(addr_info) if @new_clb != nil
|
|
101
|
+
loop do
|
|
102
|
+
# Blocking read.
|
|
103
|
+
Log.debug2('read_from_stream')
|
|
104
|
+
stream_ok, obj = Networking.read_from_stream(sock)
|
|
105
|
+
Log.debug2("Server returned from read: #{stream_ok}")
|
|
106
|
+
@obj_clb.call(addr_info, obj) if @obj_clb != nil && stream_ok
|
|
107
|
+
break if !stream_ok
|
|
108
|
+
end
|
|
109
|
+
Log.warning("Connection broken, #{addr_info.inspect}")
|
|
110
|
+
begin
|
|
111
|
+
@sockets[addr_info].close() unless @sockets[addr_info].nil?
|
|
112
|
+
rescue IOError => e
|
|
113
|
+
Log.warning("Could not close socket, #{e.to_s}.")
|
|
114
|
+
end
|
|
115
|
+
@sockets.delete(addr_info)
|
|
116
|
+
@closed_clb.call(addr_info) if @closed_clb != nil
|
|
117
|
+
end
|
|
118
|
+
# !!! note this break is needed for tests only !!!! server loop should never end.
|
|
119
|
+
# and if it ends, it will fail, and rescue will work.
|
|
120
|
+
break
|
|
121
|
+
rescue IOError => e
|
|
122
|
+
Log.warning("Connection broken during tcp_server_loop. Restarting server loop " \
|
|
123
|
+
"port:#{port}.")
|
|
124
|
+
end
|
|
125
|
+
}
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
end # TCPServer
|
|
129
|
+
|
|
130
|
+
class TCPClient
|
|
131
|
+
attr_reader :tcp_thread
|
|
132
|
+
|
|
133
|
+
def initialize(host, port, obj_clb=nil, reconnected_clb=nil)
|
|
134
|
+
@host = host
|
|
135
|
+
@port = port
|
|
136
|
+
@tcp_socket = nil
|
|
137
|
+
@obj_clb = obj_clb
|
|
138
|
+
@reconnected_clb = reconnected_clb
|
|
139
|
+
Log.debug3("Start TCPClient initialize with @obj_clb: #{@obj_clb}")
|
|
140
|
+
if @obj_clb != nil
|
|
141
|
+
@tcp_thread = start_reading
|
|
142
|
+
@tcp_thread.abort_on_exception = true
|
|
143
|
+
end
|
|
144
|
+
# Variable to signal when remote server is ready.
|
|
145
|
+
@remote_server_available = ConditionVariable.new
|
|
146
|
+
@remote_server_available_mutex = Mutex.new
|
|
147
|
+
open_socket unless socket_good?
|
|
148
|
+
Log.debug3("End TCPClient initialize.")
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def send_obj(obj)
|
|
152
|
+
Log.debug1('send_obj')
|
|
153
|
+
open_socket unless socket_good?
|
|
154
|
+
Log.debug1('after open')
|
|
155
|
+
if !socket_good?
|
|
156
|
+
Log.warning('Socket not opened for writing, skipping send.')
|
|
157
|
+
return false
|
|
158
|
+
end
|
|
159
|
+
Log.debug2("writing... socket port: #{@tcp_socket.peeraddr}")
|
|
160
|
+
bytes_written = Networking.write_to_stream(@tcp_socket, obj)
|
|
161
|
+
return bytes_written
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
# This function may be executed only from one thread!!! or in synchronized manner.
|
|
165
|
+
# private
|
|
166
|
+
def open_socket
|
|
167
|
+
Log.debug1("Connecting to content server #{@host}:#{@port}.")
|
|
168
|
+
begin
|
|
169
|
+
@tcp_socket = TCPSocket.new(@host, @port)
|
|
170
|
+
rescue Errno::ECONNREFUSED
|
|
171
|
+
Log.warning('Connection refused')
|
|
172
|
+
end
|
|
173
|
+
Log.debug3("Reconnect clb: '#{@reconnected_clb.nil? ? 'nil' : @reconnected_clb}'")
|
|
174
|
+
if socket_good?
|
|
175
|
+
@remote_server_available_mutex.synchronize {
|
|
176
|
+
@remote_server_available.signal
|
|
177
|
+
}
|
|
178
|
+
@reconnected_clb.call if @reconnected_clb != nil && socket_good?
|
|
179
|
+
end
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
private
|
|
183
|
+
def socket_good?
|
|
184
|
+
Log.debug3("socket_good? #{@tcp_socket != nil && !@tcp_socket.closed?}")
|
|
185
|
+
return @tcp_socket != nil && !@tcp_socket.closed?
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
private
|
|
189
|
+
def start_reading
|
|
190
|
+
return Thread.new do
|
|
191
|
+
loop do
|
|
192
|
+
Log.debug3('Start blocking read (TCPClient).')
|
|
193
|
+
# Blocking read.
|
|
194
|
+
if !socket_good?
|
|
195
|
+
Log.warning("Socket not good, waiting for reconnection with " \
|
|
196
|
+
"#{Params['client_retry_delay']} seconds timeout.")
|
|
197
|
+
@remote_server_available_mutex.synchronize {
|
|
198
|
+
@remote_server_available.wait(@remote_server_available_mutex,
|
|
199
|
+
Params['client_retry_delay'])
|
|
200
|
+
}
|
|
201
|
+
else
|
|
202
|
+
read_ok, obj = Networking.read_from_stream(@tcp_socket)
|
|
203
|
+
Log.debug3("Client returned from read: #{read_ok}")
|
|
204
|
+
# Handle case when socket is closed in middle.
|
|
205
|
+
# In that case we should not call obj_clb.
|
|
206
|
+
@obj_clb.call(obj) if (read_ok && @obj_clb != nil)
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
end
|
|
210
|
+
end
|
|
211
|
+
end # TCPClient
|
|
212
|
+
end
|
|
213
|
+
|
data/lib/networking.rb
ADDED