content_server 1.1.0 → 1.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (137) hide show
  1. data/bin/backup_server +8 -20
  2. data/bin/content_server +8 -20
  3. data/bin/testing_memory +60 -0
  4. data/bin/testing_server +57 -0
  5. data/ext/run_in_background/mkrf_conf.rb +34 -0
  6. data/lib/content_data/content_data.rb +613 -0
  7. data/lib/content_data/version.rb +3 -0
  8. data/lib/content_data.rb +6 -0
  9. data/lib/content_server/backup_server.rb +65 -86
  10. data/lib/content_server/content_server.rb +47 -77
  11. data/lib/content_server/file_streamer.rb +27 -33
  12. data/lib/content_server/queue_copy.rb +154 -49
  13. data/lib/content_server/queue_indexer.rb +19 -11
  14. data/lib/content_server/remote_content.rb +41 -23
  15. data/lib/content_server/server.rb +91 -0
  16. data/lib/content_server/version.rb +1 -1
  17. data/lib/content_server.rb +0 -15
  18. data/lib/email/email.rb +87 -0
  19. data/lib/email/version.rb +3 -0
  20. data/lib/email.rb +4 -0
  21. data/lib/file_copy/copy.rb +68 -0
  22. data/lib/file_copy/version.rb +4 -0
  23. data/lib/file_copy.rb +4 -0
  24. data/lib/file_indexing/index_agent.rb +170 -0
  25. data/lib/file_indexing/indexer_patterns.rb +72 -0
  26. data/lib/file_indexing/version.rb +3 -0
  27. data/lib/file_indexing.rb +9 -0
  28. data/lib/file_monitoring/file_monitoring.rb +105 -0
  29. data/lib/file_monitoring/monitor_path.rb +304 -0
  30. data/lib/file_monitoring/version.rb +3 -0
  31. data/lib/file_monitoring.rb +29 -0
  32. data/lib/file_utils/file_generator/README +97 -0
  33. data/lib/file_utils/file_generator/file_generator.rb +156 -0
  34. data/lib/file_utils/file_utils.rb +260 -0
  35. data/lib/file_utils/version.rb +3 -0
  36. data/lib/file_utils.rb +4 -0
  37. data/lib/log/version.rb +3 -0
  38. data/lib/log.rb +188 -0
  39. data/lib/networking/tcp.rb +213 -0
  40. data/lib/networking/version.rb +3 -0
  41. data/lib/networking.rb +4 -0
  42. data/lib/params/version.rb +3 -0
  43. data/lib/params.rb +419 -0
  44. data/lib/process_monitoring/monitoring.rb +85 -0
  45. data/lib/process_monitoring/monitoring_info.rb +79 -0
  46. data/lib/process_monitoring/send_email.rb +40 -0
  47. data/lib/process_monitoring/thread_safe_hash.rb +77 -0
  48. data/lib/process_monitoring/version.rb +3 -0
  49. data/lib/process_monitoring.rb +6 -0
  50. data/lib/run_in_background/version.rb +3 -0
  51. data/lib/run_in_background.rb +432 -0
  52. data/lib/testing_memory/testing_memory.rb +187 -0
  53. data/lib/testing_server/testing_server.rb +236 -0
  54. data/lib/testing_server/version.rb +3 -0
  55. data/lib/testing_server.rb +12 -0
  56. data/lib/validations/index_validations.rb +106 -0
  57. data/lib/validations/version.rb +3 -0
  58. data/lib/validations.rb +4 -0
  59. data/spec/content_data/validations_spec.rb +113 -0
  60. data/spec/file_copy/copy_spec.rb +54 -0
  61. data/spec/file_indexing/index_agent_spec.rb +53 -0
  62. data/spec/networking/tcp_spec.rb +95 -0
  63. data/spec/validations/index_validations_spec.rb +77 -0
  64. data/test/content_data/content_data_test.rb +290 -0
  65. data/test/file_generator/file_generator_spec.rb +84 -0
  66. data/test/file_indexing/index_agent_test/New.txt +0 -0
  67. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/bin/libexslt.dll +0 -0
  68. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/bin/libxslt.dll +0 -0
  69. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/bin/xsltproc.exe +0 -0
  70. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libexslt/exslt.h +102 -0
  71. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libexslt/exsltconfig.h +73 -0
  72. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libexslt/exsltexports.h +140 -0
  73. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libexslt/libexslt.h +29 -0
  74. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/attributes.h +38 -0
  75. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/documents.h +93 -0
  76. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/extensions.h +262 -0
  77. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/extra.h +80 -0
  78. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/functions.h +78 -0
  79. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/imports.h +75 -0
  80. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/keys.h +53 -0
  81. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/libxslt.h +30 -0
  82. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/namespaces.h +68 -0
  83. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/numbersInternals.h +69 -0
  84. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/pattern.h +81 -0
  85. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/preproc.h +43 -0
  86. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/security.h +104 -0
  87. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/templates.h +77 -0
  88. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/transform.h +207 -0
  89. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/trio.h +216 -0
  90. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/triodef.h +220 -0
  91. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/variables.h +91 -0
  92. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/win32config.h +101 -0
  93. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/xslt.h +103 -0
  94. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/xsltInternals.h +1967 -0
  95. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/xsltconfig.h +172 -0
  96. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/xsltexports.h +142 -0
  97. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/xsltlocale.h +57 -0
  98. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/xsltutils.h +309 -0
  99. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/xsltwin32config.h +105 -0
  100. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/lib/libexslt.lib +0 -0
  101. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/lib/libexslt_a.lib +0 -0
  102. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/lib/libxslt.lib +0 -0
  103. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/lib/libxslt_a.lib +0 -0
  104. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/readme.txt +22 -0
  105. data/test/file_indexing/index_agent_test/patterns.input +3 -0
  106. data/test/file_indexing/index_agent_test.rb +51 -0
  107. data/test/file_monitoring/file_monitoring_test/conf.yml +4 -0
  108. data/test/file_monitoring/file_monitoring_test/conf_win32.yml +5 -0
  109. data/test/file_monitoring/file_monitoring_test/log +56 -0
  110. data/test/file_monitoring/file_monitoring_test.rb +0 -0
  111. data/test/file_monitoring/monitor_path_test/dir1000/test_file.1000 +1000 -0
  112. data/test/file_monitoring/monitor_path_test/dir1000/test_file.1000.0 +1000 -0
  113. data/test/file_monitoring/monitor_path_test/dir1000/test_file.1000.1 +1000 -0
  114. data/test/file_monitoring/monitor_path_test/dir1500/test_file.1500 +1500 -0
  115. data/test/file_monitoring/monitor_path_test/dir1500/test_file.1500.0 +1500 -0
  116. data/test/file_monitoring/monitor_path_test/dir1500/test_file.1500.1 +1500 -0
  117. data/test/file_monitoring/monitor_path_test/test_file.500 +500 -0
  118. data/test/file_monitoring/monitor_path_test/test_file.500.0 +500 -0
  119. data/test/file_monitoring/monitor_path_test/test_file.500.1 +500 -0
  120. data/test/file_monitoring/monitor_path_test.rb +153 -0
  121. data/test/file_utils/fileutil_mksymlink_test/dir1000/dir1500/test_file.1500 +1500 -0
  122. data/test/file_utils/fileutil_mksymlink_test/dir1000/dir1500/test_file.1500.0 +1500 -0
  123. data/test/file_utils/fileutil_mksymlink_test/dir1000/dir1500/test_file.1500.1 +1500 -0
  124. data/test/file_utils/fileutil_mksymlink_test/dir1000/test_file.1000 +1000 -0
  125. data/test/file_utils/fileutil_mksymlink_test/dir1000/test_file.1000.0 +1000 -0
  126. data/test/file_utils/fileutil_mksymlink_test/dir1000/test_file.1000.1 +1000 -0
  127. data/test/file_utils/fileutil_mksymlink_test/test_file.500 +500 -0
  128. data/test/file_utils/fileutil_mksymlink_test/test_file.500.0 +500 -0
  129. data/test/file_utils/fileutil_mksymlink_test/test_file.500.1 +500 -0
  130. data/test/file_utils/fileutil_mksymlink_test.rb +125 -0
  131. data/test/file_utils/time_modification_test.rb +132 -0
  132. data/test/params/params_spec.rb +280 -0
  133. data/test/params/params_test.rb +43 -0
  134. data/test/run_in_background/run_in_background_test.rb +122 -0
  135. data/test/run_in_background/test_app +57 -0
  136. metadata +272 -132
  137. data/lib/content_server/globals.rb +0 -10
@@ -0,0 +1,260 @@
1
+ # Author:
2
+ # originator: Kolman Vornovizky (kolmanv@gmail.com)
3
+ # update: Alexey Nemytov (alexeyn66@gmail.com) 31/07/2012
4
+ # This file is the consist utility functionality for BBFS
5
+ # Such as automatic file generation, symlink, etc.
6
+ # Run from bbfs> ruby -Ilib bin/file_utils --command=generate_files --log_write_to_console=true --log_debug_level=3
7
+
8
+ require 'content_data'
9
+ require 'file_indexing'
10
+ require 'file_utils'
11
+ require 'file_utils/file_generator/file_generator'
12
+
13
+ require 'log'
14
+ require 'params'
15
+
16
+ module FileUtils
17
+ Params.string 'command', nil ,'path'
18
+ Params.string 'ref_cd', nil ,'reference path'
19
+ Params.string 'base_cd', nil ,'base path'
20
+ Params.string 'dest', nil ,'destination path'
21
+ Params.string 'dest_server', nil ,'server name'
22
+ Params.string 'patterns', nil ,'patterns path'
23
+ Params.string 'cd', nil ,'path'
24
+ Params.string 'cd_a', nil ,'a path'
25
+ Params.string 'cd_b', nil ,'b path'
26
+ Params.string 'cd_out', nil ,'output path'
27
+ Params.string 'cd_in', nil ,'input path'
28
+ Params.string 'exist_cd', nil, 'exist_path'
29
+ #Params.string 'config_file', 'path' ,'configuration file path'
30
+ class FileUtils
31
+ def FileUtils.run
32
+ if Params['command'] == 'mksymlink'
33
+ if Params['ref_cd'].nil?
34
+ Log.error ("--ref_cd is not set")
35
+ return
36
+ end
37
+ ref_cd = ContentData.new()
38
+ ref_cd.from_file(Params['ref_cd'])
39
+ if ref_cd.nil?
40
+ Log.error ("Error loading content data ref_cd=%s" % Params['ref_cd'])
41
+ return
42
+ end
43
+
44
+ if Params['base_cd'].nil?
45
+ Log.error ("--base_cd is not set")
46
+ return
47
+ end
48
+ base_cd = ContentData.new()
49
+ base_cd.from_file(Params['base_cd'])
50
+ if base_cd.nil?
51
+ Log.error ("Error loading content data base_cd=%s" % Params['base_cd'])
52
+ return
53
+ end
54
+
55
+ if Params['dest'].nil?
56
+ Log.error ("--dest is not set")
57
+ return
58
+ end
59
+
60
+ not_found = nil
61
+ begin
62
+ not_found = FileUtil.mksymlink(ref_cd, base_cd, Params['dest'])
63
+ rescue NotImplementedError
64
+ Log.error ("symlinks are unimplemented on this machine")
65
+ return nil
66
+ end
67
+ return not_found
68
+ elsif (Params['command'] == "merge" or
69
+ Params['command'] == "intersect" or
70
+ Params['command'] == "minus")
71
+
72
+ if Params['cd_a'].nil?
73
+ Log.error ("--cd_a is not set")
74
+ return
75
+ end
76
+ cd_a = ContentData.new()
77
+ cd_a.from_file(Params['cd_a'])
78
+ if cd_a.nil?
79
+ Log.error ("Error loading content data cd_a=%s" % Params['cd_a'])
80
+ return
81
+ end
82
+
83
+ if Params['cd_b'].nil?
84
+ Log.error ("--cd_b is not set")
85
+ return
86
+ end
87
+ cd_b = ContentData.new()
88
+ cd_b.from_file(Params['cd_b'])
89
+ if cd_b.nil?
90
+ Log.error ("Error loading content data cd_b=%s" % Params['cd_b'])
91
+ return
92
+ end
93
+
94
+ if Params['cd_b'].nil?
95
+ Log.error ("--dest is not set")
96
+ return
97
+ end
98
+
99
+ output = FileUtil.contet_data_command(Params['command'], cd_a, cd_b, Params['dest'])
100
+
101
+ elsif Params['command'] == 'unify_time'
102
+ if Params['cd'].nil?
103
+ Log.error ("--cd is not set")
104
+ return
105
+ end
106
+ cd = ContentData.new()
107
+ cd.from_file(Params['cd'])
108
+ if cd.nil?
109
+ Log.error ("Error loading content data cd=%s" % Params['cd'])
110
+ return
111
+ end
112
+ output = unify_time(cd)
113
+ # indexer
114
+ elsif Params['command'] == 'indexer'
115
+ if Params['patterns'].nil?
116
+ Log.error ("--patterns is not set")
117
+ return
118
+ end
119
+
120
+ patterns = FileIndexing::IndexerPatterns.new
121
+ Params['patterns'].split(':').each { |pattern|
122
+ Log.debug1 "Pattern: #{pattern}"
123
+ patterns.add_pattern File.expand_path(pattern)
124
+ }
125
+
126
+ unless patterns.size > 0
127
+ Log.error ("Error loading patterns=%s (empty file)" % Params['patterns'])
128
+ return
129
+ end
130
+
131
+ exist_cd = nil
132
+ if not Params['exist_cd'].nil?
133
+ exist_cd = ContentData::ContentData.new()
134
+ exist_cd.from_file(Params['exist_cd'])
135
+ if exist_cd.nil?
136
+ Log.error ("Error loading content data exist_cd=%s" % Params['exist_cd'])
137
+ return
138
+ end
139
+ end
140
+ indexer = FileIndexing::IndexAgent.new
141
+ indexer.index(patterns, exist_cd)
142
+ Log.debug1 indexer.indexed_content.to_s
143
+ # crawler
144
+ elsif Params['command'] == 'crawler'
145
+ if Params['conf_file'].nil?
146
+ Log.error ("--conf_file is not set")
147
+ return
148
+ end
149
+ if not File.exists?(Params['conf_file'])
150
+ Log.error ("config file doesn't exist conf_file=%s" % Params['conf_file'])
151
+ return
152
+ end
153
+
154
+ if Params['cd_out'].nil?
155
+ time = Tme.now.utc
156
+ Params['cd_out'] = "crawler.out.#{time.strftime('%Y/%m/%d_%H-%M-%S')}"
157
+ end
158
+ unless (Params['cd_in'].nil?)
159
+ if not File.exists?(Params['conf_file'])
160
+ Log.error ("input data file doesn't exist cd_in=%s" % Params['cd_in'])
161
+ return
162
+ end
163
+ end
164
+
165
+ conf = Configuration.new(Params['conf_file'])
166
+ threads = Array.new
167
+ conf.server_conf_vec.each do |server|
168
+ threads.push(Thread.new { Crawler.new(server, Params['cd_out'], Params['cd_in']) })
169
+ end
170
+
171
+ threads.each { |a| a.join }
172
+ join_servers_results(conf.server_conf_vec, Params['cd_out'])
173
+ elsif Params['command'] == 'generate_files'
174
+ fg = FileGenerator::FileGenerator.new()
175
+ fg.run()
176
+ end
177
+ end
178
+
179
+ def self.contet_data_command(command, cd_a, cd_b, dest_path)
180
+ dest = nil
181
+ if command == "merge"
182
+ dest = ContentData.merge(cd_a, cd_b)
183
+ elsif command == "intersect"
184
+ dest = ContentData.intersect(cd_a, cd_b)
185
+ elsif command == "minus"
186
+ dest = ContentData.remove(cd_b, cd_a)
187
+ end
188
+ if dest
189
+ dest.to_file(dest_path)
190
+ end
191
+ end
192
+
193
+ # Unify modification/first_appearance time according to input DB
194
+ # Input: ContentData
195
+ # Output: ContentData with unified times
196
+ # Files modification time attribute physically updated
197
+ # Assumption: Files in the input db are from this device.
198
+ # There is no check what device they are belong - only the check of server name.
199
+ # It is a user responsibility (meanwhile) to provide a correct input
200
+ # If file has a modification time attribute that doesn't present in the input db,
201
+ # then the assumption is that this file wasnt indexized and it will not be treated
202
+ # (e.i. we do nothing with it)
203
+ def self.unify_time(content_data)
204
+ orig_content_data = ContentData::ContentData.new(content_data)
205
+ content_data.unify_time
206
+ content_data.each_instance { |checksum, size, content_mod_time, unified_inst_mod_time, server, path|
207
+ location = [server, path]
208
+ orig_inst_mod_time = orig_content_data.get_instance_mod_time(checksum, location)
209
+ file_mtime, file_size = File.open(path) { |f| [f.mtime, f.size] }
210
+ Log.debug1 "file:#{path} file_mtime:#{file_mtime}."
211
+ Log.debug1 "update mtime:#{unified_inst_mod_time}"
212
+ Log.debug1 "original instance mtime:#{orig_inst_mod_time}."
213
+ Log.debug1 "unify instance mtime:#{unified_inst_mod_time}."
214
+ Log.debug1 "Comparison: Real file = unified? #{file_mtime.to_i == unified_inst_mod_time}"
215
+ if (file_mtime.to_i == orig_inst_mod_time) \
216
+ and file_size == size \
217
+ and (file_mtime.to_i != unified_inst_mod_time)
218
+ Log.debug1 ("Comparison results: File actual time is same as instance time before unification. Need to modify file time")
219
+ File.utime(File.atime(path), unified_inst_mod_time, path)
220
+ file_mtime = File.open(path) { |f| f.mtime }
221
+ Log.debug1 "new file mtime:#{file_mtime}."
222
+ Log.debug1 "new file mtime in seconds:#{file_mtime.to_i}."
223
+ end
224
+ }
225
+ content_data
226
+ end
227
+
228
+ # Creates directory structure and symlinks with ref_cd structure to the base_cd files and dest as a root dir
229
+ # Parameters: ref_cd [ContentData]
230
+ # base_cd [ContentData]
231
+ # dest [String]
232
+ # Output: ContentData object consists of contents/instances from ref_cd that have no target in base_cd
233
+
234
+ def self.mksymlink(ref_cd, base_cd, dest)
235
+ # symlinks are not implemented in Windows
236
+ raise NotImplementedError.new if (RUBY_PLATFORM =~ /mingw/ or RUBY_PLATFORM =~ /ms/ or RUBY_PLATFORM =~ /win/)
237
+
238
+ not_found = {}
239
+ not_found_cd = ContentData::ContentData.new
240
+ warnings = []
241
+ dest.chop! if (dest.end_with?("/") or dest.end_with?("\\"))
242
+
243
+ ref_cd.each_instance { |checksum, size, content_mod_time, inst_mod_time, server, path|
244
+ if base_cd.content_exists(checksum)
245
+ symlink_path = dest + path
246
+ ::FileUtils.mkdir_p(File.dirname(symlink_path)) unless (Dir.exists?(File.dirname(symlink_path)))
247
+ File.symlink(path, symlink_path)
248
+ else
249
+ # add instance to not_found cd
250
+ not_found_cd.add_instance(checksum, size, server, path, inst_mod_time)
251
+ warnings << "Warning: base content does not contains:'%s'" % checksum
252
+ end
253
+ }
254
+ Log.warning (warnings)
255
+ not_found_cd
256
+ end
257
+ end
258
+
259
+ end
260
+
@@ -0,0 +1,3 @@
1
+ module FileUtils
2
+ VERSION = "1.1.0"
3
+ end
data/lib/file_utils.rb ADDED
@@ -0,0 +1,4 @@
1
+ require 'file_utils/file_utils'
2
+
3
+ module FileUtils
4
+ end
@@ -0,0 +1,3 @@
1
+ module Log
2
+ VERSION = "1.1.0"
3
+ end
data/lib/log.rb ADDED
@@ -0,0 +1,188 @@
1
+ # Author: Yaron Dror (yaron.dror.bb@gmail.com)
2
+ # Description: The file contains the code which implements the 'Log' module
3
+ # Run: Add to 'require' list. Execute Log.init
4
+
5
+ #require 'email'
6
+ require 'log4r'
7
+ require 'log4r/outputter/emailoutputter'
8
+
9
+ require 'params'
10
+
11
+ # Module: Log.
12
+ # Abstruct: The Log is used to log info\warning\error\debug messages
13
+ # This module is actually a wrapper to log4r gem and serves
14
+ # as a central code to use the log utility.
15
+ module Log
16
+
17
+ #Auxiliary method to retrieve the executable name
18
+ def Log.executable_name
19
+ /([a-zA-Z0-9\-_\.]+):\d+/ =~ caller[caller.size-1]
20
+ return $1
21
+ end
22
+
23
+ # Global params
24
+ Params.integer('log_debug_level', 0 , \
25
+ 'Verbosity of logging. 0 will log only INFO messages. Other value, will log all DEBUG messages as well.')
26
+ Params.boolean('log_flush_each_message', true, \
27
+ 'If true then File and Console outputters will be flushed for each message.')
28
+ Params.boolean('log_write_to_file', true , \
29
+ 'If true then the logger will write the messages to a file.')
30
+ Params.path('log_file_name', "~/.bbfs/log/#{Log.executable_name}.log4r" , \
31
+ 'log file name: ~/.bbfs/log/<executable_name>.log')
32
+ Params.integer('log_rotation_size',1000000 , \
33
+ 'max log file size. when reaching this size, a new file is created for rest of log')
34
+ Params.boolean('log_write_to_console', false , \
35
+ 'If true then the logger will write the messages to the console.')
36
+ Params.boolean('log_write_to_email', false , \
37
+ 'If true then the logger will write the error and fatal messages to email.')
38
+ Params.string('from_email', 'bbfsdev@gmail.com', 'From gmail address for update.')
39
+ Params.string('from_email_password', '', 'From gmail password.')
40
+ Params.string('to_email', 'bbfsdev@gmail.com', 'Destination email for updates.')
41
+
42
+ #Should be called from executable right after params handling.
43
+ # Init Log level and set output to file,stdout and email according to configuration params.
44
+ def Log.init
45
+ @log4r = Log4r::Logger.new 'BBFS log'
46
+ @log4r.trace = true
47
+
48
+ #levels setup
49
+ log4r_level = Log4r::DEBUG
50
+ log4r_level = Log4r::INFO if 0 == Params['log_debug_level']
51
+
52
+ #formatters
53
+ formatter = Log4r::PatternFormatter.new(:pattern => "[%l] [%d] [%m]")
54
+
55
+ #stdout setup
56
+ if Params['log_write_to_console']
57
+ stdout_outputter = Log4r::Outputter.stdout
58
+ stdout_outputter.formatter = formatter
59
+ stdout_outputter.level = log4r_level
60
+ @log4r.outputters << stdout_outputter
61
+ end
62
+
63
+ #file setup
64
+ if Params['log_write_to_file']
65
+ if File.exist?(Params['log_file_name'])
66
+ File.delete(Params['log_file_name'])
67
+ else
68
+ dir_name = File.dirname(Params['log_file_name'])
69
+ FileUtils.mkdir_p(dir_name) unless File.directory?(dir_name)
70
+ end
71
+ file_config = {
72
+ "filename" => Params['log_file_name'],
73
+ "maxsize" => Params['log_rotation_size'],
74
+ "trunc" => true
75
+ }
76
+ file_outputter = Log4r::RollingFileOutputter.new("file_log", file_config)
77
+ file_outputter.level = log4r_level
78
+ file_outputter.formatter = formatter
79
+ @log4r.outputters << file_outputter
80
+ end
81
+
82
+ #email setup
83
+ if Params['log_write_to_email']
84
+ email_outputter = Log4r::EmailOutputter.new('email_log',
85
+ :server => 'smtp.gmail.com',
86
+ :port => 587,
87
+ :subject => "Error happened at server:'#{Params['local_server_name']}' run by #{ENV['USER']}. Service_name is #{Params['service_name']}",
88
+ :acct => Params['from_email'],
89
+ :from => Params['from_email'],
90
+ :passwd => Params['from_email_password'],
91
+ :to => Params['to_email'],
92
+ :immediate_at => 'FATAL,ERROR',
93
+ :authtype => :plain,
94
+ :tls => true,
95
+ :formatfirst => true,
96
+ :buffsize => 9999,
97
+ )
98
+ email_outputter.level = Log4r::ERROR
99
+ email_outputter.formatter = formatter
100
+ @log4r.outputters << email_outputter
101
+ end
102
+
103
+ # Write init message and user parameters
104
+ @log4r.info('BBFS Log initialized.') # log first data
105
+
106
+ # print params to console
107
+ if Params['print_params_to_stdout']
108
+ Params.get_init_info_messages().each { |msg|
109
+ puts(msg)
110
+ }
111
+ else
112
+ Log.info("Not printing executable parameters to console since param:'print_params_to_stdout' is false")
113
+ end
114
+ #print params to log info and warning
115
+ Params.get_init_info_messages().each { |msg|
116
+ Log.info(msg)
117
+ }
118
+ Params.get_init_warning_messages().each { |msg|
119
+ Log.warning(msg)
120
+ }
121
+ end
122
+
123
+ # Auxiliary method to add the calling method to the message
124
+ def Log.msg_with_caller(msg)
125
+ /([a-zA-Z0-9\-_\.]+:\d+)/ =~ caller[1]
126
+ $1 + ':' + msg
127
+ end
128
+
129
+ # Log warning massages
130
+ def Log.warning(msg)
131
+ Log.init if @log4r.nil?
132
+ @log4r.warn(msg_with_caller(msg))
133
+ Log.flush if Params['log_flush_each_message']
134
+ end
135
+
136
+ # Log error massages
137
+ def Log.error(msg)
138
+ Log.init if @log4r.nil?
139
+ @log4r.error(msg_with_caller(msg))
140
+ Log.flush if Params['log_flush_each_message']
141
+ end
142
+
143
+ # Log info massages
144
+ def Log.info(msg)
145
+ Log.init if @log4r.nil?
146
+ @log4r.info(msg_with_caller(msg))
147
+ Log.flush if Params['log_flush_each_message']
148
+ end
149
+
150
+ # Log debug level 1 massages
151
+ def Log.debug1(msg)
152
+ if Params['log_debug_level'] >= 1
153
+ Log.init if @log4r.nil?
154
+ @log4r.debug(msg_with_caller(msg))
155
+ Log.flush if Params['log_flush_each_message']
156
+ end
157
+ end
158
+
159
+ # Log debug level 2 massages
160
+ def Log.debug2(msg)
161
+ if Params['log_debug_level'] >= 2
162
+ Log.init if @log4r.nil?
163
+ @log4r.debug(msg_with_caller(msg))
164
+ Log.flush if Params['log_flush_each_message']
165
+ end
166
+ end
167
+
168
+ # Log debug level 3 massages
169
+ def Log.debug3(msg)
170
+ if Params['log_debug_level'] >= 3
171
+ Log.init if @log4r.nil?
172
+ @log4r.debug(msg_with_caller(msg))
173
+ Log.flush if Params['log_flush_each_message']
174
+ end
175
+ end
176
+
177
+ # Flush email log
178
+ def Log.flush()
179
+ return if @log4r.nil?
180
+ @log4r.outputters.each { |o|
181
+ # Not flushing to email since this will cause empty emails to be sent
182
+ # Email is already configured to immediately send mail on ERROR|FATAL messages.
183
+ o.flush unless o.is_a?(Log4r::EmailOutputter)
184
+ }
185
+ end
186
+
187
+ private_class_method(:msg_with_caller)
188
+ end
@@ -0,0 +1,213 @@
1
+ require 'log'
2
+ require 'socket'
3
+
4
+ module Networking
5
+
6
+ Params.integer('client_retry_delay', 60, 'Number of seconds before trying to reconnect.')
7
+
8
+ # TODO(kolman): To get robustness, just use try catch + return 0 bytes on write +
9
+ # return false on status on read.
10
+ def Networking.write_to_stream(stream, obj)
11
+ Log.debug3('Writing to stream.')
12
+ marshal_data = Marshal.dump(obj)
13
+ Log.debug2("Writing data size: #{marshal_data.length}")
14
+ data_size = [marshal_data.length].pack("l")
15
+ if data_size.nil? || marshal_data.nil?
16
+ Log.debug2 'Send data size is nil!'
17
+ end
18
+ begin
19
+ bytes_written = stream.write data_size
20
+ bytes_written += stream.write marshal_data
21
+ return bytes_written
22
+ rescue Exception => e
23
+ Log.warning("Could not write tcp/ip stream, #{e.to_s}")
24
+ begin
25
+ stream.close()
26
+ rescue IOError => e
27
+ Log.warning("Could not close stream, #{e.to_s}.")
28
+ end
29
+ return 0
30
+ end
31
+ end
32
+
33
+ # Returns pair [status(true/false), obj]
34
+ def Networking.read_from_stream(stream)
35
+ Log.debug3('Read from stream.')
36
+ begin
37
+ return [false, nil] unless size_of_data = stream.read(4)
38
+ size_of_data = size_of_data.unpack("l")[0]
39
+ Log.debug2("Reading data size:#{size_of_data}")
40
+ data = stream.read(size_of_data)
41
+ rescue Exception => e
42
+ Log.warning("Could not read tcp/ip stream, #{e.to_s}.")
43
+ begin
44
+ stream.close()
45
+ rescue IOError => e
46
+ Log.warning("Could not close stream, #{e.to_s}.")
47
+ end
48
+ return [false, nil]
49
+ end
50
+
51
+ unmarshalled_data = Marshal.load(data)
52
+ Log.debug2('Read good.')
53
+ return [true, unmarshalled_data]
54
+ end
55
+
56
+ # Limit on number of concurrent connections?
57
+ # The TCP server is not responsible for reconnection.
58
+ class TCPServer
59
+ attr_reader :tcp_thread
60
+
61
+ def initialize(port, obj_clb, new_clb=nil, closed_clb=nil, max_parallel=1)
62
+ @port = port
63
+ @obj_clb = obj_clb
64
+ @new_clb = new_clb
65
+ @closed_clb = closed_clb
66
+ # Max parallel connections is not implemented yet.
67
+ @max_parallel = max_parallel
68
+ @sockets = {}
69
+ @tcp_thread = run_server
70
+ @tcp_thread.abort_on_exception = true
71
+ end
72
+
73
+ def send_obj(obj, addr_info=nil)
74
+ Log.debug3("send_obj with addr_info:#{addr_info}")
75
+ unless addr_info.nil?
76
+ if @sockets.key?(addr_info)
77
+ Networking.write_to_stream(@sockets[addr_info], obj)
78
+ else
79
+ Log.warning("Could not find client socket: #{addr_info}")
80
+ return 0
81
+ end
82
+ else
83
+ out = {}
84
+ @sockets.each { |key, sock| out[key] = Networking.write_to_stream(sock, obj) }
85
+ return out
86
+ end
87
+ end
88
+
89
+ private
90
+ # Creates new thread/pool
91
+ def run_server
92
+ Log.debug3('run_server')
93
+ return Thread.new do
94
+ loop {
95
+ begin
96
+ Socket.tcp_server_loop(@port) do |sock, addr_info|
97
+ Log.debug2("----- #{@port} -----")
98
+ Log.debug2("tcp_server_loop... #{sock} #{addr_info.inspect}")
99
+ @sockets[addr_info] = sock
100
+ @new_clb.call(addr_info) if @new_clb != nil
101
+ loop do
102
+ # Blocking read.
103
+ Log.debug2('read_from_stream')
104
+ stream_ok, obj = Networking.read_from_stream(sock)
105
+ Log.debug2("Server returned from read: #{stream_ok}")
106
+ @obj_clb.call(addr_info, obj) if @obj_clb != nil && stream_ok
107
+ break if !stream_ok
108
+ end
109
+ Log.warning("Connection broken, #{addr_info.inspect}")
110
+ begin
111
+ @sockets[addr_info].close() unless @sockets[addr_info].nil?
112
+ rescue IOError => e
113
+ Log.warning("Could not close socket, #{e.to_s}.")
114
+ end
115
+ @sockets.delete(addr_info)
116
+ @closed_clb.call(addr_info) if @closed_clb != nil
117
+ end
118
+ # !!! note this break is needed for tests only !!!! server loop should never end.
119
+ # and if it ends, it will fail, and rescue will work.
120
+ break
121
+ rescue IOError => e
122
+ Log.warning("Connection broken during tcp_server_loop. Restarting server loop " \
123
+ "port:#{port}.")
124
+ end
125
+ }
126
+ end
127
+ end
128
+ end # TCPServer
129
+
130
+ class TCPClient
131
+ attr_reader :tcp_thread
132
+
133
+ def initialize(host, port, obj_clb=nil, reconnected_clb=nil)
134
+ @host = host
135
+ @port = port
136
+ @tcp_socket = nil
137
+ @obj_clb = obj_clb
138
+ @reconnected_clb = reconnected_clb
139
+ Log.debug3("Start TCPClient initialize with @obj_clb: #{@obj_clb}")
140
+ if @obj_clb != nil
141
+ @tcp_thread = start_reading
142
+ @tcp_thread.abort_on_exception = true
143
+ end
144
+ # Variable to signal when remote server is ready.
145
+ @remote_server_available = ConditionVariable.new
146
+ @remote_server_available_mutex = Mutex.new
147
+ open_socket unless socket_good?
148
+ Log.debug3("End TCPClient initialize.")
149
+ end
150
+
151
+ def send_obj(obj)
152
+ Log.debug1('send_obj')
153
+ open_socket unless socket_good?
154
+ Log.debug1('after open')
155
+ if !socket_good?
156
+ Log.warning('Socket not opened for writing, skipping send.')
157
+ return false
158
+ end
159
+ Log.debug2("writing... socket port: #{@tcp_socket.peeraddr}")
160
+ bytes_written = Networking.write_to_stream(@tcp_socket, obj)
161
+ return bytes_written
162
+ end
163
+
164
+ # This function may be executed only from one thread!!! or in synchronized manner.
165
+ # private
166
+ def open_socket
167
+ Log.debug1("Connecting to content server #{@host}:#{@port}.")
168
+ begin
169
+ @tcp_socket = TCPSocket.new(@host, @port)
170
+ rescue Errno::ECONNREFUSED
171
+ Log.warning('Connection refused')
172
+ end
173
+ Log.debug3("Reconnect clb: '#{@reconnected_clb.nil? ? 'nil' : @reconnected_clb}'")
174
+ if socket_good?
175
+ @remote_server_available_mutex.synchronize {
176
+ @remote_server_available.signal
177
+ }
178
+ @reconnected_clb.call if @reconnected_clb != nil && socket_good?
179
+ end
180
+ end
181
+
182
+ private
183
+ def socket_good?
184
+ Log.debug3("socket_good? #{@tcp_socket != nil && !@tcp_socket.closed?}")
185
+ return @tcp_socket != nil && !@tcp_socket.closed?
186
+ end
187
+
188
+ private
189
+ def start_reading
190
+ return Thread.new do
191
+ loop do
192
+ Log.debug3('Start blocking read (TCPClient).')
193
+ # Blocking read.
194
+ if !socket_good?
195
+ Log.warning("Socket not good, waiting for reconnection with " \
196
+ "#{Params['client_retry_delay']} seconds timeout.")
197
+ @remote_server_available_mutex.synchronize {
198
+ @remote_server_available.wait(@remote_server_available_mutex,
199
+ Params['client_retry_delay'])
200
+ }
201
+ else
202
+ read_ok, obj = Networking.read_from_stream(@tcp_socket)
203
+ Log.debug3("Client returned from read: #{read_ok}")
204
+ # Handle case when socket is closed in middle.
205
+ # In that case we should not call obj_clb.
206
+ @obj_clb.call(obj) if (read_ok && @obj_clb != nil)
207
+ end
208
+ end
209
+ end
210
+ end
211
+ end # TCPClient
212
+ end
213
+
@@ -0,0 +1,3 @@
1
+ module Networking
2
+ VERSION = "1.1.0"
3
+ end
data/lib/networking.rb ADDED
@@ -0,0 +1,4 @@
1
+ require 'networking/tcp'
2
+
3
+ module Networking
4
+ end
@@ -0,0 +1,3 @@
1
+ module Params
2
+ VERSION = "1.1.0"
3
+ end