content_server 1.1.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. data/bin/backup_server +8 -20
  2. data/bin/content_server +8 -20
  3. data/bin/testing_memory +60 -0
  4. data/bin/testing_server +57 -0
  5. data/ext/run_in_background/mkrf_conf.rb +34 -0
  6. data/lib/content_data/content_data.rb +613 -0
  7. data/lib/content_data/version.rb +3 -0
  8. data/lib/content_data.rb +6 -0
  9. data/lib/content_server/backup_server.rb +65 -86
  10. data/lib/content_server/content_server.rb +47 -77
  11. data/lib/content_server/file_streamer.rb +27 -33
  12. data/lib/content_server/queue_copy.rb +154 -49
  13. data/lib/content_server/queue_indexer.rb +19 -11
  14. data/lib/content_server/remote_content.rb +41 -23
  15. data/lib/content_server/server.rb +91 -0
  16. data/lib/content_server/version.rb +1 -1
  17. data/lib/content_server.rb +0 -15
  18. data/lib/email/email.rb +87 -0
  19. data/lib/email/version.rb +3 -0
  20. data/lib/email.rb +4 -0
  21. data/lib/file_copy/copy.rb +68 -0
  22. data/lib/file_copy/version.rb +4 -0
  23. data/lib/file_copy.rb +4 -0
  24. data/lib/file_indexing/index_agent.rb +170 -0
  25. data/lib/file_indexing/indexer_patterns.rb +72 -0
  26. data/lib/file_indexing/version.rb +3 -0
  27. data/lib/file_indexing.rb +9 -0
  28. data/lib/file_monitoring/file_monitoring.rb +105 -0
  29. data/lib/file_monitoring/monitor_path.rb +304 -0
  30. data/lib/file_monitoring/version.rb +3 -0
  31. data/lib/file_monitoring.rb +29 -0
  32. data/lib/file_utils/file_generator/README +97 -0
  33. data/lib/file_utils/file_generator/file_generator.rb +156 -0
  34. data/lib/file_utils/file_utils.rb +260 -0
  35. data/lib/file_utils/version.rb +3 -0
  36. data/lib/file_utils.rb +4 -0
  37. data/lib/log/version.rb +3 -0
  38. data/lib/log.rb +188 -0
  39. data/lib/networking/tcp.rb +213 -0
  40. data/lib/networking/version.rb +3 -0
  41. data/lib/networking.rb +4 -0
  42. data/lib/params/version.rb +3 -0
  43. data/lib/params.rb +419 -0
  44. data/lib/process_monitoring/monitoring.rb +85 -0
  45. data/lib/process_monitoring/monitoring_info.rb +79 -0
  46. data/lib/process_monitoring/send_email.rb +40 -0
  47. data/lib/process_monitoring/thread_safe_hash.rb +77 -0
  48. data/lib/process_monitoring/version.rb +3 -0
  49. data/lib/process_monitoring.rb +6 -0
  50. data/lib/run_in_background/version.rb +3 -0
  51. data/lib/run_in_background.rb +432 -0
  52. data/lib/testing_memory/testing_memory.rb +187 -0
  53. data/lib/testing_server/testing_server.rb +236 -0
  54. data/lib/testing_server/version.rb +3 -0
  55. data/lib/testing_server.rb +12 -0
  56. data/lib/validations/index_validations.rb +106 -0
  57. data/lib/validations/version.rb +3 -0
  58. data/lib/validations.rb +4 -0
  59. data/spec/content_data/validations_spec.rb +113 -0
  60. data/spec/file_copy/copy_spec.rb +54 -0
  61. data/spec/file_indexing/index_agent_spec.rb +53 -0
  62. data/spec/networking/tcp_spec.rb +95 -0
  63. data/spec/validations/index_validations_spec.rb +77 -0
  64. data/test/content_data/content_data_test.rb +290 -0
  65. data/test/file_generator/file_generator_spec.rb +84 -0
  66. data/test/file_indexing/index_agent_test/New.txt +0 -0
  67. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/bin/libexslt.dll +0 -0
  68. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/bin/libxslt.dll +0 -0
  69. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/bin/xsltproc.exe +0 -0
  70. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libexslt/exslt.h +102 -0
  71. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libexslt/exsltconfig.h +73 -0
  72. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libexslt/exsltexports.h +140 -0
  73. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libexslt/libexslt.h +29 -0
  74. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/attributes.h +38 -0
  75. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/documents.h +93 -0
  76. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/extensions.h +262 -0
  77. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/extra.h +80 -0
  78. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/functions.h +78 -0
  79. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/imports.h +75 -0
  80. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/keys.h +53 -0
  81. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/libxslt.h +30 -0
  82. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/namespaces.h +68 -0
  83. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/numbersInternals.h +69 -0
  84. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/pattern.h +81 -0
  85. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/preproc.h +43 -0
  86. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/security.h +104 -0
  87. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/templates.h +77 -0
  88. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/transform.h +207 -0
  89. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/trio.h +216 -0
  90. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/triodef.h +220 -0
  91. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/variables.h +91 -0
  92. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/win32config.h +101 -0
  93. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/xslt.h +103 -0
  94. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/xsltInternals.h +1967 -0
  95. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/xsltconfig.h +172 -0
  96. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/xsltexports.h +142 -0
  97. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/xsltlocale.h +57 -0
  98. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/xsltutils.h +309 -0
  99. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/include/libxslt/xsltwin32config.h +105 -0
  100. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/lib/libexslt.lib +0 -0
  101. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/lib/libexslt_a.lib +0 -0
  102. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/lib/libxslt.lib +0 -0
  103. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/lib/libxslt_a.lib +0 -0
  104. data/test/file_indexing/index_agent_test/libxslt-1.1.26.win32/readme.txt +22 -0
  105. data/test/file_indexing/index_agent_test/patterns.input +3 -0
  106. data/test/file_indexing/index_agent_test.rb +51 -0
  107. data/test/file_monitoring/file_monitoring_test/conf.yml +4 -0
  108. data/test/file_monitoring/file_monitoring_test/conf_win32.yml +5 -0
  109. data/test/file_monitoring/file_monitoring_test/log +56 -0
  110. data/test/file_monitoring/file_monitoring_test.rb +0 -0
  111. data/test/file_monitoring/monitor_path_test/dir1000/test_file.1000 +1000 -0
  112. data/test/file_monitoring/monitor_path_test/dir1000/test_file.1000.0 +1000 -0
  113. data/test/file_monitoring/monitor_path_test/dir1000/test_file.1000.1 +1000 -0
  114. data/test/file_monitoring/monitor_path_test/dir1500/test_file.1500 +1500 -0
  115. data/test/file_monitoring/monitor_path_test/dir1500/test_file.1500.0 +1500 -0
  116. data/test/file_monitoring/monitor_path_test/dir1500/test_file.1500.1 +1500 -0
  117. data/test/file_monitoring/monitor_path_test/test_file.500 +500 -0
  118. data/test/file_monitoring/monitor_path_test/test_file.500.0 +500 -0
  119. data/test/file_monitoring/monitor_path_test/test_file.500.1 +500 -0
  120. data/test/file_monitoring/monitor_path_test.rb +153 -0
  121. data/test/file_utils/fileutil_mksymlink_test/dir1000/dir1500/test_file.1500 +1500 -0
  122. data/test/file_utils/fileutil_mksymlink_test/dir1000/dir1500/test_file.1500.0 +1500 -0
  123. data/test/file_utils/fileutil_mksymlink_test/dir1000/dir1500/test_file.1500.1 +1500 -0
  124. data/test/file_utils/fileutil_mksymlink_test/dir1000/test_file.1000 +1000 -0
  125. data/test/file_utils/fileutil_mksymlink_test/dir1000/test_file.1000.0 +1000 -0
  126. data/test/file_utils/fileutil_mksymlink_test/dir1000/test_file.1000.1 +1000 -0
  127. data/test/file_utils/fileutil_mksymlink_test/test_file.500 +500 -0
  128. data/test/file_utils/fileutil_mksymlink_test/test_file.500.0 +500 -0
  129. data/test/file_utils/fileutil_mksymlink_test/test_file.500.1 +500 -0
  130. data/test/file_utils/fileutil_mksymlink_test.rb +125 -0
  131. data/test/file_utils/time_modification_test.rb +132 -0
  132. data/test/params/params_spec.rb +280 -0
  133. data/test/params/params_test.rb +43 -0
  134. data/test/run_in_background/run_in_background_test.rb +122 -0
  135. data/test/run_in_background/test_app +57 -0
  136. metadata +272 -132
  137. data/lib/content_server/globals.rb +0 -10
@@ -0,0 +1,260 @@
1
+ # Author:
2
+ # originator: Kolman Vornovizky (kolmanv@gmail.com)
3
+ # update: Alexey Nemytov (alexeyn66@gmail.com) 31/07/2012
4
+ # This file is the consist utility functionality for BBFS
5
+ # Such as automatic file generation, symlink, etc.
6
+ # Run from bbfs> ruby -Ilib bin/file_utils --command=generate_files --log_write_to_console=true --log_debug_level=3
7
+
8
+ require 'content_data'
9
+ require 'file_indexing'
10
+ require 'file_utils'
11
+ require 'file_utils/file_generator/file_generator'
12
+
13
+ require 'log'
14
+ require 'params'
15
+
16
+ module FileUtils
17
+ Params.string 'command', nil ,'path'
18
+ Params.string 'ref_cd', nil ,'reference path'
19
+ Params.string 'base_cd', nil ,'base path'
20
+ Params.string 'dest', nil ,'destination path'
21
+ Params.string 'dest_server', nil ,'server name'
22
+ Params.string 'patterns', nil ,'patterns path'
23
+ Params.string 'cd', nil ,'path'
24
+ Params.string 'cd_a', nil ,'a path'
25
+ Params.string 'cd_b', nil ,'b path'
26
+ Params.string 'cd_out', nil ,'output path'
27
+ Params.string 'cd_in', nil ,'input path'
28
+ Params.string 'exist_cd', nil, 'exist_path'
29
+ #Params.string 'config_file', 'path' ,'configuration file path'
30
+ class FileUtils
31
+ def FileUtils.run
32
+ if Params['command'] == 'mksymlink'
33
+ if Params['ref_cd'].nil?
34
+ Log.error ("--ref_cd is not set")
35
+ return
36
+ end
37
+ ref_cd = ContentData.new()
38
+ ref_cd.from_file(Params['ref_cd'])
39
+ if ref_cd.nil?
40
+ Log.error ("Error loading content data ref_cd=%s" % Params['ref_cd'])
41
+ return
42
+ end
43
+
44
+ if Params['base_cd'].nil?
45
+ Log.error ("--base_cd is not set")
46
+ return
47
+ end
48
+ base_cd = ContentData.new()
49
+ base_cd.from_file(Params['base_cd'])
50
+ if base_cd.nil?
51
+ Log.error ("Error loading content data base_cd=%s" % Params['base_cd'])
52
+ return
53
+ end
54
+
55
+ if Params['dest'].nil?
56
+ Log.error ("--dest is not set")
57
+ return
58
+ end
59
+
60
+ not_found = nil
61
+ begin
62
+ not_found = FileUtil.mksymlink(ref_cd, base_cd, Params['dest'])
63
+ rescue NotImplementedError
64
+ Log.error ("symlinks are unimplemented on this machine")
65
+ return nil
66
+ end
67
+ return not_found
68
+ elsif (Params['command'] == "merge" or
69
+ Params['command'] == "intersect" or
70
+ Params['command'] == "minus")
71
+
72
+ if Params['cd_a'].nil?
73
+ Log.error ("--cd_a is not set")
74
+ return
75
+ end
76
+ cd_a = ContentData.new()
77
+ cd_a.from_file(Params['cd_a'])
78
+ if cd_a.nil?
79
+ Log.error ("Error loading content data cd_a=%s" % Params['cd_a'])
80
+ return
81
+ end
82
+
83
+ if Params['cd_b'].nil?
84
+ Log.error ("--cd_b is not set")
85
+ return
86
+ end
87
+ cd_b = ContentData.new()
88
+ cd_b.from_file(Params['cd_b'])
89
+ if cd_b.nil?
90
+ Log.error ("Error loading content data cd_b=%s" % Params['cd_b'])
91
+ return
92
+ end
93
+
94
+ if Params['cd_b'].nil?
95
+ Log.error ("--dest is not set")
96
+ return
97
+ end
98
+
99
+ output = FileUtil.contet_data_command(Params['command'], cd_a, cd_b, Params['dest'])
100
+
101
+ elsif Params['command'] == 'unify_time'
102
+ if Params['cd'].nil?
103
+ Log.error ("--cd is not set")
104
+ return
105
+ end
106
+ cd = ContentData.new()
107
+ cd.from_file(Params['cd'])
108
+ if cd.nil?
109
+ Log.error ("Error loading content data cd=%s" % Params['cd'])
110
+ return
111
+ end
112
+ output = unify_time(cd)
113
+ # indexer
114
+ elsif Params['command'] == 'indexer'
115
+ if Params['patterns'].nil?
116
+ Log.error ("--patterns is not set")
117
+ return
118
+ end
119
+
120
+ patterns = FileIndexing::IndexerPatterns.new
121
+ Params['patterns'].split(':').each { |pattern|
122
+ Log.debug1 "Pattern: #{pattern}"
123
+ patterns.add_pattern File.expand_path(pattern)
124
+ }
125
+
126
+ unless patterns.size > 0
127
+ Log.error ("Error loading patterns=%s (empty file)" % Params['patterns'])
128
+ return
129
+ end
130
+
131
+ exist_cd = nil
132
+ if not Params['exist_cd'].nil?
133
+ exist_cd = ContentData::ContentData.new()
134
+ exist_cd.from_file(Params['exist_cd'])
135
+ if exist_cd.nil?
136
+ Log.error ("Error loading content data exist_cd=%s" % Params['exist_cd'])
137
+ return
138
+ end
139
+ end
140
+ indexer = FileIndexing::IndexAgent.new
141
+ indexer.index(patterns, exist_cd)
142
+ Log.debug1 indexer.indexed_content.to_s
143
+ # crawler
144
+ elsif Params['command'] == 'crawler'
145
+ if Params['conf_file'].nil?
146
+ Log.error ("--conf_file is not set")
147
+ return
148
+ end
149
+ if not File.exists?(Params['conf_file'])
150
+ Log.error ("config file doesn't exist conf_file=%s" % Params['conf_file'])
151
+ return
152
+ end
153
+
154
+ if Params['cd_out'].nil?
155
+ time = Tme.now.utc
156
+ Params['cd_out'] = "crawler.out.#{time.strftime('%Y/%m/%d_%H-%M-%S')}"
157
+ end
158
+ unless (Params['cd_in'].nil?)
159
+ if not File.exists?(Params['conf_file'])
160
+ Log.error ("input data file doesn't exist cd_in=%s" % Params['cd_in'])
161
+ return
162
+ end
163
+ end
164
+
165
+ conf = Configuration.new(Params['conf_file'])
166
+ threads = Array.new
167
+ conf.server_conf_vec.each do |server|
168
+ threads.push(Thread.new { Crawler.new(server, Params['cd_out'], Params['cd_in']) })
169
+ end
170
+
171
+ threads.each { |a| a.join }
172
+ join_servers_results(conf.server_conf_vec, Params['cd_out'])
173
+ elsif Params['command'] == 'generate_files'
174
+ fg = FileGenerator::FileGenerator.new()
175
+ fg.run()
176
+ end
177
+ end
178
+
179
+ def self.contet_data_command(command, cd_a, cd_b, dest_path)
180
+ dest = nil
181
+ if command == "merge"
182
+ dest = ContentData.merge(cd_a, cd_b)
183
+ elsif command == "intersect"
184
+ dest = ContentData.intersect(cd_a, cd_b)
185
+ elsif command == "minus"
186
+ dest = ContentData.remove(cd_b, cd_a)
187
+ end
188
+ if dest
189
+ dest.to_file(dest_path)
190
+ end
191
+ end
192
+
193
+ # Unify modification/first_appearance time according to input DB
194
+ # Input: ContentData
195
+ # Output: ContentData with unified times
196
+ # Files modification time attribute physically updated
197
+ # Assumption: Files in the input db are from this device.
198
+ # There is no check what device they are belong - only the check of server name.
199
+ # It is a user responsibility (meanwhile) to provide a correct input
200
+ # If file has a modification time attribute that doesn't present in the input db,
201
+ # then the assumption is that this file wasnt indexized and it will not be treated
202
+ # (e.i. we do nothing with it)
203
+ def self.unify_time(content_data)
204
+ orig_content_data = ContentData::ContentData.new(content_data)
205
+ content_data.unify_time
206
+ content_data.each_instance { |checksum, size, content_mod_time, unified_inst_mod_time, server, path|
207
+ location = [server, path]
208
+ orig_inst_mod_time = orig_content_data.get_instance_mod_time(checksum, location)
209
+ file_mtime, file_size = File.open(path) { |f| [f.mtime, f.size] }
210
+ Log.debug1 "file:#{path} file_mtime:#{file_mtime}."
211
+ Log.debug1 "update mtime:#{unified_inst_mod_time}"
212
+ Log.debug1 "original instance mtime:#{orig_inst_mod_time}."
213
+ Log.debug1 "unify instance mtime:#{unified_inst_mod_time}."
214
+ Log.debug1 "Comparison: Real file = unified? #{file_mtime.to_i == unified_inst_mod_time}"
215
+ if (file_mtime.to_i == orig_inst_mod_time) \
216
+ and file_size == size \
217
+ and (file_mtime.to_i != unified_inst_mod_time)
218
+ Log.debug1 ("Comparison results: File actual time is same as instance time before unification. Need to modify file time")
219
+ File.utime(File.atime(path), unified_inst_mod_time, path)
220
+ file_mtime = File.open(path) { |f| f.mtime }
221
+ Log.debug1 "new file mtime:#{file_mtime}."
222
+ Log.debug1 "new file mtime in seconds:#{file_mtime.to_i}."
223
+ end
224
+ }
225
+ content_data
226
+ end
227
+
228
+ # Creates directory structure and symlinks with ref_cd structure to the base_cd files and dest as a root dir
229
+ # Parameters: ref_cd [ContentData]
230
+ # base_cd [ContentData]
231
+ # dest [String]
232
+ # Output: ContentData object consists of contents/instances from ref_cd that have no target in base_cd
233
+
234
+ def self.mksymlink(ref_cd, base_cd, dest)
235
+ # symlinks are not implemented in Windows
236
+ raise NotImplementedError.new if (RUBY_PLATFORM =~ /mingw/ or RUBY_PLATFORM =~ /ms/ or RUBY_PLATFORM =~ /win/)
237
+
238
+ not_found = {}
239
+ not_found_cd = ContentData::ContentData.new
240
+ warnings = []
241
+ dest.chop! if (dest.end_with?("/") or dest.end_with?("\\"))
242
+
243
+ ref_cd.each_instance { |checksum, size, content_mod_time, inst_mod_time, server, path|
244
+ if base_cd.content_exists(checksum)
245
+ symlink_path = dest + path
246
+ ::FileUtils.mkdir_p(File.dirname(symlink_path)) unless (Dir.exists?(File.dirname(symlink_path)))
247
+ File.symlink(path, symlink_path)
248
+ else
249
+ # add instance to not_found cd
250
+ not_found_cd.add_instance(checksum, size, server, path, inst_mod_time)
251
+ warnings << "Warning: base content does not contains:'%s'" % checksum
252
+ end
253
+ }
254
+ Log.warning (warnings)
255
+ not_found_cd
256
+ end
257
+ end
258
+
259
+ end
260
+
@@ -0,0 +1,3 @@
1
+ module FileUtils
2
+ VERSION = "1.1.0"
3
+ end
data/lib/file_utils.rb ADDED
@@ -0,0 +1,4 @@
1
+ require 'file_utils/file_utils'
2
+
3
+ module FileUtils
4
+ end
@@ -0,0 +1,3 @@
1
+ module Log
2
+ VERSION = "1.1.0"
3
+ end
data/lib/log.rb ADDED
@@ -0,0 +1,188 @@
1
+ # Author: Yaron Dror (yaron.dror.bb@gmail.com)
2
+ # Description: The file contains the code which implements the 'Log' module
3
+ # Run: Add to 'require' list. Execute Log.init
4
+
5
+ #require 'email'
6
+ require 'log4r'
7
+ require 'log4r/outputter/emailoutputter'
8
+
9
+ require 'params'
10
+
11
+ # Module: Log.
12
+ # Abstruct: The Log is used to log info\warning\error\debug messages
13
+ # This module is actually a wrapper to log4r gem and serves
14
+ # as a central code to use the log utility.
15
+ module Log
16
+
17
+ #Auxiliary method to retrieve the executable name
18
+ def Log.executable_name
19
+ /([a-zA-Z0-9\-_\.]+):\d+/ =~ caller[caller.size-1]
20
+ return $1
21
+ end
22
+
23
+ # Global params
24
+ Params.integer('log_debug_level', 0 , \
25
+ 'Verbosity of logging. 0 will log only INFO messages. Other value, will log all DEBUG messages as well.')
26
+ Params.boolean('log_flush_each_message', true, \
27
+ 'If true then File and Console outputters will be flushed for each message.')
28
+ Params.boolean('log_write_to_file', true , \
29
+ 'If true then the logger will write the messages to a file.')
30
+ Params.path('log_file_name', "~/.bbfs/log/#{Log.executable_name}.log4r" , \
31
+ 'log file name: ~/.bbfs/log/<executable_name>.log')
32
+ Params.integer('log_rotation_size',1000000 , \
33
+ 'max log file size. when reaching this size, a new file is created for rest of log')
34
+ Params.boolean('log_write_to_console', false , \
35
+ 'If true then the logger will write the messages to the console.')
36
+ Params.boolean('log_write_to_email', false , \
37
+ 'If true then the logger will write the error and fatal messages to email.')
38
+ Params.string('from_email', 'bbfsdev@gmail.com', 'From gmail address for update.')
39
+ Params.string('from_email_password', '', 'From gmail password.')
40
+ Params.string('to_email', 'bbfsdev@gmail.com', 'Destination email for updates.')
41
+
42
+ #Should be called from executable right after params handling.
43
+ # Init Log level and set output to file,stdout and email according to configuration params.
44
+ def Log.init
45
+ @log4r = Log4r::Logger.new 'BBFS log'
46
+ @log4r.trace = true
47
+
48
+ #levels setup
49
+ log4r_level = Log4r::DEBUG
50
+ log4r_level = Log4r::INFO if 0 == Params['log_debug_level']
51
+
52
+ #formatters
53
+ formatter = Log4r::PatternFormatter.new(:pattern => "[%l] [%d] [%m]")
54
+
55
+ #stdout setup
56
+ if Params['log_write_to_console']
57
+ stdout_outputter = Log4r::Outputter.stdout
58
+ stdout_outputter.formatter = formatter
59
+ stdout_outputter.level = log4r_level
60
+ @log4r.outputters << stdout_outputter
61
+ end
62
+
63
+ #file setup
64
+ if Params['log_write_to_file']
65
+ if File.exist?(Params['log_file_name'])
66
+ File.delete(Params['log_file_name'])
67
+ else
68
+ dir_name = File.dirname(Params['log_file_name'])
69
+ FileUtils.mkdir_p(dir_name) unless File.directory?(dir_name)
70
+ end
71
+ file_config = {
72
+ "filename" => Params['log_file_name'],
73
+ "maxsize" => Params['log_rotation_size'],
74
+ "trunc" => true
75
+ }
76
+ file_outputter = Log4r::RollingFileOutputter.new("file_log", file_config)
77
+ file_outputter.level = log4r_level
78
+ file_outputter.formatter = formatter
79
+ @log4r.outputters << file_outputter
80
+ end
81
+
82
+ #email setup
83
+ if Params['log_write_to_email']
84
+ email_outputter = Log4r::EmailOutputter.new('email_log',
85
+ :server => 'smtp.gmail.com',
86
+ :port => 587,
87
+ :subject => "Error happened at server:'#{Params['local_server_name']}' run by #{ENV['USER']}. Service_name is #{Params['service_name']}",
88
+ :acct => Params['from_email'],
89
+ :from => Params['from_email'],
90
+ :passwd => Params['from_email_password'],
91
+ :to => Params['to_email'],
92
+ :immediate_at => 'FATAL,ERROR',
93
+ :authtype => :plain,
94
+ :tls => true,
95
+ :formatfirst => true,
96
+ :buffsize => 9999,
97
+ )
98
+ email_outputter.level = Log4r::ERROR
99
+ email_outputter.formatter = formatter
100
+ @log4r.outputters << email_outputter
101
+ end
102
+
103
+ # Write init message and user parameters
104
+ @log4r.info('BBFS Log initialized.') # log first data
105
+
106
+ # print params to console
107
+ if Params['print_params_to_stdout']
108
+ Params.get_init_info_messages().each { |msg|
109
+ puts(msg)
110
+ }
111
+ else
112
+ Log.info("Not printing executable parameters to console since param:'print_params_to_stdout' is false")
113
+ end
114
+ #print params to log info and warning
115
+ Params.get_init_info_messages().each { |msg|
116
+ Log.info(msg)
117
+ }
118
+ Params.get_init_warning_messages().each { |msg|
119
+ Log.warning(msg)
120
+ }
121
+ end
122
+
123
+ # Auxiliary method to add the calling method to the message
124
+ def Log.msg_with_caller(msg)
125
+ /([a-zA-Z0-9\-_\.]+:\d+)/ =~ caller[1]
126
+ $1 + ':' + msg
127
+ end
128
+
129
+ # Log warning massages
130
+ def Log.warning(msg)
131
+ Log.init if @log4r.nil?
132
+ @log4r.warn(msg_with_caller(msg))
133
+ Log.flush if Params['log_flush_each_message']
134
+ end
135
+
136
+ # Log error massages
137
+ def Log.error(msg)
138
+ Log.init if @log4r.nil?
139
+ @log4r.error(msg_with_caller(msg))
140
+ Log.flush if Params['log_flush_each_message']
141
+ end
142
+
143
+ # Log info massages
144
+ def Log.info(msg)
145
+ Log.init if @log4r.nil?
146
+ @log4r.info(msg_with_caller(msg))
147
+ Log.flush if Params['log_flush_each_message']
148
+ end
149
+
150
+ # Log debug level 1 massages
151
+ def Log.debug1(msg)
152
+ if Params['log_debug_level'] >= 1
153
+ Log.init if @log4r.nil?
154
+ @log4r.debug(msg_with_caller(msg))
155
+ Log.flush if Params['log_flush_each_message']
156
+ end
157
+ end
158
+
159
+ # Log debug level 2 massages
160
+ def Log.debug2(msg)
161
+ if Params['log_debug_level'] >= 2
162
+ Log.init if @log4r.nil?
163
+ @log4r.debug(msg_with_caller(msg))
164
+ Log.flush if Params['log_flush_each_message']
165
+ end
166
+ end
167
+
168
+ # Log debug level 3 massages
169
+ def Log.debug3(msg)
170
+ if Params['log_debug_level'] >= 3
171
+ Log.init if @log4r.nil?
172
+ @log4r.debug(msg_with_caller(msg))
173
+ Log.flush if Params['log_flush_each_message']
174
+ end
175
+ end
176
+
177
+ # Flush email log
178
+ def Log.flush()
179
+ return if @log4r.nil?
180
+ @log4r.outputters.each { |o|
181
+ # Not flushing to email since this will cause empty emails to be sent
182
+ # Email is already configured to immediately send mail on ERROR|FATAL messages.
183
+ o.flush unless o.is_a?(Log4r::EmailOutputter)
184
+ }
185
+ end
186
+
187
+ private_class_method(:msg_with_caller)
188
+ end
@@ -0,0 +1,213 @@
1
+ require 'log'
2
+ require 'socket'
3
+
4
+ module Networking
5
+
6
+ Params.integer('client_retry_delay', 60, 'Number of seconds before trying to reconnect.')
7
+
8
+ # TODO(kolman): To get robustness, just use try catch + return 0 bytes on write +
9
+ # return false on status on read.
10
+ def Networking.write_to_stream(stream, obj)
11
+ Log.debug3('Writing to stream.')
12
+ marshal_data = Marshal.dump(obj)
13
+ Log.debug2("Writing data size: #{marshal_data.length}")
14
+ data_size = [marshal_data.length].pack("l")
15
+ if data_size.nil? || marshal_data.nil?
16
+ Log.debug2 'Send data size is nil!'
17
+ end
18
+ begin
19
+ bytes_written = stream.write data_size
20
+ bytes_written += stream.write marshal_data
21
+ return bytes_written
22
+ rescue Exception => e
23
+ Log.warning("Could not write tcp/ip stream, #{e.to_s}")
24
+ begin
25
+ stream.close()
26
+ rescue IOError => e
27
+ Log.warning("Could not close stream, #{e.to_s}.")
28
+ end
29
+ return 0
30
+ end
31
+ end
32
+
33
+ # Returns pair [status(true/false), obj]
34
+ def Networking.read_from_stream(stream)
35
+ Log.debug3('Read from stream.')
36
+ begin
37
+ return [false, nil] unless size_of_data = stream.read(4)
38
+ size_of_data = size_of_data.unpack("l")[0]
39
+ Log.debug2("Reading data size:#{size_of_data}")
40
+ data = stream.read(size_of_data)
41
+ rescue Exception => e
42
+ Log.warning("Could not read tcp/ip stream, #{e.to_s}.")
43
+ begin
44
+ stream.close()
45
+ rescue IOError => e
46
+ Log.warning("Could not close stream, #{e.to_s}.")
47
+ end
48
+ return [false, nil]
49
+ end
50
+
51
+ unmarshalled_data = Marshal.load(data)
52
+ Log.debug2('Read good.')
53
+ return [true, unmarshalled_data]
54
+ end
55
+
56
+ # Limit on number of concurrent connections?
57
+ # The TCP server is not responsible for reconnection.
58
+ class TCPServer
59
+ attr_reader :tcp_thread
60
+
61
+ def initialize(port, obj_clb, new_clb=nil, closed_clb=nil, max_parallel=1)
62
+ @port = port
63
+ @obj_clb = obj_clb
64
+ @new_clb = new_clb
65
+ @closed_clb = closed_clb
66
+ # Max parallel connections is not implemented yet.
67
+ @max_parallel = max_parallel
68
+ @sockets = {}
69
+ @tcp_thread = run_server
70
+ @tcp_thread.abort_on_exception = true
71
+ end
72
+
73
+ def send_obj(obj, addr_info=nil)
74
+ Log.debug3("send_obj with addr_info:#{addr_info}")
75
+ unless addr_info.nil?
76
+ if @sockets.key?(addr_info)
77
+ Networking.write_to_stream(@sockets[addr_info], obj)
78
+ else
79
+ Log.warning("Could not find client socket: #{addr_info}")
80
+ return 0
81
+ end
82
+ else
83
+ out = {}
84
+ @sockets.each { |key, sock| out[key] = Networking.write_to_stream(sock, obj) }
85
+ return out
86
+ end
87
+ end
88
+
89
+ private
90
+ # Creates new thread/pool
91
+ def run_server
92
+ Log.debug3('run_server')
93
+ return Thread.new do
94
+ loop {
95
+ begin
96
+ Socket.tcp_server_loop(@port) do |sock, addr_info|
97
+ Log.debug2("----- #{@port} -----")
98
+ Log.debug2("tcp_server_loop... #{sock} #{addr_info.inspect}")
99
+ @sockets[addr_info] = sock
100
+ @new_clb.call(addr_info) if @new_clb != nil
101
+ loop do
102
+ # Blocking read.
103
+ Log.debug2('read_from_stream')
104
+ stream_ok, obj = Networking.read_from_stream(sock)
105
+ Log.debug2("Server returned from read: #{stream_ok}")
106
+ @obj_clb.call(addr_info, obj) if @obj_clb != nil && stream_ok
107
+ break if !stream_ok
108
+ end
109
+ Log.warning("Connection broken, #{addr_info.inspect}")
110
+ begin
111
+ @sockets[addr_info].close() unless @sockets[addr_info].nil?
112
+ rescue IOError => e
113
+ Log.warning("Could not close socket, #{e.to_s}.")
114
+ end
115
+ @sockets.delete(addr_info)
116
+ @closed_clb.call(addr_info) if @closed_clb != nil
117
+ end
118
+ # !!! note this break is needed for tests only !!!! server loop should never end.
119
+ # and if it ends, it will fail, and rescue will work.
120
+ break
121
+ rescue IOError => e
122
+ Log.warning("Connection broken during tcp_server_loop. Restarting server loop " \
123
+ "port:#{port}.")
124
+ end
125
+ }
126
+ end
127
+ end
128
+ end # TCPServer
129
+
130
+ class TCPClient
131
+ attr_reader :tcp_thread
132
+
133
+ def initialize(host, port, obj_clb=nil, reconnected_clb=nil)
134
+ @host = host
135
+ @port = port
136
+ @tcp_socket = nil
137
+ @obj_clb = obj_clb
138
+ @reconnected_clb = reconnected_clb
139
+ Log.debug3("Start TCPClient initialize with @obj_clb: #{@obj_clb}")
140
+ if @obj_clb != nil
141
+ @tcp_thread = start_reading
142
+ @tcp_thread.abort_on_exception = true
143
+ end
144
+ # Variable to signal when remote server is ready.
145
+ @remote_server_available = ConditionVariable.new
146
+ @remote_server_available_mutex = Mutex.new
147
+ open_socket unless socket_good?
148
+ Log.debug3("End TCPClient initialize.")
149
+ end
150
+
151
+ def send_obj(obj)
152
+ Log.debug1('send_obj')
153
+ open_socket unless socket_good?
154
+ Log.debug1('after open')
155
+ if !socket_good?
156
+ Log.warning('Socket not opened for writing, skipping send.')
157
+ return false
158
+ end
159
+ Log.debug2("writing... socket port: #{@tcp_socket.peeraddr}")
160
+ bytes_written = Networking.write_to_stream(@tcp_socket, obj)
161
+ return bytes_written
162
+ end
163
+
164
+ # This function may be executed only from one thread!!! or in synchronized manner.
165
+ # private
166
+ def open_socket
167
+ Log.debug1("Connecting to content server #{@host}:#{@port}.")
168
+ begin
169
+ @tcp_socket = TCPSocket.new(@host, @port)
170
+ rescue Errno::ECONNREFUSED
171
+ Log.warning('Connection refused')
172
+ end
173
+ Log.debug3("Reconnect clb: '#{@reconnected_clb.nil? ? 'nil' : @reconnected_clb}'")
174
+ if socket_good?
175
+ @remote_server_available_mutex.synchronize {
176
+ @remote_server_available.signal
177
+ }
178
+ @reconnected_clb.call if @reconnected_clb != nil && socket_good?
179
+ end
180
+ end
181
+
182
+ private
183
+ def socket_good?
184
+ Log.debug3("socket_good? #{@tcp_socket != nil && !@tcp_socket.closed?}")
185
+ return @tcp_socket != nil && !@tcp_socket.closed?
186
+ end
187
+
188
+ private
189
+ def start_reading
190
+ return Thread.new do
191
+ loop do
192
+ Log.debug3('Start blocking read (TCPClient).')
193
+ # Blocking read.
194
+ if !socket_good?
195
+ Log.warning("Socket not good, waiting for reconnection with " \
196
+ "#{Params['client_retry_delay']} seconds timeout.")
197
+ @remote_server_available_mutex.synchronize {
198
+ @remote_server_available.wait(@remote_server_available_mutex,
199
+ Params['client_retry_delay'])
200
+ }
201
+ else
202
+ read_ok, obj = Networking.read_from_stream(@tcp_socket)
203
+ Log.debug3("Client returned from read: #{read_ok}")
204
+ # Handle case when socket is closed in middle.
205
+ # In that case we should not call obj_clb.
206
+ @obj_clb.call(obj) if (read_ok && @obj_clb != nil)
207
+ end
208
+ end
209
+ end
210
+ end
211
+ end # TCPClient
212
+ end
213
+
@@ -0,0 +1,3 @@
1
+ module Networking
2
+ VERSION = "1.1.0"
3
+ end
data/lib/networking.rb ADDED
@@ -0,0 +1,4 @@
1
+ require 'networking/tcp'
2
+
3
+ module Networking
4
+ end
@@ -0,0 +1,3 @@
1
+ module Params
2
+ VERSION = "1.1.0"
3
+ end