logfile_transfer 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/README.md +5 -0
- data/config.yaml +26 -0
- data/example.rb +31 -0
- data/lib/logfile_transfer.rb +310 -0
- metadata +47 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: c191ef37074b871c1e90c5fc651d0555b71ef665
|
4
|
+
data.tar.gz: 8dbe070c309ce9a13ce0df2d39ae1005083db6df
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 3bc2789385bea67f06efb976e4b2a57ecbe726e2592189dc78245d0363cb3d6a17bce5ff4da617e347af2d65e7bbe853c874529ae108534aea77979df13ef265
|
7
|
+
data.tar.gz: 777ccf8d2e186c46202e8fab4d1a1aa7e7824670e9c1804501c40a4d71a3090a065f23e2e131c6fd5e34c8e3534b406183de24f735b897b0e4bf9e0ba9e0ee2f
|
data/README.md
ADDED
data/config.yaml
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
---
|
2
|
+
- !ruby/object:LogfileTransfer::FileMonitorObj
|
3
|
+
absolute_path: /data/webroot/log
|
4
|
+
dir_disallow: []
|
5
|
+
file_disallow:
|
6
|
+
- .*
|
7
|
+
file_allow:
|
8
|
+
- \.log\.
|
9
|
+
patterns:
|
10
|
+
- - gamestart
|
11
|
+
- - !ruby/object:Test {}
|
12
|
+
- !ruby/object:Test1 {}
|
13
|
+
- - gameexit
|
14
|
+
- - !ruby/object:Test {}
|
15
|
+
- !ruby/object:Test1 {}
|
16
|
+
- !ruby/object:LogfileTransfer::FileMonitorObj
|
17
|
+
absolute_path: /data/webroot/tlog
|
18
|
+
dir_disallow: []
|
19
|
+
file_disallow:
|
20
|
+
- .*
|
21
|
+
file_allow:
|
22
|
+
- \.log\.
|
23
|
+
patterns:
|
24
|
+
- - gamestart
|
25
|
+
- - !ruby/object:Test {}
|
26
|
+
- !ruby/object:Test1 {}
|
data/example.rb
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
lib_dir = File.join File.dirname(__FILE__), 'lib'
|
4
|
+
$:.unshift lib_dir unless $:.include? lib_dir
|
5
|
+
|
6
|
+
require 'scribe'
|
7
|
+
require 'logfile_transfer.rb'
|
8
|
+
|
9
|
+
class Test < LogfileTransfer::Handler
|
10
|
+
def init
|
11
|
+
end
|
12
|
+
def handle log_path, log_fn, line, line_count, pattern
|
13
|
+
if (line_count % 2) == 0
|
14
|
+
puts '+++++++++++++++++++'
|
15
|
+
else
|
16
|
+
puts '-------------------'
|
17
|
+
end
|
18
|
+
puts "#{log_path}, #{log_fn}, #{line_count}, #{pattern}"
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
class Test1 < LogfileTransfer::Handler
|
23
|
+
def init
|
24
|
+
@scribe_client = Scribe.new('127.0.0.1:1463')
|
25
|
+
end
|
26
|
+
def handle log_path, log_fn, line, line_count, pattern
|
27
|
+
@scribe_client.log(line, pattern)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
LogfileTransfer.run ARGV, 2001, File.expand_path(File.dirname(__FILE__))
|
@@ -0,0 +1,310 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require 'file-monitor'
|
4
|
+
require 'socket'
|
5
|
+
require 'yaml'
|
6
|
+
|
7
|
+
module LogfileTransfer
|
8
|
+
Stop_cmd_file_name = '.sync_cmd_stop'
|
9
|
+
Prompt_cmdline = 'ruby your.rb start [config.yaml]|stop|status'
|
10
|
+
Prompt_running = 'daemon is running.'
|
11
|
+
Prompt_exiting = 'daemon is exiting.'
|
12
|
+
Prompt_starting = 'daemon is starting.'
|
13
|
+
Prompt_no_running = 'daemon no running.'
|
14
|
+
|
15
|
+
@hostname = 'localhost'
|
16
|
+
@port = 2001
|
17
|
+
@files = {}
|
18
|
+
@threads = []
|
19
|
+
@daemon_log_file_name = 'daemon.log'
|
20
|
+
|
21
|
+
class Handler
|
22
|
+
def init
|
23
|
+
raise NotImplementedError.new("#{self.class.name}#init is abstract method.")
|
24
|
+
end
|
25
|
+
def handle
|
26
|
+
raise NotImplementedError.new("#{self.class.name}#handle is abstract method.")
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
class FileMonitorObj
|
31
|
+
attr_accessor :absolute_path, :dir_disallow, :file_disallow, :file_allow, :patterns
|
32
|
+
def initialize
|
33
|
+
@absolute_path = ''
|
34
|
+
@dir_disallow = []
|
35
|
+
@file_disallow = []
|
36
|
+
@file_allow = []
|
37
|
+
@patterns = []
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def self.log msg
|
42
|
+
@daemon_log_file.puts msg
|
43
|
+
end
|
44
|
+
|
45
|
+
def self.daemonize_app working_directory
|
46
|
+
if RUBY_VERSION < "1.9"
|
47
|
+
exit if fork
|
48
|
+
Process.setsid
|
49
|
+
exit if fork
|
50
|
+
Dir.chdir working_directory
|
51
|
+
STDIN.reopen "/dev/null"
|
52
|
+
STDOUT.reopen "/dev/null", "a"
|
53
|
+
STDERR.reopen "/dev/null", "a"
|
54
|
+
else
|
55
|
+
Process.daemon
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def self.conn cmd
|
60
|
+
s = TCPSocket.open(@hostname, @port)
|
61
|
+
s.puts cmd
|
62
|
+
|
63
|
+
while line = s.gets
|
64
|
+
puts line.chop
|
65
|
+
end
|
66
|
+
|
67
|
+
true
|
68
|
+
rescue =>e
|
69
|
+
# puts "#{e}"
|
70
|
+
false
|
71
|
+
ensure
|
72
|
+
s.close unless s==nil
|
73
|
+
end
|
74
|
+
|
75
|
+
def self.close_files curr_time = 0
|
76
|
+
if curr_time == 0
|
77
|
+
# puts 'close all of files'
|
78
|
+
@files.each do |log_file_name, log_files|
|
79
|
+
log_files[0].close
|
80
|
+
log_files[1].close
|
81
|
+
end
|
82
|
+
# puts 'empty file map'
|
83
|
+
@files = {}
|
84
|
+
@daemon_log_file.close
|
85
|
+
else
|
86
|
+
@files.each do |k, v|
|
87
|
+
if (curr_time - v[2]) > 86400000
|
88
|
+
puts "close #{k}"
|
89
|
+
v[0].close
|
90
|
+
v[1].close
|
91
|
+
@files[k] = []
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
def self.transfer log_file_name, obj
|
98
|
+
for pattern, handlers in obj.patterns
|
99
|
+
if log_file_name =~ /#{pattern}/
|
100
|
+
index = log_file_name.rindex('/')
|
101
|
+
index += 1
|
102
|
+
log_path = log_file_name[0, index]
|
103
|
+
log_fn = log_file_name[index..log_file_name.length]
|
104
|
+
|
105
|
+
loc_path = "#{log_path}.loc"
|
106
|
+
loc_file_name = "#{loc_path}/#{log_fn}"
|
107
|
+
|
108
|
+
log_file, loc_file, open_time, line_count = @files[log_file_name]
|
109
|
+
|
110
|
+
unless log_file
|
111
|
+
Dir.mkdir loc_path unless File.exist? loc_path
|
112
|
+
|
113
|
+
if File.exist? loc_file_name
|
114
|
+
loc_file = File.new(loc_file_name, 'r+')
|
115
|
+
else
|
116
|
+
loc_file = File.new(loc_file_name, 'w+')
|
117
|
+
end
|
118
|
+
loc_file.sync = true
|
119
|
+
line_count = 0
|
120
|
+
log_file = File.new(log_file_name, 'r')
|
121
|
+
open_time = Time.now.to_i
|
122
|
+
close_files open_time
|
123
|
+
end
|
124
|
+
|
125
|
+
while line = log_file.gets
|
126
|
+
line_count += 1
|
127
|
+
loc = loc_file.gets
|
128
|
+
if loc
|
129
|
+
next
|
130
|
+
end
|
131
|
+
|
132
|
+
fail = false
|
133
|
+
fail_handlers = []
|
134
|
+
|
135
|
+
handlers.each do |handler|
|
136
|
+
begin
|
137
|
+
handler.handle log_path, log_fn, line.chop, line_count, pattern
|
138
|
+
rescue => err
|
139
|
+
log "#{log_file_name}, #{line_count}, #{err}"
|
140
|
+
fail_handlers << handler.class
|
141
|
+
fail = true
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
if fail
|
146
|
+
loc_file.puts "#{line_count}, #{fail_handlers}"
|
147
|
+
else
|
148
|
+
loc_file.puts "#{line_count}"
|
149
|
+
end
|
150
|
+
|
151
|
+
end
|
152
|
+
@files[log_file_name] = [log_file, loc_file, open_time, line_count]
|
153
|
+
break
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
def self.daemon
|
159
|
+
YAML.load_file(@config_file_name).each do |obj|
|
160
|
+
log "absolute path: #{obj.absolute_path}"
|
161
|
+
log "dir disallow: #{obj.dir_disallow}"
|
162
|
+
log "file disallow: #{obj.file_disallow}"
|
163
|
+
log "file allow: #{obj.file_allow}"
|
164
|
+
obj.patterns.each do |pattern, handlers|
|
165
|
+
log "pattern: #{pattern}"
|
166
|
+
handlers.each do |handler|
|
167
|
+
handler.init
|
168
|
+
log "- -#{handler.class} initialized."
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
@threads << Thread.new do
|
173
|
+
begin
|
174
|
+
m = FileMonitor.new(obj.absolute_path)
|
175
|
+
|
176
|
+
m.filter_dirs do
|
177
|
+
obj.dir_disallow.each do |dir|
|
178
|
+
disallow /#{dir}/
|
179
|
+
end
|
180
|
+
disallow /loc$/
|
181
|
+
end
|
182
|
+
|
183
|
+
m.filter_files do
|
184
|
+
obj.file_disallow.each do |file|
|
185
|
+
disallow /#{file}/
|
186
|
+
end
|
187
|
+
obj.file_allow.each do |file|
|
188
|
+
allow /#{file}/
|
189
|
+
end
|
190
|
+
allow /#{Stop_cmd_file_name}$/
|
191
|
+
end
|
192
|
+
|
193
|
+
m.run do |events|
|
194
|
+
break if @exit_flag
|
195
|
+
events.each do |event|
|
196
|
+
flags = event.flags
|
197
|
+
if flags.include?(:modify) or flags.include?(:moved_to) or flags.include?(:create)
|
198
|
+
transfer event.absolute_name, obj
|
199
|
+
end
|
200
|
+
end
|
201
|
+
end
|
202
|
+
|
203
|
+
log "#{obj.absolute_path} file monitor thread exit."
|
204
|
+
rescue =>err
|
205
|
+
# puts err
|
206
|
+
log err
|
207
|
+
end
|
208
|
+
end
|
209
|
+
end
|
210
|
+
|
211
|
+
@threads << Thread.new do
|
212
|
+
server = TCPServer.open(@hostname, @port)
|
213
|
+
|
214
|
+
loop do
|
215
|
+
client = server.accept
|
216
|
+
|
217
|
+
cmd = client.gets
|
218
|
+
|
219
|
+
case cmd.chop
|
220
|
+
when 'stop'
|
221
|
+
client.puts(Prompt_exiting)
|
222
|
+
@exit_flag = true;
|
223
|
+
|
224
|
+
YAML.load_file(@config_file_name).each do |obj|
|
225
|
+
system "touch #{obj.absolute_path}/#{Stop_cmd_file_name}"
|
226
|
+
end
|
227
|
+
|
228
|
+
sleep 1
|
229
|
+
|
230
|
+
YAML.load_file(@config_file_name).each do |obj|
|
231
|
+
system "unlink #{obj.absolute_path}/#{Stop_cmd_file_name}"
|
232
|
+
end
|
233
|
+
|
234
|
+
# puts 'client.close'
|
235
|
+
client.close
|
236
|
+
|
237
|
+
break;
|
238
|
+
when 'status'
|
239
|
+
close_files Time.now.to_i
|
240
|
+
client.puts(Prompt_running)
|
241
|
+
client.puts(@config_file_title)
|
242
|
+
@files.each do |log_file_name, log_files|
|
243
|
+
client.puts "log file: #{log_file_name}, loc file: #{log_files[1].path}, open time: #{Time.at(log_files[2])}, lines: #{log_files[3]}"
|
244
|
+
end
|
245
|
+
end
|
246
|
+
|
247
|
+
# puts 'client.close'
|
248
|
+
client.close
|
249
|
+
end
|
250
|
+
|
251
|
+
close_files
|
252
|
+
# @files.each do |k, v|
|
253
|
+
# puts "#{k}: #{v}"
|
254
|
+
# end
|
255
|
+
unless server == nil
|
256
|
+
# puts 'server.close'
|
257
|
+
server.close
|
258
|
+
end
|
259
|
+
end
|
260
|
+
|
261
|
+
@threads.each { |t| t.join }
|
262
|
+
end
|
263
|
+
|
264
|
+
def self.run argv, port, working_directory
|
265
|
+
@port = port
|
266
|
+
|
267
|
+
if argv.length < 1
|
268
|
+
puts Prompt_cmdline
|
269
|
+
exit
|
270
|
+
end
|
271
|
+
|
272
|
+
cmd = argv[0]
|
273
|
+
# puts "cmd: #{cmd}"
|
274
|
+
|
275
|
+
@exit_flag = false;
|
276
|
+
|
277
|
+
case cmd
|
278
|
+
when 'start'
|
279
|
+
if argv.length < 2
|
280
|
+
@config_file_name = "#{working_directory}/config.yaml"
|
281
|
+
elsif argv[1][0] == '/'
|
282
|
+
@config_file_name = argv[1]
|
283
|
+
else
|
284
|
+
@config_file_name = "#{working_directory}/#{argv[1]}"
|
285
|
+
end
|
286
|
+
|
287
|
+
@config_file_title = "config file: #{@config_file_name}"
|
288
|
+
|
289
|
+
unless File.exist? @config_file_name
|
290
|
+
puts "#{@config_file_title} no exist!"
|
291
|
+
exit
|
292
|
+
end
|
293
|
+
|
294
|
+
exit if conn 'status'
|
295
|
+
|
296
|
+
puts Prompt_starting
|
297
|
+
|
298
|
+
daemonize_app working_directory
|
299
|
+
@daemon_log_file_name = "#{working_directory}/#{@daemon_log_file_name}"
|
300
|
+
@daemon_log_file = File.new @daemon_log_file_name, 'a'
|
301
|
+
@daemon_log_file.sync = true
|
302
|
+
log "-------------#{Time.now}-----------------"
|
303
|
+
daemon
|
304
|
+
when /stop|status/
|
305
|
+
puts Prompt_no_running unless conn cmd
|
306
|
+
else
|
307
|
+
puts Prompt_cmdline
|
308
|
+
end
|
309
|
+
end
|
310
|
+
end
|
metadata
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: logfile_transfer
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Cong Yan
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2013-10-10 00:00:00.000000000 Z
|
12
|
+
dependencies: []
|
13
|
+
description: Ruby monitoring and transform logfiles daemon.
|
14
|
+
email: xiaoyaoyouzizai@gmail.com
|
15
|
+
executables: []
|
16
|
+
extensions: []
|
17
|
+
extra_rdoc_files: []
|
18
|
+
files:
|
19
|
+
- lib/logfile_transfer.rb
|
20
|
+
- example.rb
|
21
|
+
- config.yaml
|
22
|
+
- README.md
|
23
|
+
homepage: https://github.com/xiaoyaoyouzizai/logfile_transfer
|
24
|
+
licenses:
|
25
|
+
- Apache License, Version 2.0
|
26
|
+
metadata: {}
|
27
|
+
post_install_message:
|
28
|
+
rdoc_options: []
|
29
|
+
require_paths:
|
30
|
+
- lib
|
31
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
32
|
+
requirements:
|
33
|
+
- - '>='
|
34
|
+
- !ruby/object:Gem::Version
|
35
|
+
version: '0'
|
36
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
requirements: []
|
42
|
+
rubyforge_project:
|
43
|
+
rubygems_version: 2.1.4
|
44
|
+
signing_key:
|
45
|
+
specification_version: 4
|
46
|
+
summary: logfile transfer
|
47
|
+
test_files: []
|