fileminer 1.1.1 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +15 -6
- data/bin/fileminer-genconf +30 -3
- data/conf/fileminer_default.yml +15 -1
- data/lib/fileminer/miner.rb +77 -19
- data/lib/fileminer/version.rb +1 -1
- data/lib/fileminer.rb +10 -5
- metadata +6 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 93670c0973d05575fdab215da684c633c9367f2288168cecab359b0464c539de
|
4
|
+
data.tar.gz: 55eaa5c573e308585da35ec30423988c049ef25669ea161a069a5c3392d75ee5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3786c83518c206f3a1bec0ed6b13ce46f3c1604655be8a2d48afa66e8bf202d40fb3bbda5c4c3521e5bbd7265ce7773fa114f03520f162876f6a19ce7ca0c474
|
7
|
+
data.tar.gz: 729cba5d996c05301a8ed4870180f89f74511b0709d96c666df43ba7755d17c4a56d5531f19d98e5180029ff8283268671a4ed258582dc8ef0d437d34d22272a
|
data/README.md
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
# fileminer
|
2
|
-
[![Gem Version](https://d25lcipzij17d.cloudfront.net/badge.svg?id=rb&type=6&v=1.
|
2
|
+
[![Gem Version](https://d25lcipzij17d.cloudfront.net/badge.svg?id=rb&type=6&v=1.2.0&x2=0)](https://rubygems.org/gems/fileminer)
|
3
3
|
[![MIT licensed](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/fmjsjx/fileminer/blob/master/LICENSE)
|
4
4
|
|
5
5
|
|
@@ -11,10 +11,10 @@ A simple line based file/log transfer tool coding by ruby.
|
|
11
11
|
Install fileminer from RubyGems:
|
12
12
|
```
|
13
13
|
$ gem install fileminer
|
14
|
-
Fetching fileminer-1.
|
15
|
-
Successfully installed fileminer-1.
|
16
|
-
Parsing documentation for fileminer-1.
|
17
|
-
Installing ri documentation for fileminer-1.
|
14
|
+
Fetching fileminer-1.2.0.gem
|
15
|
+
Successfully installed fileminer-1.2.0
|
16
|
+
Parsing documentation for fileminer-1.2.0
|
17
|
+
Installing ri documentation for fileminer-1.2.0
|
18
18
|
Done installing documentation for fileminer after 0 seconds
|
19
19
|
1 gem installed
|
20
20
|
$ _
|
@@ -32,7 +32,13 @@ At first, we should generate the fileminer configuration file.
|
|
32
32
|
We provide a command tool 'fileminer-genconf' to generate configurations:
|
33
33
|
```
|
34
34
|
$ fileminer-genconf -h
|
35
|
-
Usage:
|
35
|
+
Usage:
|
36
|
+
fileminer-genconf [options]
|
37
|
+
|
38
|
+
Samples:
|
39
|
+
fileminer-genconf -t fileminer -o Generate config on /etc/fileminer/fileminer.yml
|
40
|
+
fileminer-genconf -t supervisor -o -l Generate ./fileminer.ini with logfile on
|
41
|
+
/var/log/fileminer/stderr.log
|
36
42
|
|
37
43
|
Options:
|
38
44
|
-t, --type fileminer|supervisor Type of the config file to be generated
|
@@ -40,7 +46,10 @@ Options:
|
|
40
46
|
-o, --out [path] Output content to a file
|
41
47
|
For type fileminer, default is /etc/fileminer/fileminer.yml
|
42
48
|
For type supervisor, default is ./fileminer.ini
|
49
|
+
-l, --logfile [path] Logfile configured on supervisor config file
|
50
|
+
Default is /var/log/fileminer/stderr.log
|
43
51
|
-h, --help Print help
|
52
|
+
|
44
53
|
$ _
|
45
54
|
```
|
46
55
|
|
data/bin/fileminer-genconf
CHANGED
@@ -29,6 +29,20 @@ def gen_fileminer_conf(options)
|
|
29
29
|
end
|
30
30
|
|
31
31
|
def gen_supervisor_conf(options)
|
32
|
+
logfile = options[:logfile]
|
33
|
+
if logfile.nil? && options.key?(:logfile)
|
34
|
+
logfile = '/var/log/fileminer/stderr.log'
|
35
|
+
end
|
36
|
+
stdouts = ''
|
37
|
+
unless logfile.nil?
|
38
|
+
stdouts = <<-EOS
|
39
|
+
stderr_logfile=#{logfile}
|
40
|
+
stderr_logfile_maxbytes=10MB
|
41
|
+
stderr_logfile_backups=10
|
42
|
+
EOS
|
43
|
+
log_dir = File.dirname logfile
|
44
|
+
Dir.mkdirs log_dir unless Dir.exists? log_dir
|
45
|
+
end
|
32
46
|
wrapper_home = File.join ENV['GEM_HOME'], 'wrappers'
|
33
47
|
content = <<-EOS
|
34
48
|
[program:fileminer]
|
@@ -37,6 +51,7 @@ autostart=true
|
|
37
51
|
autorestart=true
|
38
52
|
stopsinal=INT
|
39
53
|
user=root
|
54
|
+
#{stdouts}
|
40
55
|
EOS
|
41
56
|
if options.key? :out
|
42
57
|
out = options[:out]
|
@@ -44,7 +59,6 @@ user=root
|
|
44
59
|
unless dst.end_with?('.ini') || dst.end_with?('.conf')
|
45
60
|
dst = File.join dst, 'fileminer.ini'
|
46
61
|
end
|
47
|
-
|
48
62
|
dst_dir = File.dirname dst
|
49
63
|
Dir.mkdirs dst_dir unless Dir.exists? dst_dir
|
50
64
|
File.open(dst, 'w') { |io| io.write content }
|
@@ -57,7 +71,13 @@ end
|
|
57
71
|
options = Hash.new
|
58
72
|
OptionParser.new do |opts|
|
59
73
|
# banner
|
60
|
-
opts.banner = "Usage:
|
74
|
+
opts.banner = "Usage:"
|
75
|
+
opts.separator ' fileminer-genconf [options]'
|
76
|
+
opts.separator ''
|
77
|
+
opts.separator 'Samples:'
|
78
|
+
opts.separator ' fileminer-genconf -t fileminer -o Generate config on /etc/fileminer/fileminer.yml'
|
79
|
+
opts.separator ' fileminer-genconf -t supervisor -o -l Generate ./fileminer.ini with logfile on'
|
80
|
+
opts.separator ' /var/log/fileminer/stderr.log'
|
61
81
|
opts.separator ''
|
62
82
|
opts.separator 'Options:'
|
63
83
|
|
@@ -83,6 +103,13 @@ OptionParser.new do |opts|
|
|
83
103
|
options[:out] = value
|
84
104
|
end
|
85
105
|
|
106
|
+
# logfile, just for type=supervisor
|
107
|
+
opts.on('-l', '--logfile [path]',
|
108
|
+
'Logfile configured on supervisor config file',
|
109
|
+
'Default is /var/log/fileminer/stderr.log') do |value|
|
110
|
+
options[:logfile] = value
|
111
|
+
end
|
112
|
+
|
86
113
|
end.parse!
|
87
114
|
|
88
115
|
case options[:type]
|
@@ -90,4 +117,4 @@ when :fileminer
|
|
90
117
|
gen_fileminer_conf options
|
91
118
|
when :supervisor
|
92
119
|
gen_supervisor_conf options
|
93
|
-
end
|
120
|
+
end
|
data/conf/fileminer_default.yml
CHANGED
@@ -13,13 +13,27 @@
|
|
13
13
|
# default value is -1
|
14
14
|
#max_lines_of_each_file: -1
|
15
15
|
|
16
|
+
# default value is 5s
|
17
|
+
#sleep_time_when_no_more_data: 5s
|
18
|
+
|
16
19
|
# fileminer inputs
|
17
20
|
fileminer.inputs:
|
18
21
|
|
22
|
+
# setup the path of work dir
|
23
|
+
# default value is /var/lib/fileminer
|
24
|
+
#work_dir: /var/lib/fileminer
|
25
|
+
|
19
26
|
# setup the path of the registry file
|
20
|
-
# default value is /
|
27
|
+
# default value is ${work_dir}/registry
|
21
28
|
#registry_path: /var/lib/fileminer/registry
|
22
29
|
|
30
|
+
# setup the path of the history file
|
31
|
+
# default value is ${work_dir}/history
|
32
|
+
#history_path: /var/lib/fileminer/history
|
33
|
+
|
34
|
+
# default value is 20
|
35
|
+
#max_eof_files: 20
|
36
|
+
|
23
37
|
# file paths
|
24
38
|
paths:
|
25
39
|
- /path/to/*.log
|
data/lib/fileminer/miner.rb
CHANGED
@@ -6,7 +6,8 @@ require_relative 'tools/io'
|
|
6
6
|
class Miner
|
7
7
|
|
8
8
|
DEFAULTS = {
|
9
|
-
|
9
|
+
work_dir: '/var/lib/fileminer',
|
10
|
+
max_eof_files: 20,
|
10
11
|
eof_seconds: 86400,
|
11
12
|
batch_lines: 200,
|
12
13
|
}
|
@@ -25,6 +26,14 @@ class Miner
|
|
25
26
|
# fix options by DEFAULTS
|
26
27
|
DEFAULTS.each { |k, v| options[k] = v unless options.key? k }
|
27
28
|
@registry_path = options[:registry_path]
|
29
|
+
if @registry_path.nil?
|
30
|
+
@registry_path = "#{options[:work_dir]}/registry"
|
31
|
+
end
|
32
|
+
@history_path = options[:history_path]
|
33
|
+
if @history_path.nil?
|
34
|
+
@history_path = "#{options[:work_dir]}/history"
|
35
|
+
end
|
36
|
+
@max_eof_files = options[:max_eof_files]
|
28
37
|
@paths = options[:paths]
|
29
38
|
@eof_seconds = options[:eof_seconds]
|
30
39
|
@batch_lines = options[:batch_lines]
|
@@ -35,8 +44,15 @@ class Miner
|
|
35
44
|
end
|
36
45
|
@files = []
|
37
46
|
@active_files = []
|
47
|
+
@history = []
|
48
|
+
if File.exist? @history_path
|
49
|
+
@history = File.open(@history_path) { |io| JSON.parse(io.read) }
|
50
|
+
else
|
51
|
+
parent_dir = File.dirname @history_path
|
52
|
+
Dir.mkdirs parent_dir unless Dir.exist? parent_dir
|
53
|
+
end
|
38
54
|
if File.exist? @registry_path
|
39
|
-
File.open(@registry_path) { |io|
|
55
|
+
@files = File.open(@registry_path) { |io| JSON.parse(io.read, {symbolize_names: true}) }
|
40
56
|
@active_files = @files.select { |record| !record[:eof] }
|
41
57
|
else
|
42
58
|
parent_dir = File.dirname @registry_path
|
@@ -44,37 +60,40 @@ class Miner
|
|
44
60
|
end
|
45
61
|
end
|
46
62
|
|
63
|
+
# Save work status
|
64
|
+
def save_work_status
|
65
|
+
save_history
|
66
|
+
save_registry
|
67
|
+
end
|
68
|
+
|
47
69
|
# Save registry
|
48
70
|
def save_registry
|
49
71
|
File.open(@registry_path, 'w') { |io| io.write @files.to_json }
|
50
72
|
end
|
51
73
|
|
74
|
+
# Save history
|
75
|
+
def save_history
|
76
|
+
File.open(@history_path, 'w') { |io| io.write @history.to_json }
|
77
|
+
end
|
78
|
+
|
52
79
|
# Refresh
|
53
80
|
def refresh_files
|
54
81
|
now = Time.now
|
55
82
|
file_paths = Set.new
|
56
83
|
file_paths.merge Dir[*@paths].select { |path| File.file? path }
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
unless record[:eof]
|
61
|
-
if file_exists
|
62
|
-
# check if EOF
|
63
|
-
if record[:pos] == File.size(path) && now - File.mtime(path) > @eof_seconds
|
64
|
-
record[:eof] = true
|
65
|
-
end
|
66
|
-
else
|
67
|
-
# missing file, set :eof to true
|
68
|
-
record[:eof] = true
|
69
|
-
end
|
70
|
-
end
|
71
|
-
!record[:eof]
|
84
|
+
active_files, eof_size = select_active_files now, file_paths
|
85
|
+
if eof_size > @max_eof_files
|
86
|
+
move_eof_to_history
|
72
87
|
end
|
73
|
-
|
88
|
+
history_files = Set.new @history
|
89
|
+
file_paths.select do |path|
|
90
|
+
! history_files.member? path
|
91
|
+
end.each do |path|
|
74
92
|
record = {path: path, pos: 0, eof: false}
|
75
93
|
@files << record
|
76
|
-
|
94
|
+
active_files << record
|
77
95
|
end
|
96
|
+
@active_files = active_files
|
78
97
|
@files_refresh_time = now
|
79
98
|
end
|
80
99
|
|
@@ -108,4 +127,43 @@ class Miner
|
|
108
127
|
Time.now - @files_refresh_time >= refresh_files_time_trigger
|
109
128
|
end
|
110
129
|
|
130
|
+
private
|
131
|
+
def select_active_files(now, file_paths)
|
132
|
+
eof_seconds = @eof_seconds
|
133
|
+
eof_size = 0
|
134
|
+
active_files = @files.select do |record|
|
135
|
+
path = record[:path]
|
136
|
+
file_exists = file_paths.delete? path
|
137
|
+
if record[:eof]
|
138
|
+
eof_size += 1
|
139
|
+
else
|
140
|
+
if file_exists
|
141
|
+
# check if EOF
|
142
|
+
if record[:pos] == File.size(path) && now - File.mtime(path) > eof_seconds
|
143
|
+
record[:eof] = true
|
144
|
+
eof_size += 1
|
145
|
+
end
|
146
|
+
else
|
147
|
+
# missing file, set :eof to true
|
148
|
+
record[:eof] = true
|
149
|
+
eof_size += 1
|
150
|
+
end
|
151
|
+
end
|
152
|
+
!record[:eof]
|
153
|
+
end
|
154
|
+
[active_files, eof_size]
|
155
|
+
end
|
156
|
+
|
157
|
+
private
|
158
|
+
def move_eof_to_history
|
159
|
+
to_removed_paths = @files.select do |record|
|
160
|
+
record[:eof]
|
161
|
+
end.map do |record|
|
162
|
+
record[:path]
|
163
|
+
end
|
164
|
+
path_set = Set.new to_removed_paths
|
165
|
+
@files.delete_if { |record| path_set.member? record[:path] }
|
166
|
+
@history.concat to_removed_paths
|
167
|
+
end
|
168
|
+
|
111
169
|
end
|
data/lib/fileminer/version.rb
CHANGED
data/lib/fileminer.rb
CHANGED
@@ -14,6 +14,7 @@ class FileMiner
|
|
14
14
|
max_time_of_each_mining: '5s',
|
15
15
|
max_lines_of_each_mining: -1,
|
16
16
|
max_lines_of_each_file: -1,
|
17
|
+
sleep_time_when_no_more_data: '5s',
|
17
18
|
}
|
18
19
|
|
19
20
|
attr_reader :miner, :output, :running
|
@@ -59,6 +60,8 @@ class FileMiner
|
|
59
60
|
end
|
60
61
|
# refresh_files_time_trigger
|
61
62
|
@refresh_files_time_trigger = parse_time conf[:refresh_files_time_trigger], 'refresh_files_time_trigger on fileminer.settings'
|
63
|
+
# sleep seconds when no more data
|
64
|
+
@sleep_seconds_when_no_more_data = parse_time conf[:sleep_time_when_no_more_data], 'sleep_time_when_no_more_data on fileminer.settings'
|
62
65
|
end
|
63
66
|
|
64
67
|
def parse_time(value, conf_name)
|
@@ -168,12 +171,13 @@ class FileMiner
|
|
168
171
|
def mine_once
|
169
172
|
start_time = Time.now
|
170
173
|
full_lines = 0
|
171
|
-
|
174
|
+
miner = @miner
|
175
|
+
miner.active_files.all? do |record|
|
172
176
|
mining_next = true
|
173
177
|
if record[:pos] < File.size(record[:path])
|
174
178
|
file_lines = 0
|
175
179
|
loop do
|
176
|
-
lines =
|
180
|
+
lines = miner.read_lines record
|
177
181
|
break if lines.empty?
|
178
182
|
send_lines record, lines
|
179
183
|
file_lines += lines.size
|
@@ -193,6 +197,7 @@ class FileMiner
|
|
193
197
|
def start_mining
|
194
198
|
unless @running
|
195
199
|
@running = true
|
200
|
+
sleep_seconds = @sleep_seconds_when_no_more_data
|
196
201
|
while @running
|
197
202
|
begin
|
198
203
|
files_refreshed = check_files
|
@@ -200,13 +205,13 @@ class FileMiner
|
|
200
205
|
# sleep 5 seconds if no more data
|
201
206
|
# TODO using settings instead in future
|
202
207
|
if sent_lines == 0
|
203
|
-
@miner.
|
204
|
-
sleep
|
208
|
+
@miner.save_work_status if files_refreshed
|
209
|
+
sleep sleep_seconds
|
205
210
|
end
|
206
211
|
rescue => e
|
207
212
|
@logger.error e
|
208
213
|
# sleep for a little while to wait output recover
|
209
|
-
sleep
|
214
|
+
sleep sleep_seconds if @running
|
210
215
|
end
|
211
216
|
end
|
212
217
|
@miner.save_registry
|
metadata
CHANGED
@@ -1,16 +1,18 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fileminer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Fang MinJie
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-06-
|
11
|
+
date: 2019-06-26 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
|
-
description:
|
13
|
+
description: |2
|
14
|
+
FileMiner is a simple file/log transfer tool implemented in Ruby.
|
15
|
+
Up to now, FileMiner has only tested on CRuby.
|
14
16
|
email:
|
15
17
|
- fmjsjx@163.com
|
16
18
|
executables:
|
@@ -57,5 +59,5 @@ requirements: []
|
|
57
59
|
rubygems_version: 3.0.4
|
58
60
|
signing_key:
|
59
61
|
specification_version: 4
|
60
|
-
summary: A simple file/log transfer tool
|
62
|
+
summary: A simple file/log transfer tool.
|
61
63
|
test_files: []
|