hq-log-monitor-client 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bin/hq-log-monitor-client +8 -0
- data/features/context.feature +146 -0
- data/features/detect-rotation.feature +86 -0
- data/features/match.feature +75 -0
- data/features/skip-files.feature +95 -0
- data/features/support/env.rb +70 -0
- data/features/support/steps.rb +68 -0
- data/lib/hq/log-monitor-client/script.rb +394 -0
- metadata +188 -0
@@ -0,0 +1,146 @@
|
|
1
|
+
Feature: Log monitor client provides context lines correctly
|
2
|
+
Background:
|
3
|
+
|
4
|
+
Given a file "default.config":
|
5
|
+
"""
|
6
|
+
<log-monitor-client-config>
|
7
|
+
<cache path="cache"/>
|
8
|
+
<client class="class" host="host"/>
|
9
|
+
<server url="${server-url}"/>
|
10
|
+
<service name="service">
|
11
|
+
<fileset>
|
12
|
+
<scan glob="*.log"/>
|
13
|
+
<match type="critical" regex="CRITICAL" before="2" after="2"/>
|
14
|
+
<match type="warning" regex="WARNING" before="2" after="2"/>
|
15
|
+
</fileset>
|
16
|
+
</service>
|
17
|
+
</log-monitor-client-config>
|
18
|
+
"""
|
19
|
+
|
20
|
+
Scenario: Middle of large file
|
21
|
+
|
22
|
+
Given a file "logfile.log":
|
23
|
+
"""
|
24
|
+
NOTICE line 0
|
25
|
+
NOTICE line 1
|
26
|
+
NOTICE line 2
|
27
|
+
WARNING line 3
|
28
|
+
NOTICE line 4
|
29
|
+
NOTICE line 5
|
30
|
+
NOTICE line 6
|
31
|
+
"""
|
32
|
+
|
33
|
+
When I run log-monitor-client with config "default.config"
|
34
|
+
|
35
|
+
Then the following events should be submitted:
|
36
|
+
"""
|
37
|
+
{
|
38
|
+
type: warning,
|
39
|
+
source: { class: class, host: host, service: service },
|
40
|
+
location: { file: logfile.log, line: 3 },
|
41
|
+
lines: {
|
42
|
+
before: [
|
43
|
+
NOTICE line 1,
|
44
|
+
NOTICE line 2,
|
45
|
+
],
|
46
|
+
matching: WARNING line 3,
|
47
|
+
after: [
|
48
|
+
NOTICE line 4,
|
49
|
+
NOTICE line 5,
|
50
|
+
],
|
51
|
+
}
|
52
|
+
}
|
53
|
+
"""
|
54
|
+
|
55
|
+
Scenario: Start of large file
|
56
|
+
|
57
|
+
Given a file "logfile.log":
|
58
|
+
"""
|
59
|
+
NOTICE line 0
|
60
|
+
WARNING line 1
|
61
|
+
NOTICE line 2
|
62
|
+
NOTICE line 3
|
63
|
+
NOTICE line 4
|
64
|
+
"""
|
65
|
+
|
66
|
+
When I run log-monitor-client with config "default.config"
|
67
|
+
|
68
|
+
Then the following events should be submitted:
|
69
|
+
"""
|
70
|
+
{
|
71
|
+
type: warning,
|
72
|
+
source: { class: class, host: host, service: service },
|
73
|
+
location: { file: logfile.log, line: 1 },
|
74
|
+
lines: {
|
75
|
+
before: [
|
76
|
+
NOTICE line 0,
|
77
|
+
],
|
78
|
+
matching: WARNING line 1,
|
79
|
+
after: [
|
80
|
+
NOTICE line 2,
|
81
|
+
NOTICE line 3,
|
82
|
+
],
|
83
|
+
}
|
84
|
+
}
|
85
|
+
"""
|
86
|
+
|
87
|
+
Scenario: End of large file
|
88
|
+
|
89
|
+
Given a file "logfile.log":
|
90
|
+
"""
|
91
|
+
NOTICE line 0
|
92
|
+
NOTICE line 1
|
93
|
+
NOTICE line 2
|
94
|
+
WARNING line 3
|
95
|
+
NOTICE line 4
|
96
|
+
"""
|
97
|
+
|
98
|
+
When I run log-monitor-client with config "default.config"
|
99
|
+
|
100
|
+
Then the following events should be submitted:
|
101
|
+
"""
|
102
|
+
{
|
103
|
+
type: warning,
|
104
|
+
source: { class: class, host: host, service: service },
|
105
|
+
location: { file: logfile.log, line: 3 },
|
106
|
+
lines: {
|
107
|
+
before: [
|
108
|
+
NOTICE line 1,
|
109
|
+
NOTICE line 2,
|
110
|
+
],
|
111
|
+
matching: WARNING line 3,
|
112
|
+
after: [
|
113
|
+
NOTICE line 4,
|
114
|
+
],
|
115
|
+
}
|
116
|
+
}
|
117
|
+
"""
|
118
|
+
|
119
|
+
Scenario: Middle of short file
|
120
|
+
|
121
|
+
Given a file "logfile.log":
|
122
|
+
"""
|
123
|
+
NOTICE line 0
|
124
|
+
WARNING line 1
|
125
|
+
NOTICE line 2
|
126
|
+
"""
|
127
|
+
|
128
|
+
When I run log-monitor-client with config "default.config"
|
129
|
+
|
130
|
+
Then the following events should be submitted:
|
131
|
+
"""
|
132
|
+
{
|
133
|
+
type: warning,
|
134
|
+
source: { class: class, host: host, service: service },
|
135
|
+
location: { file: logfile.log, line: 1 },
|
136
|
+
lines: {
|
137
|
+
before: [
|
138
|
+
NOTICE line 0,
|
139
|
+
],
|
140
|
+
matching: WARNING line 1,
|
141
|
+
after: [
|
142
|
+
NOTICE line 2,
|
143
|
+
],
|
144
|
+
}
|
145
|
+
}
|
146
|
+
"""
|
@@ -0,0 +1,86 @@
|
|
1
|
+
Feature: Log monitor client does detects rotated log files
|
2
|
+
|
3
|
+
Background:
|
4
|
+
|
5
|
+
Given a file "default.config":
|
6
|
+
"""
|
7
|
+
<log-monitor-client-config>
|
8
|
+
<cache path="cache"/>
|
9
|
+
<client class="class" host="host"/>
|
10
|
+
<server url="${server-url}"/>
|
11
|
+
<service name="service">
|
12
|
+
<fileset>
|
13
|
+
<scan glob="*.log"/>
|
14
|
+
<match type="critical" regex="CRITICAL"/>
|
15
|
+
<match type="warning" regex="WARNING"/>
|
16
|
+
</fileset>
|
17
|
+
</service>
|
18
|
+
</log-monitor-client-config>
|
19
|
+
"""
|
20
|
+
|
21
|
+
Scenario: Log file rotated
|
22
|
+
|
23
|
+
Given a file "logfile.log":
|
24
|
+
"""
|
25
|
+
WARNING This is an old warning 0
|
26
|
+
"""
|
27
|
+
And I have run log-monitor-client with config "default.config"
|
28
|
+
And I have updated file "logfile.log" changing the timestamp:
|
29
|
+
"""
|
30
|
+
WARNING This is a new warning 0
|
31
|
+
WARNING This is a new warning 1
|
32
|
+
"""
|
33
|
+
|
34
|
+
When I run log-monitor-client with config "default.config"
|
35
|
+
|
36
|
+
Then the following events should be submitted:
|
37
|
+
"""
|
38
|
+
{
|
39
|
+
type: warning,
|
40
|
+
source: { class: class, host: host, service: service },
|
41
|
+
location: { file: logfile.log, line: 0 },
|
42
|
+
lines: {
|
43
|
+
before: [],
|
44
|
+
matching: WARNING This is a new warning 0,
|
45
|
+
after: [],
|
46
|
+
},
|
47
|
+
},
|
48
|
+
{
|
49
|
+
type: warning,
|
50
|
+
source: { class: class, host: host, service: service },
|
51
|
+
location: { file: logfile.log, line: 1 },
|
52
|
+
lines: {
|
53
|
+
before: [],
|
54
|
+
matching: WARNING This is a new warning 1,
|
55
|
+
after: [],
|
56
|
+
},
|
57
|
+
}
|
58
|
+
"""
|
59
|
+
|
60
|
+
Scenario: Log file not rotated
|
61
|
+
|
62
|
+
Given a file "logfile.log":
|
63
|
+
"""
|
64
|
+
WARNING This is an old warning 0
|
65
|
+
"""
|
66
|
+
And I have run log-monitor-client with config "default.config"
|
67
|
+
And I have updated file "logfile.log" changing the timestamp:
|
68
|
+
"""
|
69
|
+
WARNING This is an old warning 0
|
70
|
+
WARNING This is a new warning 1
|
71
|
+
"""
|
72
|
+
When I run log-monitor-client with config "default.config"
|
73
|
+
|
74
|
+
Then the following events should be submitted:
|
75
|
+
"""
|
76
|
+
{
|
77
|
+
type: warning,
|
78
|
+
source: { class: class, host: host, service: service },
|
79
|
+
location: { file: logfile.log, line: 1 },
|
80
|
+
lines: {
|
81
|
+
before: [],
|
82
|
+
matching: WARNING This is a new warning 1,
|
83
|
+
after: [],
|
84
|
+
},
|
85
|
+
}
|
86
|
+
"""
|
@@ -0,0 +1,75 @@
|
|
1
|
+
Feature: Log monitor client correctly reports matching lines
|
2
|
+
|
3
|
+
Background:
|
4
|
+
Given a file "default.config":
|
5
|
+
"""
|
6
|
+
<log-monitor-client-config>
|
7
|
+
<cache path="cache"/>
|
8
|
+
<client class="class" host="host"/>
|
9
|
+
<server url="${server-url}"/>
|
10
|
+
<service name="service">
|
11
|
+
<fileset>
|
12
|
+
<scan glob="*.log"/>
|
13
|
+
<match type="critical" regex="CRITICAL"/>
|
14
|
+
<match type="warning" regex="WARNING"/>
|
15
|
+
</fileset>
|
16
|
+
</service>
|
17
|
+
</log-monitor-client-config>
|
18
|
+
"""
|
19
|
+
|
20
|
+
Scenario: Ignore lines which don't match any pattern
|
21
|
+
|
22
|
+
Given a file "logfile.log":
|
23
|
+
"""
|
24
|
+
NOTICE Not an error
|
25
|
+
"""
|
26
|
+
|
27
|
+
When I run log-monitor-client with config "default.config"
|
28
|
+
|
29
|
+
Then no events should be submitted
|
30
|
+
|
31
|
+
Scenario: Produce events for lines which match a pattern
|
32
|
+
|
33
|
+
Given a file "logfile.log":
|
34
|
+
"""
|
35
|
+
WARNING This is a warning
|
36
|
+
"""
|
37
|
+
|
38
|
+
When I run log-monitor-client with config "default.config"
|
39
|
+
|
40
|
+
Then the following events should be submitted:
|
41
|
+
"""
|
42
|
+
{
|
43
|
+
type: warning,
|
44
|
+
source: { class: class, host: host, service: service },
|
45
|
+
location: { file: logfile.log, line: 0 },
|
46
|
+
lines: {
|
47
|
+
before: [],
|
48
|
+
matching: WARNING This is a warning,
|
49
|
+
after: [],
|
50
|
+
},
|
51
|
+
}
|
52
|
+
"""
|
53
|
+
|
54
|
+
Scenario: Only produce an event for the first matched pattern
|
55
|
+
|
56
|
+
Given a file "logfile.log":
|
57
|
+
"""
|
58
|
+
CRITICAL WARNING This is a confused log entry
|
59
|
+
"""
|
60
|
+
|
61
|
+
When I run log-monitor-client with config "default.config"
|
62
|
+
|
63
|
+
Then the following events should be submitted:
|
64
|
+
"""
|
65
|
+
{
|
66
|
+
type: critical,
|
67
|
+
source: { class: class, host: host, service: service },
|
68
|
+
location: { file: logfile.log, line: 0 },
|
69
|
+
lines: {
|
70
|
+
before: [],
|
71
|
+
matching: CRITICAL WARNING This is a confused log entry,
|
72
|
+
after: [],
|
73
|
+
},
|
74
|
+
}
|
75
|
+
"""
|
@@ -0,0 +1,95 @@
|
|
1
|
+
Feature: Log monitor client does skips files which don't appear to have changed
|
2
|
+
|
3
|
+
Background:
|
4
|
+
|
5
|
+
Given a file "default.config":
|
6
|
+
"""
|
7
|
+
<log-monitor-client-config>
|
8
|
+
<cache path="cache"/>
|
9
|
+
<client class="class" host="host"/>
|
10
|
+
<server url="${server-url}"/>
|
11
|
+
<service name="service">
|
12
|
+
<fileset>
|
13
|
+
<scan glob="*.log"/>
|
14
|
+
<match type="critical" regex="CRITICAL"/>
|
15
|
+
<match type="warning" regex="WARNING"/>
|
16
|
+
</fileset>
|
17
|
+
</service>
|
18
|
+
</log-monitor-client-config>
|
19
|
+
"""
|
20
|
+
|
21
|
+
Scenario: Ignore lines which don't match any pattern
|
22
|
+
When I run log-monitor-client with config "default.config"
|
23
|
+
Then no events should be submitted
|
24
|
+
|
25
|
+
Scenario: Timestamp changed, size unchanged
|
26
|
+
|
27
|
+
Given a file "logfile.log":
|
28
|
+
"""
|
29
|
+
NOTICE This is a notice [padding]
|
30
|
+
"""
|
31
|
+
And I have run log-monitor-client with config "default.config"
|
32
|
+
And I have updated file "logfile.log" changing the timestamp:
|
33
|
+
"""
|
34
|
+
CRITICAL This is a critical error
|
35
|
+
"""
|
36
|
+
|
37
|
+
When I run log-monitor-client with config "default.config"
|
38
|
+
|
39
|
+
Then the following events should be submitted:
|
40
|
+
"""
|
41
|
+
{
|
42
|
+
type: critical,
|
43
|
+
source: { class: class, host: host, service: service },
|
44
|
+
location: { file: logfile.log, line: 0 },
|
45
|
+
lines: {
|
46
|
+
before: [],
|
47
|
+
matching: CRITICAL This is a critical error,
|
48
|
+
after: [],
|
49
|
+
},
|
50
|
+
}
|
51
|
+
"""
|
52
|
+
|
53
|
+
Scenario: Size changed, timestamp unchanged
|
54
|
+
|
55
|
+
Given a file "logfile.log":
|
56
|
+
"""
|
57
|
+
NOTICE This is a notice
|
58
|
+
"""
|
59
|
+
And I have run log-monitor-client with config "default.config"
|
60
|
+
And I have updated file "logfile.log" without changing the timestamp:
|
61
|
+
"""
|
62
|
+
CRITICAL This is a critical error
|
63
|
+
"""
|
64
|
+
|
65
|
+
When I run log-monitor-client with config "default.config"
|
66
|
+
|
67
|
+
Then the following events should be submitted:
|
68
|
+
"""
|
69
|
+
{
|
70
|
+
type: critical,
|
71
|
+
source: { class: class, host: host, service: service },
|
72
|
+
location: { file: logfile.log, line: 0 },
|
73
|
+
lines: {
|
74
|
+
before: [],
|
75
|
+
matching: CRITICAL This is a critical error,
|
76
|
+
after: [],
|
77
|
+
},
|
78
|
+
}
|
79
|
+
"""
|
80
|
+
|
81
|
+
Scenario: Size and timestamp unchanged
|
82
|
+
|
83
|
+
Given a file "logfile.log":
|
84
|
+
"""
|
85
|
+
NOTICE This is a notice [padding]
|
86
|
+
"""
|
87
|
+
And I have run log-monitor-client with config "default.config"
|
88
|
+
And I have updated file "logfile.log" without changing the timestamp:
|
89
|
+
"""
|
90
|
+
CRITICAL This is a critical error
|
91
|
+
"""
|
92
|
+
|
93
|
+
When I run log-monitor-client with config "default.config"
|
94
|
+
|
95
|
+
Then no events should be submitted
|
@@ -0,0 +1,70 @@
|
|
1
|
+
require "tempfile"
|
2
|
+
require "tmpdir"
|
3
|
+
require "webrick"
|
4
|
+
|
5
|
+
require "hq/log-monitor-client/script"
|
6
|
+
|
7
|
+
# web server to recieve events
|
8
|
+
|
9
|
+
$web_config = {
|
10
|
+
:Port => 10000 + rand(55535),
|
11
|
+
:AccessLog => [],
|
12
|
+
:Logger => WEBrick::Log::new("/dev/null", 7),
|
13
|
+
:DoNotReverseLookup => true,
|
14
|
+
}
|
15
|
+
|
16
|
+
$web_server =
|
17
|
+
WEBrick::HTTPServer.new \
|
18
|
+
$web_config
|
19
|
+
|
20
|
+
$web_server_url =
|
21
|
+
"http://localhost:%s/submit-log-event" % [
|
22
|
+
$web_config[:Port],
|
23
|
+
]
|
24
|
+
|
25
|
+
Thread.new do
|
26
|
+
$web_server.start
|
27
|
+
end
|
28
|
+
|
29
|
+
at_exit do
|
30
|
+
$web_server.shutdown
|
31
|
+
end
|
32
|
+
|
33
|
+
$web_server.mount_proc "/submit-log-event" do
|
34
|
+
|request, response|
|
35
|
+
|
36
|
+
event = MultiJson.load request.body
|
37
|
+
|
38
|
+
$events_received << event
|
39
|
+
|
40
|
+
end
|
41
|
+
|
42
|
+
# set up and tear down
|
43
|
+
|
44
|
+
Before do
|
45
|
+
|
46
|
+
$events_received = []
|
47
|
+
|
48
|
+
@old_dir = Dir.pwd
|
49
|
+
@temp_dir = Dir.mktmpdir
|
50
|
+
Dir.chdir @temp_dir
|
51
|
+
|
52
|
+
end
|
53
|
+
|
54
|
+
After do
|
55
|
+
|
56
|
+
FileUtils.remove_entry_secure @temp_dir
|
57
|
+
Dir.chdir @old_dir
|
58
|
+
|
59
|
+
end
|
60
|
+
|
61
|
+
def write_file file_name, file_contents
|
62
|
+
|
63
|
+
file_contents.gsub! "${server-url}", $web_server_url
|
64
|
+
|
65
|
+
File.open file_name, "w" do
|
66
|
+
|file_io|
|
67
|
+
file_io.print file_contents
|
68
|
+
end
|
69
|
+
|
70
|
+
end
|
@@ -0,0 +1,68 @@
|
|
1
|
+
Given /^(?:I have updated|a) file "(.*?)":$/ do
|
2
|
+
|file_name, file_contents|
|
3
|
+
|
4
|
+
write_file file_name, file_contents
|
5
|
+
|
6
|
+
end
|
7
|
+
|
8
|
+
Given /^I have updated file "(.*?)" without changing the timestamp:$/ do
|
9
|
+
|file_name, file_contents|
|
10
|
+
|
11
|
+
file_mtime = File.mtime file_name
|
12
|
+
|
13
|
+
write_file file_name, file_contents
|
14
|
+
|
15
|
+
file_atime = File.atime file_name
|
16
|
+
File.utime file_atime, file_mtime, file_name
|
17
|
+
|
18
|
+
end
|
19
|
+
|
20
|
+
Given /^I have updated file "(.*?)" changing the timestamp:$/ do
|
21
|
+
|file_name, file_contents|
|
22
|
+
|
23
|
+
file_mtime = File.mtime file_name
|
24
|
+
|
25
|
+
write_file file_name, file_contents
|
26
|
+
|
27
|
+
file_atime = File.atime file_name
|
28
|
+
File.utime file_atime, file_mtime + 1, file_name
|
29
|
+
|
30
|
+
end
|
31
|
+
|
32
|
+
When /^I have run log\-monitor\-client with config "(.*?)"$/ do
|
33
|
+
|config_name|
|
34
|
+
|
35
|
+
script = HQ::LogMonitorClient::Script.new
|
36
|
+
|
37
|
+
script.stdout = File.open "/dev/null", "w"
|
38
|
+
script.stderr = File.open "/dev/null", "w"
|
39
|
+
|
40
|
+
script.args = [ "--config", config_name ]
|
41
|
+
script.main
|
42
|
+
|
43
|
+
end
|
44
|
+
|
45
|
+
When /^I run log\-monitor\-client with config "(.*?)"$/ do
|
46
|
+
|config_name|
|
47
|
+
|
48
|
+
$events_received = []
|
49
|
+
|
50
|
+
@script = HQ::LogMonitorClient::Script.new
|
51
|
+
|
52
|
+
@script.stdout = StringIO.new
|
53
|
+
@script.stderr = StringIO.new
|
54
|
+
|
55
|
+
@script.args = [ "--config", config_name ]
|
56
|
+
@script.main
|
57
|
+
|
58
|
+
end
|
59
|
+
|
60
|
+
Then /^no events should be submitted$/ do
|
61
|
+
$events_received.should == []
|
62
|
+
end
|
63
|
+
|
64
|
+
Then /^the following events should be submitted:$/ do
|
65
|
+
|events_str|
|
66
|
+
events_expected = YAML.load "[#{events_str}]"
|
67
|
+
$events_received.should == events_expected
|
68
|
+
end
|
@@ -0,0 +1,394 @@
|
|
1
|
+
require "hq/tools/getopt"
|
2
|
+
require "net/http"
|
3
|
+
require "multi_json"
|
4
|
+
require "xml"
|
5
|
+
|
6
|
+
module HQ
|
7
|
+
module LogMonitorClient
|
8
|
+
class Script
|
9
|
+
|
10
|
+
attr_accessor :args
|
11
|
+
attr_accessor :status
|
12
|
+
|
13
|
+
attr_accessor :stdout
|
14
|
+
attr_accessor :stderr
|
15
|
+
|
16
|
+
def main
|
17
|
+
process_args
|
18
|
+
read_config
|
19
|
+
read_cache
|
20
|
+
perform_checks
|
21
|
+
write_cache
|
22
|
+
end
|
23
|
+
|
24
|
+
def process_args
|
25
|
+
|
26
|
+
@opts, @args =
|
27
|
+
Tools::Getopt.process @args, [
|
28
|
+
|
29
|
+
{ :name => :config,
|
30
|
+
:required => true },
|
31
|
+
|
32
|
+
]
|
33
|
+
|
34
|
+
@args.empty? \
|
35
|
+
or raise "Extra args on command line"
|
36
|
+
|
37
|
+
end
|
38
|
+
|
39
|
+
def read_config
|
40
|
+
|
41
|
+
config_doc =
|
42
|
+
XML::Document.file @opts[:config]
|
43
|
+
|
44
|
+
@config_elem =
|
45
|
+
config_doc.root
|
46
|
+
|
47
|
+
@cache_elem =
|
48
|
+
@config_elem.find_first("cache")
|
49
|
+
|
50
|
+
@client_elem =
|
51
|
+
@config_elem.find_first("client")
|
52
|
+
|
53
|
+
@server_elem =
|
54
|
+
@config_elem.find_first("server")
|
55
|
+
|
56
|
+
@service_elems =
|
57
|
+
@config_elem.find("service").to_a
|
58
|
+
|
59
|
+
end
|
60
|
+
|
61
|
+
def read_cache
|
62
|
+
|
63
|
+
cache_path = @cache_elem["path"]
|
64
|
+
|
65
|
+
if File.exist? cache_path
|
66
|
+
|
67
|
+
@cache =
|
68
|
+
YAML.load File.read cache_path
|
69
|
+
|
70
|
+
else
|
71
|
+
|
72
|
+
@cache = {
|
73
|
+
files: {},
|
74
|
+
}
|
75
|
+
|
76
|
+
end
|
77
|
+
|
78
|
+
end
|
79
|
+
|
80
|
+
def write_cache
|
81
|
+
|
82
|
+
cache_path = @cache_elem["path"]
|
83
|
+
cache_temp_path = "#{cache_path}.new"
|
84
|
+
|
85
|
+
File.open cache_temp_path, "w" do
|
86
|
+
|cache_temp_io|
|
87
|
+
|
88
|
+
cache_temp_io.write YAML.dump @cache
|
89
|
+
cache_temp_io.fsync
|
90
|
+
|
91
|
+
end
|
92
|
+
|
93
|
+
File.rename cache_temp_path, cache_path
|
94
|
+
|
95
|
+
end
|
96
|
+
|
97
|
+
def perform_checks
|
98
|
+
|
99
|
+
@service_elems.each do
|
100
|
+
|service_elem|
|
101
|
+
|
102
|
+
fileset_elems = service_elem.find("fileset").to_a
|
103
|
+
|
104
|
+
fileset_elems.each do
|
105
|
+
|fileset_elem|
|
106
|
+
|
107
|
+
scan_elems = fileset_elem.find("scan").to_a
|
108
|
+
match_elems = fileset_elem.find("match").to_a
|
109
|
+
|
110
|
+
max_before =
|
111
|
+
match_elems.map {
|
112
|
+
|match_elem|
|
113
|
+
(match_elem["before"] || 0).to_i
|
114
|
+
}.max
|
115
|
+
|
116
|
+
max_after =
|
117
|
+
match_elems.map {
|
118
|
+
|match_elem|
|
119
|
+
(match_elem["after"] || 0).to_i
|
120
|
+
}.max
|
121
|
+
|
122
|
+
# find files
|
123
|
+
|
124
|
+
file_names =
|
125
|
+
scan_elems.map {
|
126
|
+
|scan_elem|
|
127
|
+
Dir[scan_elem["glob"]]
|
128
|
+
}.flatten
|
129
|
+
|
130
|
+
# scan files
|
131
|
+
|
132
|
+
file_names.each do
|
133
|
+
|file_name|
|
134
|
+
|
135
|
+
file_mtime = File.mtime file_name
|
136
|
+
file_size = File.size file_name
|
137
|
+
|
138
|
+
# fast check for modified files
|
139
|
+
|
140
|
+
cache_file = @cache[:files][file_name]
|
141
|
+
|
142
|
+
if cache_file &&
|
143
|
+
file_mtime == cache_file[:mtime] &&
|
144
|
+
file_size == cache_file[:size]
|
145
|
+
next
|
146
|
+
end
|
147
|
+
|
148
|
+
# scan the file for matching lines
|
149
|
+
|
150
|
+
mode = cache_file ? :scan : :report
|
151
|
+
|
152
|
+
File.open file_name, "r" do
|
153
|
+
|file_io|
|
154
|
+
|
155
|
+
file_reader =
|
156
|
+
ContextReader.new \
|
157
|
+
file_io,
|
158
|
+
max_before + max_after + 1
|
159
|
+
|
160
|
+
file_hash = 0
|
161
|
+
|
162
|
+
# check if the file has changed
|
163
|
+
|
164
|
+
if cache_file
|
165
|
+
|
166
|
+
if file_size < cache_file[:size]
|
167
|
+
|
168
|
+
changed = true
|
169
|
+
|
170
|
+
else
|
171
|
+
|
172
|
+
changed = false
|
173
|
+
|
174
|
+
cache_file[:lines].times do
|
175
|
+
|
176
|
+
line = file_reader.gets
|
177
|
+
|
178
|
+
unless line
|
179
|
+
changed = true
|
180
|
+
break
|
181
|
+
end
|
182
|
+
|
183
|
+
file_hash = [ file_hash, line.hash ].hash
|
184
|
+
|
185
|
+
end
|
186
|
+
|
187
|
+
if file_hash != cache_file[:hash]
|
188
|
+
changed = true
|
189
|
+
end
|
190
|
+
|
191
|
+
end
|
192
|
+
|
193
|
+
end
|
194
|
+
|
195
|
+
# go back to start if it changed
|
196
|
+
|
197
|
+
if changed
|
198
|
+
file_io.seek 0
|
199
|
+
file_reader.reset
|
200
|
+
file_hash = 0
|
201
|
+
end
|
202
|
+
|
203
|
+
# scan the new part of the file
|
204
|
+
|
205
|
+
while line = file_reader.gets
|
206
|
+
|
207
|
+
file_hash = [ file_hash, line.hash ].hash
|
208
|
+
|
209
|
+
# check for a match
|
210
|
+
|
211
|
+
match_elem =
|
212
|
+
match_elems.find {
|
213
|
+
|match_elem|
|
214
|
+
line =~ /#{match_elem["regex"]}/
|
215
|
+
}
|
216
|
+
|
217
|
+
# report the match
|
218
|
+
|
219
|
+
if match_elem
|
220
|
+
|
221
|
+
# get context
|
222
|
+
|
223
|
+
lines_before =
|
224
|
+
file_reader.lines_before \
|
225
|
+
(match_elem["before"] || 0).to_i + 1
|
226
|
+
|
227
|
+
lines_before.pop
|
228
|
+
|
229
|
+
lines_after =
|
230
|
+
file_reader.lines_after \
|
231
|
+
(match_elem["after"] || 0).to_i
|
232
|
+
|
233
|
+
# send event
|
234
|
+
|
235
|
+
submit_event({
|
236
|
+
type: match_elem["type"],
|
237
|
+
source: {
|
238
|
+
class: @client_elem["class"],
|
239
|
+
host: @client_elem["host"],
|
240
|
+
service: service_elem["name"],
|
241
|
+
},
|
242
|
+
location: {
|
243
|
+
file: file_name,
|
244
|
+
line: file_reader.last_line_number,
|
245
|
+
},
|
246
|
+
lines: {
|
247
|
+
before: lines_before,
|
248
|
+
matching: line,
|
249
|
+
after: lines_after,
|
250
|
+
},
|
251
|
+
})
|
252
|
+
|
253
|
+
end
|
254
|
+
|
255
|
+
end
|
256
|
+
|
257
|
+
# save the file's current info in the cache
|
258
|
+
|
259
|
+
@cache[:files][file_name] = {
|
260
|
+
mtime: file_mtime,
|
261
|
+
size: file_size,
|
262
|
+
lines: file_reader.next_line_number,
|
263
|
+
hash: file_hash,
|
264
|
+
}
|
265
|
+
|
266
|
+
end
|
267
|
+
|
268
|
+
end
|
269
|
+
|
270
|
+
end
|
271
|
+
|
272
|
+
end
|
273
|
+
|
274
|
+
end
|
275
|
+
|
276
|
+
def submit_event event
|
277
|
+
|
278
|
+
url =
|
279
|
+
URI.parse @server_elem["url"]
|
280
|
+
|
281
|
+
http =
|
282
|
+
Net::HTTP.new url.host, url.port
|
283
|
+
|
284
|
+
request =
|
285
|
+
Net::HTTP::Post.new url.path
|
286
|
+
|
287
|
+
request.body =
|
288
|
+
MultiJson.dump event
|
289
|
+
|
290
|
+
response =
|
291
|
+
http.request request
|
292
|
+
|
293
|
+
end
|
294
|
+
|
295
|
+
class ContextReader
|
296
|
+
|
297
|
+
def initialize source, buffer_size
|
298
|
+
|
299
|
+
@source = source
|
300
|
+
@buffer_size = buffer_size
|
301
|
+
|
302
|
+
@buffer = Array.new @buffer_size
|
303
|
+
|
304
|
+
reset
|
305
|
+
|
306
|
+
end
|
307
|
+
|
308
|
+
def lines_before_count
|
309
|
+
return @buffer_cursor - @buffer_start
|
310
|
+
end
|
311
|
+
|
312
|
+
def lines_after_count
|
313
|
+
return @buffer_end - @buffer_cursor
|
314
|
+
end
|
315
|
+
|
316
|
+
def lines_before count
|
317
|
+
count = [ count, lines_before_count ].min
|
318
|
+
return (0...count).map {
|
319
|
+
|i| @buffer[(@buffer_cursor - count + i) % @buffer_size]
|
320
|
+
}
|
321
|
+
end
|
322
|
+
|
323
|
+
def lines_after count
|
324
|
+
count = [ count, @buffer_size ].min
|
325
|
+
while lines_after_count < count
|
326
|
+
read_next_line or break
|
327
|
+
end
|
328
|
+
count = [ count, @buffer_end - @buffer_cursor].min
|
329
|
+
return (0...count).map {
|
330
|
+
|i| @buffer[(@buffer_cursor + i) % @buffer_size]
|
331
|
+
}
|
332
|
+
end
|
333
|
+
|
334
|
+
def read_next_line
|
335
|
+
|
336
|
+
# read a line
|
337
|
+
|
338
|
+
line = @source.gets
|
339
|
+
return false unless line
|
340
|
+
|
341
|
+
line.strip!
|
342
|
+
line.freeze
|
343
|
+
|
344
|
+
# shrink buffer if full
|
345
|
+
|
346
|
+
if @buffer_end - @buffer_start == @buffer_size
|
347
|
+
@buffer_start += 1
|
348
|
+
end
|
349
|
+
|
350
|
+
# add line to buffer
|
351
|
+
|
352
|
+
@buffer[@buffer_end % @buffer_size] = line
|
353
|
+
@buffer_end += 1
|
354
|
+
|
355
|
+
return true
|
356
|
+
|
357
|
+
end
|
358
|
+
|
359
|
+
def gets
|
360
|
+
|
361
|
+
# make sure the next line is in the buffer
|
362
|
+
|
363
|
+
if lines_after_count == 0
|
364
|
+
read_next_line or return nil
|
365
|
+
end
|
366
|
+
|
367
|
+
# return the line, advancing the cursor
|
368
|
+
|
369
|
+
ret = @buffer[@buffer_cursor % @buffer_size]
|
370
|
+
@buffer_cursor += 1
|
371
|
+
return ret
|
372
|
+
|
373
|
+
end
|
374
|
+
|
375
|
+
def last_line_number
|
376
|
+
raise "No last line" unless @buffer_cursor > 0
|
377
|
+
@buffer_cursor - 1
|
378
|
+
end
|
379
|
+
|
380
|
+
def next_line_number
|
381
|
+
@buffer_cursor
|
382
|
+
end
|
383
|
+
|
384
|
+
def reset
|
385
|
+
@buffer_start = 0
|
386
|
+
@buffer_cursor = 0
|
387
|
+
@buffer_end = 0
|
388
|
+
end
|
389
|
+
|
390
|
+
end
|
391
|
+
|
392
|
+
end
|
393
|
+
end
|
394
|
+
end
|
metadata
ADDED
@@ -0,0 +1,188 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: hq-log-monitor-client
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.0
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- James Pharaoh
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2013-03-21 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: hq-tools
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: 0.2.0
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 0.2.0
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: libxml-ruby
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: 2.6.0
|
38
|
+
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: 2.6.0
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: capybara
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ! '>='
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: 2.0.2
|
54
|
+
type: :development
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 2.0.2
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: cucumber
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - ! '>='
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: 1.2.1
|
70
|
+
type: :development
|
71
|
+
prerelease: false
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ! '>='
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: 1.2.1
|
78
|
+
- !ruby/object:Gem::Dependency
|
79
|
+
name: rake
|
80
|
+
requirement: !ruby/object:Gem::Requirement
|
81
|
+
none: false
|
82
|
+
requirements:
|
83
|
+
- - ! '>='
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: 10.0.3
|
86
|
+
type: :development
|
87
|
+
prerelease: false
|
88
|
+
version_requirements: !ruby/object:Gem::Requirement
|
89
|
+
none: false
|
90
|
+
requirements:
|
91
|
+
- - ! '>='
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: 10.0.3
|
94
|
+
- !ruby/object:Gem::Dependency
|
95
|
+
name: rspec
|
96
|
+
requirement: !ruby/object:Gem::Requirement
|
97
|
+
none: false
|
98
|
+
requirements:
|
99
|
+
- - ! '>='
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
version: 2.12.0
|
102
|
+
type: :development
|
103
|
+
prerelease: false
|
104
|
+
version_requirements: !ruby/object:Gem::Requirement
|
105
|
+
none: false
|
106
|
+
requirements:
|
107
|
+
- - ! '>='
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: 2.12.0
|
110
|
+
- !ruby/object:Gem::Dependency
|
111
|
+
name: rspec_junit_formatter
|
112
|
+
requirement: !ruby/object:Gem::Requirement
|
113
|
+
none: false
|
114
|
+
requirements:
|
115
|
+
- - ! '>='
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
none: false
|
122
|
+
requirements:
|
123
|
+
- - ! '>='
|
124
|
+
- !ruby/object:Gem::Version
|
125
|
+
version: '0'
|
126
|
+
- !ruby/object:Gem::Dependency
|
127
|
+
name: simplecov
|
128
|
+
requirement: !ruby/object:Gem::Requirement
|
129
|
+
none: false
|
130
|
+
requirements:
|
131
|
+
- - ! '>='
|
132
|
+
- !ruby/object:Gem::Version
|
133
|
+
version: '0'
|
134
|
+
type: :development
|
135
|
+
prerelease: false
|
136
|
+
version_requirements: !ruby/object:Gem::Requirement
|
137
|
+
none: false
|
138
|
+
requirements:
|
139
|
+
- - ! '>='
|
140
|
+
- !ruby/object:Gem::Version
|
141
|
+
version: '0'
|
142
|
+
description: HQ log monitor system, per-host component
|
143
|
+
email:
|
144
|
+
- james@phsys.co.uk
|
145
|
+
executables:
|
146
|
+
- hq-log-monitor-client
|
147
|
+
extensions: []
|
148
|
+
extra_rdoc_files: []
|
149
|
+
files:
|
150
|
+
- lib/hq/log-monitor-client/script.rb
|
151
|
+
- features/skip-files.feature
|
152
|
+
- features/match.feature
|
153
|
+
- features/context.feature
|
154
|
+
- features/detect-rotation.feature
|
155
|
+
- features/support/steps.rb
|
156
|
+
- features/support/env.rb
|
157
|
+
- bin/hq-log-monitor-client
|
158
|
+
homepage: https://github.com/jamespharaoh/hq-log-monitor-client
|
159
|
+
licenses: []
|
160
|
+
post_install_message:
|
161
|
+
rdoc_options: []
|
162
|
+
require_paths:
|
163
|
+
- lib
|
164
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
165
|
+
none: false
|
166
|
+
requirements:
|
167
|
+
- - ! '>='
|
168
|
+
- !ruby/object:Gem::Version
|
169
|
+
version: '0'
|
170
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
171
|
+
none: false
|
172
|
+
requirements:
|
173
|
+
- - ! '>='
|
174
|
+
- !ruby/object:Gem::Version
|
175
|
+
version: 1.3.6
|
176
|
+
requirements: []
|
177
|
+
rubyforge_project: hq-log-monitor-client
|
178
|
+
rubygems_version: 1.8.23
|
179
|
+
signing_key:
|
180
|
+
specification_version: 3
|
181
|
+
summary: HQ log monitor client
|
182
|
+
test_files:
|
183
|
+
- features/skip-files.feature
|
184
|
+
- features/match.feature
|
185
|
+
- features/context.feature
|
186
|
+
- features/detect-rotation.feature
|
187
|
+
- features/support/steps.rb
|
188
|
+
- features/support/env.rb
|