analogger 0.9.1 → 0.9.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +2 -2
- data/README +61 -15
- data/analogger.gemspec +1 -1
- data/bin/analogger +25 -27
- data/lib/swiftcore/Analogger.rb +68 -69
- data/lib/swiftcore/Analogger/AnaloggerProtocol.rb +4 -5
- data/lib/swiftcore/Analogger/Client.rb +10 -11
- data/lib/swiftcore/Analogger/version.rb +1 -1
- metadata +4 -5
- data/lib/swiftcore/Analogger.rb.orig +0 -291
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f3ec71320cfc00f16d6073e54ce2f9b3134f511b046365c99738c756a38ee00d
|
4
|
+
data.tar.gz: '086bc16bd8cfb9c971b6c73aa1ccfa6342474cc7a7e3d79d795b28a55dfc8746'
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 33afab0699d63cd5a0f78d70b4533900001a112a76ea9d5f585aa0a920127abffc2960436d4b2b3609a58044d3cbfe87081196634172c5a80fd3c9a74f531ee7
|
7
|
+
data.tar.gz: 368dd4f344260e13d885d6c6cc9b8d2b97561ae0e407b440c438cc1113d6dbc5d1c61306065711061cc0a035612a761b07d60d5e1b9d25f05fc1b7db30aa2f69
|
data/Gemfile.lock
CHANGED
data/README
CHANGED
@@ -1,19 +1,65 @@
|
|
1
|
-
|
1
|
+
# Analogger
|
2
2
|
|
3
|
-
|
4
|
-
Copyright:: (C) 2007 by Kirk Haines. All Rights Reserved.
|
5
|
-
Email:: wyhaines@gmail.com
|
6
|
-
License:: Ruby's License
|
3
|
+
## Overview
|
7
4
|
|
8
|
-
|
9
|
-
|
10
|
-
|
5
|
+
Analogger is a fast asynchronous logging service and client library. It is
|
6
|
+
implemented in Ruby, and currently uses EventMachine in the server, though
|
7
|
+
there is a plan on the roadmap to enable it to run with a pure Ruby event
|
8
|
+
reactor.
|
11
9
|
|
12
|
-
Analogger
|
13
|
-
|
14
|
-
|
15
|
-
|
10
|
+
Analogger was originally written over a decade ago, in response to a need to
|
11
|
+
maintain a central logging server to accumulate logs from numerous web
|
12
|
+
applications to a single location. It takes very little time to send a logging
|
13
|
+
message, making it a very low impact logger for performance sensitive
|
14
|
+
applications. It has been continuously used in production since then, albeit
|
15
|
+
in a version not released publicly.
|
16
16
|
|
17
|
-
|
18
|
-
|
19
|
-
|
17
|
+
## Usage
|
18
|
+
|
19
|
+
Analogger is configured through a YAML formatted file:
|
20
|
+
|
21
|
+
```yaml
|
22
|
+
host: mycompany-logger-1-nyc1.private
|
23
|
+
port: 6766
|
24
|
+
default_log: /var/log/analogger_default
|
25
|
+
daemonize: true
|
26
|
+
syncinterval: 5
|
27
|
+
logs:
|
28
|
+
- service:
|
29
|
+
- default
|
30
|
+
logfile: /var/log/analogger/default
|
31
|
+
cull: true
|
32
|
+
- service:
|
33
|
+
- project-development
|
34
|
+
logfile: /var/log/analogger/project-development.log
|
35
|
+
- service:
|
36
|
+
- project-production
|
37
|
+
logfile: /var/log/analogger/project-production.log
|
38
|
+
cull: true
|
39
|
+
```
|
40
|
+
|
41
|
+
### Configuration Variables
|
42
|
+
|
43
|
+
* port
|
44
|
+
|
45
|
+
The port to listen for connections on. 6766 is the default.
|
46
|
+
|
47
|
+
* host
|
48
|
+
|
49
|
+
The hostname or IP to bind to when listening for connections.
|
50
|
+
|
51
|
+
* default_log: /var/log/analogger_default
|
52
|
+
|
53
|
+
This is the file to send logs to which don't appear to match any named service in the configuration.
|
54
|
+
|
55
|
+
* daemonize
|
56
|
+
|
57
|
+
Whether or not to detach an analogger process as a daemon process. You normally want this to be true.
|
58
|
+
|
59
|
+
* syncinterval
|
60
|
+
|
61
|
+
Analogger will run a thread every X seconds to ensure that any currently buffered log contents are synchronized to disk. Analogger tries to write any buffered logs before it exits if it receives a signal which would cause the process to die. However, in the event that this is not possible, only the logs received since the last sync interval would be at risk.
|
62
|
+
|
63
|
+
* logs
|
64
|
+
|
65
|
+
This is a list of defined logging services. Each consists of a service label, a logging destination (the path to the log file for that service), and optionally a `cull` attribute which, if true, causes analogger to deduplicate logs, eliminating consecutive repeats of the same message and instead emitting a summary of how many records like the one above the summary were culled.
|
data/analogger.gemspec
CHANGED
@@ -38,6 +38,6 @@ Gem::Specification.new do |spec|
|
|
38
38
|
spec.add_development_dependency "bundler", "~> 1.10"
|
39
39
|
spec.add_development_dependency "rake", "~> 11.0"
|
40
40
|
spec.add_development_dependency "minitest", "~> 5"
|
41
|
-
spec.add_runtime_dependency "eventmachine", "~> 1.2"
|
41
|
+
spec.add_runtime_dependency "eventmachine", "~> 1.2.5"
|
42
42
|
end
|
43
43
|
|
data/bin/analogger
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
|
1
|
+
#!/usr/bin/env ruby
|
2
2
|
|
3
3
|
require 'yaml'
|
4
4
|
require 'optparse'
|
@@ -47,39 +47,39 @@ module Swiftcore
|
|
47
47
|
# cull: true
|
48
48
|
#
|
49
49
|
OptionParser.new do |opts|
|
50
|
-
opts.banner = "Analogger v#{Swiftcore::Analogger::VERSION}\nUsage: analogger.rb [options]"
|
51
|
-
opts.separator
|
52
|
-
opts.on(
|
50
|
+
opts.banner = -"Analogger v#{Swiftcore::Analogger::VERSION}\nUsage: analogger.rb [options]"
|
51
|
+
opts.separator -""
|
52
|
+
opts.on(-"-c", -"--config CONFFILE", "The configuration file to read.") do |conf|
|
53
53
|
config = YAML.load(File.read(conf))
|
54
54
|
end
|
55
|
-
opts.on(
|
56
|
-
config[
|
55
|
+
opts.on(-"-p", -"--port [PORT]", Integer, "The port to receive connections on.") do |port|
|
56
|
+
config[-"port"] = port
|
57
57
|
end
|
58
|
-
opts.on(
|
59
|
-
config[
|
58
|
+
opts.on(-"-h", -"--host [HOST]", String, "The host to bind the connection to.") do |host|
|
59
|
+
config[-"host"] = host
|
60
60
|
end
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
opts.on(
|
65
|
-
config[
|
61
|
+
opts.on(-"-r",-"--controlkey [KEY]",String,-"The secret key that authenticates a control session.") do |secret|
|
62
|
+
config[-"secret"] = secret
|
63
|
+
end
|
64
|
+
opts.on(-"-k", -"--key [KEY]", String, -"The secret key that authenticates a valid client session.") do |secret|
|
65
|
+
config[-"key"] = secret
|
66
66
|
end
|
67
|
-
opts.on(
|
68
|
-
config[
|
67
|
+
opts.on(-"-i", -"--interval [INTERVAL]", Integer, -"The interval between queue writes. Defaults to 1 second.") do |interval|
|
68
|
+
config[-"interval"] = interval
|
69
69
|
end
|
70
|
-
opts.on(
|
71
|
-
config[
|
70
|
+
opts.on(-"-s", -"--syncinterval [INTERVAL]", Integer, -"The interval between queue syncs. Defaults to 60 seconds.") do |interval|
|
71
|
+
config[-"syncinterval"] = interval
|
72
72
|
end
|
73
|
-
opts.on(
|
74
|
-
config[
|
73
|
+
opts.on(-"-d", -"--default [PATH]", String, -"The default log destination. Defaults to stdout.") do |default|
|
74
|
+
config[-"default_log"] = default
|
75
75
|
end
|
76
|
-
opts.on(
|
77
|
-
config[
|
76
|
+
opts.on(-"-x", -"--daemonize", -"Tell the Analogger to daemonize itself.") do
|
77
|
+
config[-"daemonize"] = true
|
78
78
|
end
|
79
|
-
opts.on(
|
80
|
-
config[
|
79
|
+
opts.on(-"-w", -"--writepid [FILENAME]", -"The filename to write a PID file to.") do |pidfile|
|
80
|
+
config[-"pidfile"] = pidfile || 'analogger.pid'
|
81
81
|
end
|
82
|
-
opts.on(
|
82
|
+
opts.on(-"-v", -"--version", -"Show the current version of Analogger.") do
|
83
83
|
puts "Analogger v#{Swiftcore::Analogger::VERSION}"
|
84
84
|
exit
|
85
85
|
end
|
@@ -95,6 +95,4 @@ module Swiftcore
|
|
95
95
|
end
|
96
96
|
end
|
97
97
|
|
98
|
-
|
99
|
-
catch(:hup) {Swiftcore::AnaloggerExec.run}
|
100
|
-
end
|
98
|
+
Swiftcore::AnaloggerExec.run
|
data/lib/swiftcore/Analogger.rb
CHANGED
@@ -6,25 +6,9 @@ require 'swiftcore/Analogger/AnaloggerProtocol'
|
|
6
6
|
|
7
7
|
module Swiftcore
|
8
8
|
class Analogger
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
Cdaemonize = 'daemonize'.freeze
|
13
|
-
Cdefault = 'default'.freeze
|
14
|
-
Cdefault_log = 'default_log'.freeze
|
15
|
-
Chost = 'host'.freeze
|
16
|
-
Cinterval = 'interval'.freeze
|
17
|
-
Ckey = 'key'.freeze
|
18
|
-
Clogfile = 'logfile'.freeze
|
19
|
-
Clogs = 'logs'.freeze
|
20
|
-
Cport = 'port'.freeze
|
21
|
-
Csecret = 'secret'.freeze
|
22
|
-
Cservice = 'service'.freeze
|
23
|
-
Clevels = 'levels'.freeze
|
24
|
-
Csyncinterval = 'syncinterval'.freeze
|
25
|
-
Cpidfile = 'pidfile'.freeze
|
26
|
-
DefaultSeverityLevels = ['debug','info','warn','error','fatal'].inject({}){|h,k|h[k]=true;h}
|
27
|
-
TimeFormat = '%Y/%m/%d %H:%M:%S'.freeze
|
9
|
+
EXEC_ARGUMENTS = [File.expand_path(Process.argv0), *ARGV]
|
10
|
+
|
11
|
+
DefaultSeverityLevels = [-"debug",-"info",-"warn",-"error",-"fatal"].inject({}){|h,k|h[k]=true;h}
|
28
12
|
|
29
13
|
class NoPortProvided < Exception; def to_s; "The port to bind to was not provided."; end; end
|
30
14
|
class BadPort < Exception
|
@@ -37,6 +21,7 @@ module Swiftcore
|
|
37
21
|
|
38
22
|
EXIT_SIGNALS = %w[INT TERM]
|
39
23
|
RELOAD_SIGNALS = %w[HUP]
|
24
|
+
RESTART_SIGNALS = %w[USR2]
|
40
25
|
|
41
26
|
class << self
|
42
27
|
def safe_trap(siglist, &operation)
|
@@ -45,8 +30,8 @@ module Swiftcore
|
|
45
30
|
|
46
31
|
def start(config,protocol = AnaloggerProtocol)
|
47
32
|
@config = config
|
48
|
-
daemonize if @config[
|
49
|
-
File.open(@config[
|
33
|
+
daemonize if @config[-"daemonize"]
|
34
|
+
File.open(@config[-"pidfile"],-"w+") {|fh| fh.puts $$} if @config[-"pidfile"]
|
50
35
|
@logs = Hash.new {|h,k| h[k] = new_log(k)}
|
51
36
|
@queue = Hash.new {|h,k| h[k] = []}
|
52
37
|
postprocess_config_load
|
@@ -57,8 +42,8 @@ module Swiftcore
|
|
57
42
|
@wcount = 0
|
58
43
|
@server = nil
|
59
44
|
safe_trap(EXIT_SIGNALS) {handle_pending_and_exit}
|
60
|
-
safe_trap(RELOAD_SIGNALS) {
|
61
|
-
|
45
|
+
safe_trap(RELOAD_SIGNALS) {cleanup_and_reopen}
|
46
|
+
safe_trap(RESTART_SIGNALS) {exec(*EXEC_ARGUMENTS)}
|
62
47
|
|
63
48
|
#####
|
64
49
|
# This is gross. EM needs to change so that it defaults to the faster
|
@@ -80,17 +65,17 @@ module Swiftcore
|
|
80
65
|
flush_queue
|
81
66
|
cleanup
|
82
67
|
end
|
83
|
-
@server = EventMachine.start_server @config[
|
68
|
+
@server = EventMachine.start_server @config[-"host"], @config[-"port"], protocol
|
84
69
|
EventMachine.add_periodic_timer(1) {Analogger.update_now}
|
85
|
-
EventMachine.add_periodic_timer(@config[
|
86
|
-
EventMachine.add_periodic_timer(@config[
|
70
|
+
EventMachine.add_periodic_timer(@config[-"interval"]) {write_queue}
|
71
|
+
EventMachine.add_periodic_timer(@config[-"syncinterval"]) {flush_queue}
|
87
72
|
}
|
88
73
|
exit
|
89
74
|
end
|
90
75
|
|
91
76
|
def daemonize
|
92
77
|
if (child_pid = fork)
|
93
|
-
puts "PID #{child_pid}" unless @config[
|
78
|
+
puts "PID #{child_pid}" unless @config[-"pidfile"]
|
94
79
|
exit!
|
95
80
|
end
|
96
81
|
Process.setsid
|
@@ -101,8 +86,8 @@ module Swiftcore
|
|
101
86
|
puts "Platform (#{RUBY_PLATFORM}) does not appear to support fork/setsid; skipping"
|
102
87
|
end
|
103
88
|
|
104
|
-
def new_log(facility =
|
105
|
-
Log.new({
|
89
|
+
def new_log(facility = -"default", levels = @config[-"levels"] || DefaultSeverityLevels, log = @config[-"default_log"], cull = true)
|
90
|
+
Log.new({-"service" => facility, -"levels" => levels, -"logfile" => log, -"cull" => cull})
|
106
91
|
end
|
107
92
|
|
108
93
|
# Before exiting, try to get any logs that are still in memory handled and written to disk.
|
@@ -129,8 +114,15 @@ module Swiftcore
|
|
129
114
|
end
|
130
115
|
end
|
131
116
|
|
117
|
+
def cleanup_and_reopen
|
118
|
+
@logs.each do |service,l|
|
119
|
+
l.logfile.fsync if !l.logfile.closed? and l.logfile.fileno > 2
|
120
|
+
l.logfile.reopen(l.logfile.path, -"ab+") if l.logfile.fileno > 2
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
132
124
|
def update_now
|
133
|
-
@now = Time.now.strftime(
|
125
|
+
@now = Time.now.strftime(-"%Y/%m/%d %H:%M:%S")
|
134
126
|
end
|
135
127
|
|
136
128
|
def config
|
@@ -142,26 +134,26 @@ module Swiftcore
|
|
142
134
|
end
|
143
135
|
|
144
136
|
def populate_logs
|
145
|
-
@config[
|
146
|
-
next unless log[
|
147
|
-
if Array === log[
|
148
|
-
log[
|
149
|
-
@logs[loglog] = new_log(loglog,log[
|
137
|
+
@config[-"logs"].each do |log|
|
138
|
+
next unless log[-"service"]
|
139
|
+
if Array === log[-"service"]
|
140
|
+
log[-"service"].each do |loglog|
|
141
|
+
@logs[loglog] = new_log(loglog,log[-"levels"],logfile_destination(log[-"logfile"]),log[-"cull"])
|
150
142
|
end
|
151
143
|
else
|
152
|
-
@logs[log[
|
144
|
+
@logs[log[-"service"]] = new_log(log[-"service"],log[-"levels"],logfile_destination(log[-"logfile"]),log[-"cull"])
|
153
145
|
end
|
154
146
|
end
|
155
147
|
end
|
156
148
|
|
157
149
|
def postprocess_config_load
|
158
|
-
@config[
|
159
|
-
if @config[
|
160
|
-
@config[
|
150
|
+
@config[-"logs"] ||= []
|
151
|
+
if @config[-"levels"]
|
152
|
+
@config[-"levels"] = normalize_levels(@config[-"levels"])
|
161
153
|
end
|
162
154
|
|
163
|
-
@config[
|
164
|
-
log[
|
155
|
+
@config[-"logs"].each do |log|
|
156
|
+
log[-"levels"] = normalize_levels(log[-"levels"])
|
165
157
|
end
|
166
158
|
end
|
167
159
|
|
@@ -180,18 +172,18 @@ module Swiftcore
|
|
180
172
|
end
|
181
173
|
|
182
174
|
def check_config_settings
|
183
|
-
raise NoPortProvided unless @config[
|
184
|
-
raise BadPort.new(@config[
|
175
|
+
raise NoPortProvided unless @config[-"port"]
|
176
|
+
raise BadPort.new(@config[-"port"]) unless @config[-"port"].to_i > 0
|
185
177
|
end
|
186
178
|
|
187
179
|
def set_config_defaults
|
188
|
-
@config[
|
189
|
-
@config[
|
190
|
-
@config[
|
191
|
-
@config[
|
192
|
-
@config[
|
193
|
-
@config[
|
194
|
-
@logs[
|
180
|
+
@config[-"host"] ||= -"127.0.0.1"
|
181
|
+
@config[-"interval"] ||= 1
|
182
|
+
@config[-"syncinterval"] ||= 60
|
183
|
+
@config[-"syncinterval"] = nil if @config[-"syncinterval"] == 0
|
184
|
+
@config[-"default_log"] = @config[-"default_log"].nil? || @config[-"default_log"] == -"-" ? -"STDOUT" : @config[-"default_log"]
|
185
|
+
@config[-"default_log"] = logfile_destination(@config[-"default_log"])
|
186
|
+
@logs[-"default"] = new_log
|
195
187
|
end
|
196
188
|
|
197
189
|
def logfile_destination(logfile)
|
@@ -199,7 +191,7 @@ module Swiftcore
|
|
199
191
|
if logfile.is_a?(IO)
|
200
192
|
return $stdout if logfile == $stdout
|
201
193
|
return $stderr if logfile == $stderr
|
202
|
-
return logfile.reopen(logfile.path,
|
194
|
+
return logfile.reopen(logfile.path, -"ab+")
|
203
195
|
end
|
204
196
|
|
205
197
|
if logfile =~ /^STDOUT$/i
|
@@ -207,7 +199,7 @@ module Swiftcore
|
|
207
199
|
elsif logfile =~ /^STDERR$/i
|
208
200
|
$stderr
|
209
201
|
else
|
210
|
-
File.open(logfile,
|
202
|
+
File.open(logfile, -"ab+")
|
211
203
|
end
|
212
204
|
end
|
213
205
|
|
@@ -243,25 +235,26 @@ module Swiftcore
|
|
243
235
|
last_count += 1
|
244
236
|
next
|
245
237
|
elsif last_count > 0
|
246
|
-
lf.write_nonblock "#{@now}|#{last_sv.join(
|
238
|
+
lf.write_nonblock "#{@now}|#{last_sv.join(-"|")}|Last message repeated #{last_count} times\n"
|
247
239
|
last_sv = last_m = nil
|
248
240
|
last_count = 0
|
249
241
|
end
|
250
|
-
lf.write_nonblock "#{@now}|#{m.join(
|
242
|
+
lf.write_nonblock "#{@now}|#{m.join(-"|")}\n"
|
251
243
|
last_m = m.last
|
252
244
|
last_sv = m[0..1]
|
253
245
|
else
|
254
|
-
lf.write_nonblock "#{@now}|#{m.join(
|
246
|
+
lf.write_nonblock "#{@now}|#{m.join(-"|")}\n"
|
255
247
|
end
|
256
248
|
@wcount += 1
|
257
249
|
end
|
258
|
-
lf.write_nonblock "#{@now}|#{last_sv.join(
|
250
|
+
lf.write_nonblock "#{@now}|#{last_sv.join(-"|")}|Last message repeated #{last_count} times\n" if cull and last_count > 0
|
259
251
|
end
|
260
252
|
@queue.each {|service,q| q.clear}
|
261
253
|
end
|
262
254
|
|
263
255
|
def flush_queue
|
264
256
|
@logs.each_value do |l|
|
257
|
+
#if !l.logfile.closed? and l.logfile.fileno > 2
|
265
258
|
if l.logfile.fileno > 2
|
266
259
|
l.logfile.fdatasync rescue l.logfile.fsync
|
267
260
|
end
|
@@ -269,30 +262,36 @@ module Swiftcore
|
|
269
262
|
end
|
270
263
|
|
271
264
|
def key
|
272
|
-
@config[
|
265
|
+
@config[-"key"].to_s
|
273
266
|
end
|
274
267
|
|
275
268
|
end
|
276
269
|
|
277
|
-
|
270
|
+
class Log
|
271
|
+
attr_reader :service, :levels, :logfile, :cull
|
278
272
|
|
279
|
-
|
280
|
-
|
273
|
+
def initialize(spec)
|
274
|
+
@service = spec[-"service"]
|
275
|
+
@levels = spec[-"levels"]
|
276
|
+
@logfile = spec[-"logfile"]
|
277
|
+
@cull = spec[-"cull"]
|
278
|
+
end
|
281
279
|
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
280
|
+
def to_s
|
281
|
+
"service: #{@service}\nlevels: #{@levels.inspect}\nlogfile: #{@logfile}\ncull: #{@cull}\n"
|
282
|
+
end
|
283
|
+
|
284
|
+
def ==(n)
|
285
|
+
n.service == @service &&
|
286
|
+
n.levels == @levels &&
|
287
|
+
n.logfile == @logfile &&
|
288
|
+
n.cull == @cull
|
289
|
+
end
|
288
290
|
|
289
|
-
def to_s
|
290
|
-
"service: #{@service}\nlevels: #{@levels.inspect}\nlogfile: #{@logfile}\ncull: #{@cull}\n"
|
291
291
|
end
|
292
292
|
end
|
293
293
|
|
294
294
|
class AnaloggerProtocol < EventMachine::Connection
|
295
|
-
Ci = 'i'.freeze
|
296
295
|
Rcolon = /:/
|
297
296
|
|
298
297
|
LoggerClass = Analogger
|
@@ -3,7 +3,6 @@ module Swiftcore
|
|
3
3
|
|
4
4
|
MaxMessageLength = 8192
|
5
5
|
MaxLengthBytes = MaxMessageLength.to_s.length
|
6
|
-
Semaphore = "||"
|
7
6
|
|
8
7
|
def setup
|
9
8
|
@length = nil
|
@@ -32,11 +31,11 @@ module Swiftcore
|
|
32
31
|
peer = peer ? ::Socket.unpack_sockaddr_in(peer)[1] : 'UNK'
|
33
32
|
if l == ck
|
34
33
|
LoggerClass.add_log([:default, :error, "Max Length Exceeded from #{peer} -- #{l}/#{MaxMessageLength}"])
|
35
|
-
send_data
|
34
|
+
send_data(-"error: max length exceeded\n")
|
36
35
|
close_connection_after_writing
|
37
36
|
else
|
38
37
|
LoggerClass.add_log([:default, :error, "checksum failed from #{peer} -- #{l}/#{ck}"])
|
39
|
-
send_data
|
38
|
+
send_data(-"error: checksum failed\n")
|
40
39
|
close_connection_after_writing
|
41
40
|
end
|
42
41
|
end
|
@@ -54,9 +53,9 @@ module Swiftcore
|
|
54
53
|
unless @authenticated
|
55
54
|
if msg.last == LoggerClass.key
|
56
55
|
@authenticated = true
|
57
|
-
send_data
|
56
|
+
send_data(-"accepted\n")
|
58
57
|
else
|
59
|
-
send_data
|
58
|
+
send_data(-"denied\n")
|
60
59
|
close_connection_after_writing
|
61
60
|
end
|
62
61
|
else
|
@@ -43,17 +43,13 @@ module Swiftcore
|
|
43
43
|
class Client
|
44
44
|
|
45
45
|
class FailedToAuthenticate < StandardError
|
46
|
-
def initialize(hots = "UNK", port = 6766)
|
46
|
+
def initialize(hots = -"UNK", port = 6766)
|
47
47
|
super("Failed to authenticate to the Analogger server at #{destination}:#{port}")
|
48
48
|
end
|
49
49
|
end
|
50
50
|
|
51
|
-
Cauthentication = 'authentication'.freeze
|
52
|
-
Ci = 'i'.freeze
|
53
|
-
|
54
51
|
MaxMessageLength = 8192
|
55
52
|
MaxLengthBytes = MaxMessageLength.to_s.length
|
56
|
-
Semaphore = "||"
|
57
53
|
ConnectionFailureTimeout = 86400 * 2 # Log locally for a long time if Analogger server goes down.
|
58
54
|
MaxFailureCount = (2**(0.size * 8 - 2) - 1) # Max integer -- i.e. really big
|
59
55
|
PersistentQueueLimit = 10737412742 # Default to allowing around 10GB temporary local log storage
|
@@ -115,15 +111,18 @@ module Swiftcore
|
|
115
111
|
|
116
112
|
#-----
|
117
113
|
|
118
|
-
def initialize(service =
|
114
|
+
def initialize(service = -"default", host = -"127.0.0.1" , port = 6766, key = nil)
|
119
115
|
@service = service.to_s
|
120
116
|
@key = key
|
121
117
|
@host = host
|
122
118
|
@port = port
|
119
|
+
@socket = nil
|
123
120
|
klass = self.class
|
124
121
|
@connection_failure_timeout = klass.connection_failure_timeout
|
125
122
|
@max_failure_count = klass.max_failure_count
|
126
123
|
@persistent_queue_limit = klass.persistent_queue_limit
|
124
|
+
@destination = nil
|
125
|
+
@reconnection_thread = nil
|
127
126
|
@authenticated = false
|
128
127
|
@total_count = 0
|
129
128
|
@logfile = nil
|
@@ -173,7 +172,7 @@ module Swiftcore
|
|
173
172
|
end
|
174
173
|
|
175
174
|
def tmplog_prefix
|
176
|
-
File.join(Dir.tmpdir, "analogger-SERVICE-PID.log")
|
175
|
+
File.join(Dir.tmpdir, -"analogger-SERVICE-PID.log")
|
177
176
|
end
|
178
177
|
|
179
178
|
def tmplog
|
@@ -181,7 +180,7 @@ module Swiftcore
|
|
181
180
|
end
|
182
181
|
|
183
182
|
def tmplogs
|
184
|
-
Dir[tmplog_prefix.gsub(/SERVICE/, @service).gsub(/PID
|
183
|
+
Dir[tmplog_prefix.gsub(/SERVICE/, @service).gsub(/PID/,-"*")].sort_by {|f| File.mtime(f)}
|
185
184
|
end
|
186
185
|
|
187
186
|
def tmplog=(val)
|
@@ -222,7 +221,7 @@ module Swiftcore
|
|
222
221
|
|
223
222
|
def setup_local_logging
|
224
223
|
unless @logfile && !@logfile.closed?
|
225
|
-
@logfile = File.open(tmplog
|
224
|
+
@logfile = File.open(tmplog,-"a+")
|
226
225
|
@destination = :local
|
227
226
|
end
|
228
227
|
end
|
@@ -297,7 +296,7 @@ module Swiftcore
|
|
297
296
|
|
298
297
|
def authenticate
|
299
298
|
begin
|
300
|
-
_remote_log(@service,
|
299
|
+
_remote_log(@service, -"authentication", "#{@key}")
|
301
300
|
response = @socket.gets
|
302
301
|
rescue Exception
|
303
302
|
response = nil
|
@@ -354,7 +353,7 @@ module Swiftcore
|
|
354
353
|
records.each_index do |n|
|
355
354
|
record = records[n]
|
356
355
|
next if record =~ /^\#/
|
357
|
-
service, severity, msg = record.split(":", 3)
|
356
|
+
service, severity, msg = record.split(-":", 3)
|
358
357
|
msg = msg.chomp.gsub(/\x00\x00/, "\n")
|
359
358
|
begin
|
360
359
|
_remote_log(service, severity, msg)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: analogger
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.9.
|
4
|
+
version: 0.9.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kirk Haines
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-03-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -58,14 +58,14 @@ dependencies:
|
|
58
58
|
requirements:
|
59
59
|
- - "~>"
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version:
|
61
|
+
version: 1.2.5
|
62
62
|
type: :runtime
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
66
|
- - "~>"
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version:
|
68
|
+
version: 1.2.5
|
69
69
|
description: Analogger provides a fast and very stable asynchronous central logging
|
70
70
|
service capable of handling heavy logging loads. It has been in production use for
|
71
71
|
almost a decade.
|
@@ -86,7 +86,6 @@ files:
|
|
86
86
|
- external/package.rb
|
87
87
|
- external/test_support.rb
|
88
88
|
- lib/swiftcore/Analogger.rb
|
89
|
-
- lib/swiftcore/Analogger.rb.orig
|
90
89
|
- lib/swiftcore/Analogger/AnaloggerProtocol.rb
|
91
90
|
- lib/swiftcore/Analogger/Client.rb
|
92
91
|
- lib/swiftcore/Analogger/EMClient.rb
|
@@ -1,291 +0,0 @@
|
|
1
|
-
require 'socket'
|
2
|
-
begin
|
3
|
-
load_attempted ||= false
|
4
|
-
require 'eventmachine'
|
5
|
-
require 'benchmark'
|
6
|
-
rescue LoadError => e
|
7
|
-
unless load_attempted
|
8
|
-
load_attempted = true
|
9
|
-
require 'rubygems'
|
10
|
-
retry
|
11
|
-
end
|
12
|
-
raise e
|
13
|
-
end
|
14
|
-
|
15
|
-
module Swiftcore
|
16
|
-
class Analogger
|
17
|
-
C_colon = ':'.freeze
|
18
|
-
C_bar = '|'.freeze
|
19
|
-
Ccull = 'cull'.freeze
|
20
|
-
Cdaemonize = 'daemonize'.freeze
|
21
|
-
Cdefault = 'default'.freeze
|
22
|
-
Cdefault_log = 'default_log'.freeze
|
23
|
-
Cepoll = 'epoll'.freeze
|
24
|
-
Chost = 'host'.freeze
|
25
|
-
Cinterval = 'interval'.freeze
|
26
|
-
Ckey = 'key'.freeze
|
27
|
-
Ckqueue = 'kqueue'.freeze
|
28
|
-
Clevels = 'levels'.freeze
|
29
|
-
Clogfile = 'logfile'.freeze
|
30
|
-
Clogs = 'logs'.freeze
|
31
|
-
Cpidfile = 'pidfile'.freeze
|
32
|
-
Cport = 'port'.freeze
|
33
|
-
Croll = 'roll'.freeze
|
34
|
-
Croll_age = 'roll_age'.freeze
|
35
|
-
Croll_size = 'roll_size'.freeze
|
36
|
-
Croll_interval = 'roll_interval'.freeze
|
37
|
-
Csecret = 'secret'.freeze
|
38
|
-
Cservice = 'service'.freeze
|
39
|
-
Csyncinterval = 'syncinterval'.freeze
|
40
|
-
DefaultSeverityLevels = ['debug','info','warn','error','fatal'].inject({}){|h,k|h[k]=true;h}
|
41
|
-
TimeFormat = '%Y/%m/%d %H:%M:%S'.freeze
|
42
|
-
|
43
|
-
class NoPortProvided < Exception; def to_s; "The port to bind to was not provided."; end; end
|
44
|
-
class BadPort < Exception
|
45
|
-
def initialize(port)
|
46
|
-
@port = port
|
47
|
-
end
|
48
|
-
|
49
|
-
def to_s; "The port provided (#{@port}) is invalid."; end
|
50
|
-
end
|
51
|
-
|
52
|
-
EXIT_SIGNALS = %w[INT TERM]
|
53
|
-
RELOAD_SIGNALS = %w[HUP]
|
54
|
-
|
55
|
-
class << self
|
56
|
-
def safe_trap(siglist, &operation)
|
57
|
-
(Signal.list.keys & siglist).each {|sig| trap(sig, &operation)}
|
58
|
-
end
|
59
|
-
|
60
|
-
def start(config,protocol = AnaloggerProtocol)
|
61
|
-
@config = config
|
62
|
-
daemonize if @config[Cdaemonize]
|
63
|
-
File.open(@config[Cpidfile],'w+') {|fh| fh.puts $$} if @config[Cpidfile]
|
64
|
-
@logs = Hash.new {|h,k| h[k] = new_log(k)}
|
65
|
-
@queue = Hash.new {|h,k| h[k] = []}
|
66
|
-
postprocess_config_load
|
67
|
-
check_config_settings
|
68
|
-
populate_logs
|
69
|
-
set_config_defaults
|
70
|
-
@rcount = 0
|
71
|
-
@wcount = 0
|
72
|
-
safe_trap(EXIT_SIGNALS) {cleanup;exit}
|
73
|
-
safe_trap(RELOAD_SIGNALS) {cleanup;throw :hup}
|
74
|
-
|
75
|
-
if @config[Cepoll] or @config[Ckqueue]
|
76
|
-
EventMachine.epoll if @config[Cepoll]
|
77
|
-
EventMachine.kqueue if @config[Ckqueue]
|
78
|
-
|
79
|
-
EventMachine.set_descriptor_table_size(4096)
|
80
|
-
end
|
81
|
-
|
82
|
-
EventMachine.run {
|
83
|
-
EventMachine.start_server @config[Chost], @config[Cport], protocol
|
84
|
-
EventMachine.add_periodic_timer(1) {Analogger.update_now}
|
85
|
-
EventMachine.add_periodic_timer(@config[Cinterval]) {write_queue}
|
86
|
-
EventMachine.add_periodic_timer(@config[Csyncinterval]) {flush_queue}
|
87
|
-
EventMachine.add_periodic_timer(@config[Crollinterval]) {roll logs}
|
88
|
-
}
|
89
|
-
end
|
90
|
-
|
91
|
-
def daemonize
|
92
|
-
if (child_pid = fork)
|
93
|
-
puts "PID #{child_pid}" unless @config[Cpidfile]
|
94
|
-
exit!
|
95
|
-
end
|
96
|
-
Process.setsid
|
97
|
-
|
98
|
-
rescue Exception
|
99
|
-
puts "Platform (#{RUBY_PLATFORM}) does not appear to support fork/setsid; skipping"
|
100
|
-
end
|
101
|
-
|
102
|
-
def new_log(facility = Cdefault, levels = @config[Clevels] || DefaultSeverityLevels, log = @config[Cdefault_log], cull = true, roll = @config[Croll], roll_age = @config[Croll_age], roll_size = @config[Croll_size])
|
103
|
-
Log.new({Cservice => facility, Clevels => levels, Clogfile => log, Ccull => cull, Croll => roll, Croll_age => roll_age, Croll_size => roll_size})
|
104
|
-
end
|
105
|
-
|
106
|
-
def cleanup
|
107
|
-
@logs.each do |service,l|
|
108
|
-
l.logfile.fsync if !l.logfile.closed? and l.logfile.fileno > 2
|
109
|
-
l.logfile.close unless l.logfile.closed? or l.logfile.fileno < 3
|
110
|
-
end
|
111
|
-
end
|
112
|
-
|
113
|
-
def roll_logs
|
114
|
-
@logs.each do |service,l|
|
115
|
-
end
|
116
|
-
end
|
117
|
-
|
118
|
-
def update_now
|
119
|
-
@now = Time.now.strftime(TimeFormat)
|
120
|
-
end
|
121
|
-
|
122
|
-
def config
|
123
|
-
@config
|
124
|
-
end
|
125
|
-
|
126
|
-
def config=(conf)
|
127
|
-
@config = conf
|
128
|
-
end
|
129
|
-
|
130
|
-
def populate_logs
|
131
|
-
@config[Clogs].each do |log|
|
132
|
-
next unless log[Cservice]
|
133
|
-
roll = log[Croll] || log[Croll_age] || log[Croll_size] ? true : false
|
134
|
-
if Array === log[Cservice]
|
135
|
-
log[Cservice].each do |loglog|
|
136
|
-
@logs[loglog] = new_log(loglog,log[Clevels],logfile_destination(log[Clogfile]),log[Ccull],roll,log[Croll_age],log[Croll_size])
|
137
|
-
end
|
138
|
-
else
|
139
|
-
@logs[log[Cservice]] = new_log(log[Cservice],log[Clevels],logfile_destination(log[Clogfile]),log[Ccull],roll,log[Croll_age],log[Croll_size])
|
140
|
-
end
|
141
|
-
end
|
142
|
-
end
|
143
|
-
|
144
|
-
def postprocess_config_load
|
145
|
-
@config[Clogs] ||= []
|
146
|
-
if @config[Clevels]
|
147
|
-
@config[Clevels] = normalize_levels(@config[Clevels])
|
148
|
-
end
|
149
|
-
|
150
|
-
@config[Clogs].each do |log|
|
151
|
-
log[Clevels] = normalize_levels(log[Clevels])
|
152
|
-
end
|
153
|
-
end
|
154
|
-
|
155
|
-
def normalize_levels(levels)
|
156
|
-
if String === levels and levels =~ /,/
|
157
|
-
levels.split(/,/).inject({}) {|h,k| h[k.to_s] = true; h}
|
158
|
-
elsif Array === levels
|
159
|
-
levels.inject({}) {|h,k| h[k.to_s] = true; h}
|
160
|
-
elsif levels.nil?
|
161
|
-
DefaultSeverityLevels
|
162
|
-
elsif !(Hash === levels)
|
163
|
-
[levels.to_s => true]
|
164
|
-
else
|
165
|
-
levels
|
166
|
-
end
|
167
|
-
end
|
168
|
-
|
169
|
-
def check_config_settings
|
170
|
-
raise NoPortProvided unless @config[Cport]
|
171
|
-
raise BadPort.new(@config[Cport]) unless @config[Cport].to_i > 0
|
172
|
-
end
|
173
|
-
|
174
|
-
def set_config_defaults
|
175
|
-
@config[Chost] ||= '127.0.0.1'
|
176
|
-
@config[Cinterval] ||= 1
|
177
|
-
@config[Csyncinterval] ||= 60
|
178
|
-
@config[Csyncinterval] = nil if @config[Csyncinterval] == 0
|
179
|
-
@config[Cdefault_log] = @config[Cdefault_log].nil? || @config[Cdefault_log] == '-' ? 'STDOUT' : @config[Cdefault_log]
|
180
|
-
@config[Cdefault_log] = logfile_destination(@config[Cdefault_log])
|
181
|
-
@logs['default'] = new_log
|
182
|
-
end
|
183
|
-
|
184
|
-
def logfile_destination(logfile)
|
185
|
-
# We're reloading if it's already an IO.
|
186
|
-
if logfile.is_a?(IO)
|
187
|
-
return $stdout if logfile == $stdout
|
188
|
-
return $stderr if logfile == $stderr
|
189
|
-
return logfile.reopen(logfile.path, 'ab+')
|
190
|
-
end
|
191
|
-
|
192
|
-
if logfile =~ /^STDOUT$/i
|
193
|
-
$stdout
|
194
|
-
elsif logfile =~ /^STDERR$/i
|
195
|
-
$stderr
|
196
|
-
else
|
197
|
-
File.open(logfile,'ab+')
|
198
|
-
end
|
199
|
-
end
|
200
|
-
|
201
|
-
def add_log(log)
|
202
|
-
@queue[log.first] << log
|
203
|
-
@rcount += 1
|
204
|
-
end
|
205
|
-
|
206
|
-
def write_queue
|
207
|
-
@queue.each do |service,q|
|
208
|
-
last_sv = nil
|
209
|
-
last_m = nil
|
210
|
-
last_count = 0
|
211
|
-
next unless log = @logs[service]
|
212
|
-
lf = log.logfile
|
213
|
-
cull = log.cull
|
214
|
-
levels = log.levels
|
215
|
-
q.each do |m|
|
216
|
-
next unless levels.has_key?(m[1])
|
217
|
-
if cull
|
218
|
-
if m.last == last_m and m[0..1] == last_sv
|
219
|
-
last_count += 1
|
220
|
-
next
|
221
|
-
elsif last_count > 0
|
222
|
-
lf.write_nonblock "#{@now}|#{last_sv.join(C_bar)}|Last message repeated #{last_count} times\n"
|
223
|
-
last_sv = last_m = nil
|
224
|
-
last_count = 0
|
225
|
-
end
|
226
|
-
lf.write_nonblock "#{@now}|#{m.join(C_bar)}\n"
|
227
|
-
last_m = m.last
|
228
|
-
last_sv = m[0..1]
|
229
|
-
else
|
230
|
-
lf.write_nonblock "#{@now}|#{m.join(C_bar)}\n"
|
231
|
-
end
|
232
|
-
@wcount += 1
|
233
|
-
end
|
234
|
-
lf.write_nonblock "#{@now}|#{last_sv.join(C_bar)}|Last message repeated #{last_count} times\n" if cull and last_count > 0
|
235
|
-
end
|
236
|
-
@queue.each {|service,q| q.clear}
|
237
|
-
end
|
238
|
-
|
239
|
-
def flush_queue
|
240
|
-
@logs.each_value {|l| l.logfile.fsync if l.logfile.fileno > 2}
|
241
|
-
end
|
242
|
-
|
243
|
-
def key
|
244
|
-
@config[Ckey].to_s
|
245
|
-
end
|
246
|
-
|
247
|
-
end
|
248
|
-
|
249
|
-
end
|
250
|
-
|
251
|
-
class Log
|
252
|
-
attr_reader :service, :levels, :logfile, :cull, :roll, :roll_age, :roll_size
|
253
|
-
|
254
|
-
def initialize(spec)
|
255
|
-
@service = spec[Analogger::Cservice]
|
256
|
-
@levels = spec[Analogger::Clevels]
|
257
|
-
@logfile = spec[Analogger::Clogfile]
|
258
|
-
@cull = spec[Analogger::Ccull]
|
259
|
-
@roll = spec[Analogger::Croll]
|
260
|
-
@roll_inteval = spec[Analogger::Croll_age]
|
261
|
-
@roll_size = spec[Analogger::Croll_size]
|
262
|
-
end
|
263
|
-
|
264
|
-
def to_s
|
265
|
-
"service: #{@service}\nlevels: #{@levels.inspect}\nlogfile: #{@logfile}\ncull: #{@cull}\n"
|
266
|
-
end
|
267
|
-
end
|
268
|
-
|
269
|
-
class AnaloggerProtocol < EventMachine::Connection
|
270
|
-
Ci = 'i'.freeze
|
271
|
-
Rcolon = /:/
|
272
|
-
MaxMessageLength = 8192
|
273
|
-
|
274
|
-
LoggerClass = Analogger
|
275
|
-
|
276
|
-
def post_init
|
277
|
-
setup
|
278
|
-
end
|
279
|
-
|
280
|
-
end
|
281
|
-
end
|
282
|
-
|
283
|
-
case RUBY_VERSION
|
284
|
-
when /^1.8/
|
285
|
-
require 'swiftcore/Analogger/receive_data_18.rb'
|
286
|
-
when /^1.9/
|
287
|
-
require 'swiftcore/Analogger/receive_data_19.rb'
|
288
|
-
else
|
289
|
-
raise "We're sorry, but Analogger is not supported for ruby versions prior to 1.8.x (and is untested below 1.8.5)."
|
290
|
-
end
|
291
|
-
|