analogger 0.9.1 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +2 -2
- data/README +61 -15
- data/analogger.gemspec +1 -1
- data/bin/analogger +25 -27
- data/lib/swiftcore/Analogger.rb +68 -69
- data/lib/swiftcore/Analogger/AnaloggerProtocol.rb +4 -5
- data/lib/swiftcore/Analogger/Client.rb +10 -11
- data/lib/swiftcore/Analogger/version.rb +1 -1
- metadata +4 -5
- data/lib/swiftcore/Analogger.rb.orig +0 -291
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f3ec71320cfc00f16d6073e54ce2f9b3134f511b046365c99738c756a38ee00d
|
4
|
+
data.tar.gz: '086bc16bd8cfb9c971b6c73aa1ccfa6342474cc7a7e3d79d795b28a55dfc8746'
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 33afab0699d63cd5a0f78d70b4533900001a112a76ea9d5f585aa0a920127abffc2960436d4b2b3609a58044d3cbfe87081196634172c5a80fd3c9a74f531ee7
|
7
|
+
data.tar.gz: 368dd4f344260e13d885d6c6cc9b8d2b97561ae0e407b440c438cc1113d6dbc5d1c61306065711061cc0a035612a761b07d60d5e1b9d25f05fc1b7db30aa2f69
|
data/Gemfile.lock
CHANGED
data/README
CHANGED
@@ -1,19 +1,65 @@
|
|
1
|
-
|
1
|
+
# Analogger
|
2
2
|
|
3
|
-
|
4
|
-
Copyright:: (C) 2007 by Kirk Haines. All Rights Reserved.
|
5
|
-
Email:: wyhaines@gmail.com
|
6
|
-
License:: Ruby's License
|
3
|
+
## Overview
|
7
4
|
|
8
|
-
|
9
|
-
|
10
|
-
|
5
|
+
Analogger is a fast asynchronous logging service and client library. It is
|
6
|
+
implemented in Ruby, and currently uses EventMachine in the server, though
|
7
|
+
there is a plan on the roadmap to enable it to run with a pure Ruby event
|
8
|
+
reactor.
|
11
9
|
|
12
|
-
Analogger
|
13
|
-
|
14
|
-
|
15
|
-
|
10
|
+
Analogger was originally written over a decade ago, in response to a need to
|
11
|
+
maintain a central logging server to accumulate logs from numerous web
|
12
|
+
applications to a single location. It takes very little time to send a logging
|
13
|
+
message, making it a very low impact logger for performance sensitive
|
14
|
+
applications. It has been continuously used in production since then, albeit
|
15
|
+
in a version not released publicly.
|
16
16
|
|
17
|
-
|
18
|
-
|
19
|
-
|
17
|
+
## Usage
|
18
|
+
|
19
|
+
Analogger is configured through a YAML formatted file:
|
20
|
+
|
21
|
+
```yaml
|
22
|
+
host: mycompany-logger-1-nyc1.private
|
23
|
+
port: 6766
|
24
|
+
default_log: /var/log/analogger_default
|
25
|
+
daemonize: true
|
26
|
+
syncinterval: 5
|
27
|
+
logs:
|
28
|
+
- service:
|
29
|
+
- default
|
30
|
+
logfile: /var/log/analogger/default
|
31
|
+
cull: true
|
32
|
+
- service:
|
33
|
+
- project-development
|
34
|
+
logfile: /var/log/analogger/project-development.log
|
35
|
+
- service:
|
36
|
+
- project-production
|
37
|
+
logfile: /var/log/analogger/project-production.log
|
38
|
+
cull: true
|
39
|
+
```
|
40
|
+
|
41
|
+
### Configuration Variables
|
42
|
+
|
43
|
+
* port
|
44
|
+
|
45
|
+
The port to listen for connections on. 6766 is the default.
|
46
|
+
|
47
|
+
* host
|
48
|
+
|
49
|
+
The hostname or IP to bind to when listening for connections.
|
50
|
+
|
51
|
+
* default_log: /var/log/analogger_default
|
52
|
+
|
53
|
+
This is the file to send logs to which don't appear to match any named service in the configuration.
|
54
|
+
|
55
|
+
* daemonize
|
56
|
+
|
57
|
+
Whether or not to detach an analogger process as a daemon process. You normally want this to be true.
|
58
|
+
|
59
|
+
* syncinterval
|
60
|
+
|
61
|
+
Analogger will run a thread every X seconds to ensure that any currently buffered log contents are synchronized to disk. Analogger tries to write any buffered logs before it exits if it receives a signal which would cause the process to die. However, in the event that this is not possible, only the logs received since the last sync interval would be at risk.
|
62
|
+
|
63
|
+
* logs
|
64
|
+
|
65
|
+
This is a list of defined logging services. Each consists of a service label, a logging destination (the path to the log file for that service), and optionally a `cull` attribute which, if true, causes analogger to deduplicate logs, eliminating consecutive repeats of the same message and instead emitting a summary of how many records like the one above the summary were culled.
|
data/analogger.gemspec
CHANGED
@@ -38,6 +38,6 @@ Gem::Specification.new do |spec|
|
|
38
38
|
spec.add_development_dependency "bundler", "~> 1.10"
|
39
39
|
spec.add_development_dependency "rake", "~> 11.0"
|
40
40
|
spec.add_development_dependency "minitest", "~> 5"
|
41
|
-
spec.add_runtime_dependency "eventmachine", "~> 1.2"
|
41
|
+
spec.add_runtime_dependency "eventmachine", "~> 1.2.5"
|
42
42
|
end
|
43
43
|
|
data/bin/analogger
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
|
1
|
+
#!/usr/bin/env ruby
|
2
2
|
|
3
3
|
require 'yaml'
|
4
4
|
require 'optparse'
|
@@ -47,39 +47,39 @@ module Swiftcore
|
|
47
47
|
# cull: true
|
48
48
|
#
|
49
49
|
OptionParser.new do |opts|
|
50
|
-
opts.banner = "Analogger v#{Swiftcore::Analogger::VERSION}\nUsage: analogger.rb [options]"
|
51
|
-
opts.separator
|
52
|
-
opts.on(
|
50
|
+
opts.banner = -"Analogger v#{Swiftcore::Analogger::VERSION}\nUsage: analogger.rb [options]"
|
51
|
+
opts.separator -""
|
52
|
+
opts.on(-"-c", -"--config CONFFILE", "The configuration file to read.") do |conf|
|
53
53
|
config = YAML.load(File.read(conf))
|
54
54
|
end
|
55
|
-
opts.on(
|
56
|
-
config[
|
55
|
+
opts.on(-"-p", -"--port [PORT]", Integer, "The port to receive connections on.") do |port|
|
56
|
+
config[-"port"] = port
|
57
57
|
end
|
58
|
-
opts.on(
|
59
|
-
config[
|
58
|
+
opts.on(-"-h", -"--host [HOST]", String, "The host to bind the connection to.") do |host|
|
59
|
+
config[-"host"] = host
|
60
60
|
end
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
opts.on(
|
65
|
-
config[
|
61
|
+
opts.on(-"-r",-"--controlkey [KEY]",String,-"The secret key that authenticates a control session.") do |secret|
|
62
|
+
config[-"secret"] = secret
|
63
|
+
end
|
64
|
+
opts.on(-"-k", -"--key [KEY]", String, -"The secret key that authenticates a valid client session.") do |secret|
|
65
|
+
config[-"key"] = secret
|
66
66
|
end
|
67
|
-
opts.on(
|
68
|
-
config[
|
67
|
+
opts.on(-"-i", -"--interval [INTERVAL]", Integer, -"The interval between queue writes. Defaults to 1 second.") do |interval|
|
68
|
+
config[-"interval"] = interval
|
69
69
|
end
|
70
|
-
opts.on(
|
71
|
-
config[
|
70
|
+
opts.on(-"-s", -"--syncinterval [INTERVAL]", Integer, -"The interval between queue syncs. Defaults to 60 seconds.") do |interval|
|
71
|
+
config[-"syncinterval"] = interval
|
72
72
|
end
|
73
|
-
opts.on(
|
74
|
-
config[
|
73
|
+
opts.on(-"-d", -"--default [PATH]", String, -"The default log destination. Defaults to stdout.") do |default|
|
74
|
+
config[-"default_log"] = default
|
75
75
|
end
|
76
|
-
opts.on(
|
77
|
-
config[
|
76
|
+
opts.on(-"-x", -"--daemonize", -"Tell the Analogger to daemonize itself.") do
|
77
|
+
config[-"daemonize"] = true
|
78
78
|
end
|
79
|
-
opts.on(
|
80
|
-
config[
|
79
|
+
opts.on(-"-w", -"--writepid [FILENAME]", -"The filename to write a PID file to.") do |pidfile|
|
80
|
+
config[-"pidfile"] = pidfile || 'analogger.pid'
|
81
81
|
end
|
82
|
-
opts.on(
|
82
|
+
opts.on(-"-v", -"--version", -"Show the current version of Analogger.") do
|
83
83
|
puts "Analogger v#{Swiftcore::Analogger::VERSION}"
|
84
84
|
exit
|
85
85
|
end
|
@@ -95,6 +95,4 @@ module Swiftcore
|
|
95
95
|
end
|
96
96
|
end
|
97
97
|
|
98
|
-
|
99
|
-
catch(:hup) {Swiftcore::AnaloggerExec.run}
|
100
|
-
end
|
98
|
+
Swiftcore::AnaloggerExec.run
|
data/lib/swiftcore/Analogger.rb
CHANGED
@@ -6,25 +6,9 @@ require 'swiftcore/Analogger/AnaloggerProtocol'
|
|
6
6
|
|
7
7
|
module Swiftcore
|
8
8
|
class Analogger
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
Cdaemonize = 'daemonize'.freeze
|
13
|
-
Cdefault = 'default'.freeze
|
14
|
-
Cdefault_log = 'default_log'.freeze
|
15
|
-
Chost = 'host'.freeze
|
16
|
-
Cinterval = 'interval'.freeze
|
17
|
-
Ckey = 'key'.freeze
|
18
|
-
Clogfile = 'logfile'.freeze
|
19
|
-
Clogs = 'logs'.freeze
|
20
|
-
Cport = 'port'.freeze
|
21
|
-
Csecret = 'secret'.freeze
|
22
|
-
Cservice = 'service'.freeze
|
23
|
-
Clevels = 'levels'.freeze
|
24
|
-
Csyncinterval = 'syncinterval'.freeze
|
25
|
-
Cpidfile = 'pidfile'.freeze
|
26
|
-
DefaultSeverityLevels = ['debug','info','warn','error','fatal'].inject({}){|h,k|h[k]=true;h}
|
27
|
-
TimeFormat = '%Y/%m/%d %H:%M:%S'.freeze
|
9
|
+
EXEC_ARGUMENTS = [File.expand_path(Process.argv0), *ARGV]
|
10
|
+
|
11
|
+
DefaultSeverityLevels = [-"debug",-"info",-"warn",-"error",-"fatal"].inject({}){|h,k|h[k]=true;h}
|
28
12
|
|
29
13
|
class NoPortProvided < Exception; def to_s; "The port to bind to was not provided."; end; end
|
30
14
|
class BadPort < Exception
|
@@ -37,6 +21,7 @@ module Swiftcore
|
|
37
21
|
|
38
22
|
EXIT_SIGNALS = %w[INT TERM]
|
39
23
|
RELOAD_SIGNALS = %w[HUP]
|
24
|
+
RESTART_SIGNALS = %w[USR2]
|
40
25
|
|
41
26
|
class << self
|
42
27
|
def safe_trap(siglist, &operation)
|
@@ -45,8 +30,8 @@ module Swiftcore
|
|
45
30
|
|
46
31
|
def start(config,protocol = AnaloggerProtocol)
|
47
32
|
@config = config
|
48
|
-
daemonize if @config[
|
49
|
-
File.open(@config[
|
33
|
+
daemonize if @config[-"daemonize"]
|
34
|
+
File.open(@config[-"pidfile"],-"w+") {|fh| fh.puts $$} if @config[-"pidfile"]
|
50
35
|
@logs = Hash.new {|h,k| h[k] = new_log(k)}
|
51
36
|
@queue = Hash.new {|h,k| h[k] = []}
|
52
37
|
postprocess_config_load
|
@@ -57,8 +42,8 @@ module Swiftcore
|
|
57
42
|
@wcount = 0
|
58
43
|
@server = nil
|
59
44
|
safe_trap(EXIT_SIGNALS) {handle_pending_and_exit}
|
60
|
-
safe_trap(RELOAD_SIGNALS) {
|
61
|
-
|
45
|
+
safe_trap(RELOAD_SIGNALS) {cleanup_and_reopen}
|
46
|
+
safe_trap(RESTART_SIGNALS) {exec(*EXEC_ARGUMENTS)}
|
62
47
|
|
63
48
|
#####
|
64
49
|
# This is gross. EM needs to change so that it defaults to the faster
|
@@ -80,17 +65,17 @@ module Swiftcore
|
|
80
65
|
flush_queue
|
81
66
|
cleanup
|
82
67
|
end
|
83
|
-
@server = EventMachine.start_server @config[
|
68
|
+
@server = EventMachine.start_server @config[-"host"], @config[-"port"], protocol
|
84
69
|
EventMachine.add_periodic_timer(1) {Analogger.update_now}
|
85
|
-
EventMachine.add_periodic_timer(@config[
|
86
|
-
EventMachine.add_periodic_timer(@config[
|
70
|
+
EventMachine.add_periodic_timer(@config[-"interval"]) {write_queue}
|
71
|
+
EventMachine.add_periodic_timer(@config[-"syncinterval"]) {flush_queue}
|
87
72
|
}
|
88
73
|
exit
|
89
74
|
end
|
90
75
|
|
91
76
|
def daemonize
|
92
77
|
if (child_pid = fork)
|
93
|
-
puts "PID #{child_pid}" unless @config[
|
78
|
+
puts "PID #{child_pid}" unless @config[-"pidfile"]
|
94
79
|
exit!
|
95
80
|
end
|
96
81
|
Process.setsid
|
@@ -101,8 +86,8 @@ module Swiftcore
|
|
101
86
|
puts "Platform (#{RUBY_PLATFORM}) does not appear to support fork/setsid; skipping"
|
102
87
|
end
|
103
88
|
|
104
|
-
def new_log(facility =
|
105
|
-
Log.new({
|
89
|
+
def new_log(facility = -"default", levels = @config[-"levels"] || DefaultSeverityLevels, log = @config[-"default_log"], cull = true)
|
90
|
+
Log.new({-"service" => facility, -"levels" => levels, -"logfile" => log, -"cull" => cull})
|
106
91
|
end
|
107
92
|
|
108
93
|
# Before exiting, try to get any logs that are still in memory handled and written to disk.
|
@@ -129,8 +114,15 @@ module Swiftcore
|
|
129
114
|
end
|
130
115
|
end
|
131
116
|
|
117
|
+
def cleanup_and_reopen
|
118
|
+
@logs.each do |service,l|
|
119
|
+
l.logfile.fsync if !l.logfile.closed? and l.logfile.fileno > 2
|
120
|
+
l.logfile.reopen(l.logfile.path, -"ab+") if l.logfile.fileno > 2
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
132
124
|
def update_now
|
133
|
-
@now = Time.now.strftime(
|
125
|
+
@now = Time.now.strftime(-"%Y/%m/%d %H:%M:%S")
|
134
126
|
end
|
135
127
|
|
136
128
|
def config
|
@@ -142,26 +134,26 @@ module Swiftcore
|
|
142
134
|
end
|
143
135
|
|
144
136
|
def populate_logs
|
145
|
-
@config[
|
146
|
-
next unless log[
|
147
|
-
if Array === log[
|
148
|
-
log[
|
149
|
-
@logs[loglog] = new_log(loglog,log[
|
137
|
+
@config[-"logs"].each do |log|
|
138
|
+
next unless log[-"service"]
|
139
|
+
if Array === log[-"service"]
|
140
|
+
log[-"service"].each do |loglog|
|
141
|
+
@logs[loglog] = new_log(loglog,log[-"levels"],logfile_destination(log[-"logfile"]),log[-"cull"])
|
150
142
|
end
|
151
143
|
else
|
152
|
-
@logs[log[
|
144
|
+
@logs[log[-"service"]] = new_log(log[-"service"],log[-"levels"],logfile_destination(log[-"logfile"]),log[-"cull"])
|
153
145
|
end
|
154
146
|
end
|
155
147
|
end
|
156
148
|
|
157
149
|
def postprocess_config_load
|
158
|
-
@config[
|
159
|
-
if @config[
|
160
|
-
@config[
|
150
|
+
@config[-"logs"] ||= []
|
151
|
+
if @config[-"levels"]
|
152
|
+
@config[-"levels"] = normalize_levels(@config[-"levels"])
|
161
153
|
end
|
162
154
|
|
163
|
-
@config[
|
164
|
-
log[
|
155
|
+
@config[-"logs"].each do |log|
|
156
|
+
log[-"levels"] = normalize_levels(log[-"levels"])
|
165
157
|
end
|
166
158
|
end
|
167
159
|
|
@@ -180,18 +172,18 @@ module Swiftcore
|
|
180
172
|
end
|
181
173
|
|
182
174
|
def check_config_settings
|
183
|
-
raise NoPortProvided unless @config[
|
184
|
-
raise BadPort.new(@config[
|
175
|
+
raise NoPortProvided unless @config[-"port"]
|
176
|
+
raise BadPort.new(@config[-"port"]) unless @config[-"port"].to_i > 0
|
185
177
|
end
|
186
178
|
|
187
179
|
def set_config_defaults
|
188
|
-
@config[
|
189
|
-
@config[
|
190
|
-
@config[
|
191
|
-
@config[
|
192
|
-
@config[
|
193
|
-
@config[
|
194
|
-
@logs[
|
180
|
+
@config[-"host"] ||= -"127.0.0.1"
|
181
|
+
@config[-"interval"] ||= 1
|
182
|
+
@config[-"syncinterval"] ||= 60
|
183
|
+
@config[-"syncinterval"] = nil if @config[-"syncinterval"] == 0
|
184
|
+
@config[-"default_log"] = @config[-"default_log"].nil? || @config[-"default_log"] == -"-" ? -"STDOUT" : @config[-"default_log"]
|
185
|
+
@config[-"default_log"] = logfile_destination(@config[-"default_log"])
|
186
|
+
@logs[-"default"] = new_log
|
195
187
|
end
|
196
188
|
|
197
189
|
def logfile_destination(logfile)
|
@@ -199,7 +191,7 @@ module Swiftcore
|
|
199
191
|
if logfile.is_a?(IO)
|
200
192
|
return $stdout if logfile == $stdout
|
201
193
|
return $stderr if logfile == $stderr
|
202
|
-
return logfile.reopen(logfile.path,
|
194
|
+
return logfile.reopen(logfile.path, -"ab+")
|
203
195
|
end
|
204
196
|
|
205
197
|
if logfile =~ /^STDOUT$/i
|
@@ -207,7 +199,7 @@ module Swiftcore
|
|
207
199
|
elsif logfile =~ /^STDERR$/i
|
208
200
|
$stderr
|
209
201
|
else
|
210
|
-
File.open(logfile,
|
202
|
+
File.open(logfile, -"ab+")
|
211
203
|
end
|
212
204
|
end
|
213
205
|
|
@@ -243,25 +235,26 @@ module Swiftcore
|
|
243
235
|
last_count += 1
|
244
236
|
next
|
245
237
|
elsif last_count > 0
|
246
|
-
lf.write_nonblock "#{@now}|#{last_sv.join(
|
238
|
+
lf.write_nonblock "#{@now}|#{last_sv.join(-"|")}|Last message repeated #{last_count} times\n"
|
247
239
|
last_sv = last_m = nil
|
248
240
|
last_count = 0
|
249
241
|
end
|
250
|
-
lf.write_nonblock "#{@now}|#{m.join(
|
242
|
+
lf.write_nonblock "#{@now}|#{m.join(-"|")}\n"
|
251
243
|
last_m = m.last
|
252
244
|
last_sv = m[0..1]
|
253
245
|
else
|
254
|
-
lf.write_nonblock "#{@now}|#{m.join(
|
246
|
+
lf.write_nonblock "#{@now}|#{m.join(-"|")}\n"
|
255
247
|
end
|
256
248
|
@wcount += 1
|
257
249
|
end
|
258
|
-
lf.write_nonblock "#{@now}|#{last_sv.join(
|
250
|
+
lf.write_nonblock "#{@now}|#{last_sv.join(-"|")}|Last message repeated #{last_count} times\n" if cull and last_count > 0
|
259
251
|
end
|
260
252
|
@queue.each {|service,q| q.clear}
|
261
253
|
end
|
262
254
|
|
263
255
|
def flush_queue
|
264
256
|
@logs.each_value do |l|
|
257
|
+
#if !l.logfile.closed? and l.logfile.fileno > 2
|
265
258
|
if l.logfile.fileno > 2
|
266
259
|
l.logfile.fdatasync rescue l.logfile.fsync
|
267
260
|
end
|
@@ -269,30 +262,36 @@ module Swiftcore
|
|
269
262
|
end
|
270
263
|
|
271
264
|
def key
|
272
|
-
@config[
|
265
|
+
@config[-"key"].to_s
|
273
266
|
end
|
274
267
|
|
275
268
|
end
|
276
269
|
|
277
|
-
|
270
|
+
class Log
|
271
|
+
attr_reader :service, :levels, :logfile, :cull
|
278
272
|
|
279
|
-
|
280
|
-
|
273
|
+
def initialize(spec)
|
274
|
+
@service = spec[-"service"]
|
275
|
+
@levels = spec[-"levels"]
|
276
|
+
@logfile = spec[-"logfile"]
|
277
|
+
@cull = spec[-"cull"]
|
278
|
+
end
|
281
279
|
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
280
|
+
def to_s
|
281
|
+
"service: #{@service}\nlevels: #{@levels.inspect}\nlogfile: #{@logfile}\ncull: #{@cull}\n"
|
282
|
+
end
|
283
|
+
|
284
|
+
def ==(n)
|
285
|
+
n.service == @service &&
|
286
|
+
n.levels == @levels &&
|
287
|
+
n.logfile == @logfile &&
|
288
|
+
n.cull == @cull
|
289
|
+
end
|
288
290
|
|
289
|
-
def to_s
|
290
|
-
"service: #{@service}\nlevels: #{@levels.inspect}\nlogfile: #{@logfile}\ncull: #{@cull}\n"
|
291
291
|
end
|
292
292
|
end
|
293
293
|
|
294
294
|
class AnaloggerProtocol < EventMachine::Connection
|
295
|
-
Ci = 'i'.freeze
|
296
295
|
Rcolon = /:/
|
297
296
|
|
298
297
|
LoggerClass = Analogger
|
@@ -3,7 +3,6 @@ module Swiftcore
|
|
3
3
|
|
4
4
|
MaxMessageLength = 8192
|
5
5
|
MaxLengthBytes = MaxMessageLength.to_s.length
|
6
|
-
Semaphore = "||"
|
7
6
|
|
8
7
|
def setup
|
9
8
|
@length = nil
|
@@ -32,11 +31,11 @@ module Swiftcore
|
|
32
31
|
peer = peer ? ::Socket.unpack_sockaddr_in(peer)[1] : 'UNK'
|
33
32
|
if l == ck
|
34
33
|
LoggerClass.add_log([:default, :error, "Max Length Exceeded from #{peer} -- #{l}/#{MaxMessageLength}"])
|
35
|
-
send_data
|
34
|
+
send_data(-"error: max length exceeded\n")
|
36
35
|
close_connection_after_writing
|
37
36
|
else
|
38
37
|
LoggerClass.add_log([:default, :error, "checksum failed from #{peer} -- #{l}/#{ck}"])
|
39
|
-
send_data
|
38
|
+
send_data(-"error: checksum failed\n")
|
40
39
|
close_connection_after_writing
|
41
40
|
end
|
42
41
|
end
|
@@ -54,9 +53,9 @@ module Swiftcore
|
|
54
53
|
unless @authenticated
|
55
54
|
if msg.last == LoggerClass.key
|
56
55
|
@authenticated = true
|
57
|
-
send_data
|
56
|
+
send_data(-"accepted\n")
|
58
57
|
else
|
59
|
-
send_data
|
58
|
+
send_data(-"denied\n")
|
60
59
|
close_connection_after_writing
|
61
60
|
end
|
62
61
|
else
|
@@ -43,17 +43,13 @@ module Swiftcore
|
|
43
43
|
class Client
|
44
44
|
|
45
45
|
class FailedToAuthenticate < StandardError
|
46
|
-
def initialize(hots = "UNK", port = 6766)
|
46
|
+
def initialize(hots = -"UNK", port = 6766)
|
47
47
|
super("Failed to authenticate to the Analogger server at #{destination}:#{port}")
|
48
48
|
end
|
49
49
|
end
|
50
50
|
|
51
|
-
Cauthentication = 'authentication'.freeze
|
52
|
-
Ci = 'i'.freeze
|
53
|
-
|
54
51
|
MaxMessageLength = 8192
|
55
52
|
MaxLengthBytes = MaxMessageLength.to_s.length
|
56
|
-
Semaphore = "||"
|
57
53
|
ConnectionFailureTimeout = 86400 * 2 # Log locally for a long time if Analogger server goes down.
|
58
54
|
MaxFailureCount = (2**(0.size * 8 - 2) - 1) # Max integer -- i.e. really big
|
59
55
|
PersistentQueueLimit = 10737412742 # Default to allowing around 10GB temporary local log storage
|
@@ -115,15 +111,18 @@ module Swiftcore
|
|
115
111
|
|
116
112
|
#-----
|
117
113
|
|
118
|
-
def initialize(service =
|
114
|
+
def initialize(service = -"default", host = -"127.0.0.1" , port = 6766, key = nil)
|
119
115
|
@service = service.to_s
|
120
116
|
@key = key
|
121
117
|
@host = host
|
122
118
|
@port = port
|
119
|
+
@socket = nil
|
123
120
|
klass = self.class
|
124
121
|
@connection_failure_timeout = klass.connection_failure_timeout
|
125
122
|
@max_failure_count = klass.max_failure_count
|
126
123
|
@persistent_queue_limit = klass.persistent_queue_limit
|
124
|
+
@destination = nil
|
125
|
+
@reconnection_thread = nil
|
127
126
|
@authenticated = false
|
128
127
|
@total_count = 0
|
129
128
|
@logfile = nil
|
@@ -173,7 +172,7 @@ module Swiftcore
|
|
173
172
|
end
|
174
173
|
|
175
174
|
def tmplog_prefix
|
176
|
-
File.join(Dir.tmpdir, "analogger-SERVICE-PID.log")
|
175
|
+
File.join(Dir.tmpdir, -"analogger-SERVICE-PID.log")
|
177
176
|
end
|
178
177
|
|
179
178
|
def tmplog
|
@@ -181,7 +180,7 @@ module Swiftcore
|
|
181
180
|
end
|
182
181
|
|
183
182
|
def tmplogs
|
184
|
-
Dir[tmplog_prefix.gsub(/SERVICE/, @service).gsub(/PID
|
183
|
+
Dir[tmplog_prefix.gsub(/SERVICE/, @service).gsub(/PID/,-"*")].sort_by {|f| File.mtime(f)}
|
185
184
|
end
|
186
185
|
|
187
186
|
def tmplog=(val)
|
@@ -222,7 +221,7 @@ module Swiftcore
|
|
222
221
|
|
223
222
|
def setup_local_logging
|
224
223
|
unless @logfile && !@logfile.closed?
|
225
|
-
@logfile = File.open(tmplog
|
224
|
+
@logfile = File.open(tmplog,-"a+")
|
226
225
|
@destination = :local
|
227
226
|
end
|
228
227
|
end
|
@@ -297,7 +296,7 @@ module Swiftcore
|
|
297
296
|
|
298
297
|
def authenticate
|
299
298
|
begin
|
300
|
-
_remote_log(@service,
|
299
|
+
_remote_log(@service, -"authentication", "#{@key}")
|
301
300
|
response = @socket.gets
|
302
301
|
rescue Exception
|
303
302
|
response = nil
|
@@ -354,7 +353,7 @@ module Swiftcore
|
|
354
353
|
records.each_index do |n|
|
355
354
|
record = records[n]
|
356
355
|
next if record =~ /^\#/
|
357
|
-
service, severity, msg = record.split(":", 3)
|
356
|
+
service, severity, msg = record.split(-":", 3)
|
358
357
|
msg = msg.chomp.gsub(/\x00\x00/, "\n")
|
359
358
|
begin
|
360
359
|
_remote_log(service, severity, msg)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: analogger
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.9.
|
4
|
+
version: 0.9.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kirk Haines
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-03-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -58,14 +58,14 @@ dependencies:
|
|
58
58
|
requirements:
|
59
59
|
- - "~>"
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version:
|
61
|
+
version: 1.2.5
|
62
62
|
type: :runtime
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
66
|
- - "~>"
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version:
|
68
|
+
version: 1.2.5
|
69
69
|
description: Analogger provides a fast and very stable asynchronous central logging
|
70
70
|
service capable of handling heavy logging loads. It has been in production use for
|
71
71
|
almost a decade.
|
@@ -86,7 +86,6 @@ files:
|
|
86
86
|
- external/package.rb
|
87
87
|
- external/test_support.rb
|
88
88
|
- lib/swiftcore/Analogger.rb
|
89
|
-
- lib/swiftcore/Analogger.rb.orig
|
90
89
|
- lib/swiftcore/Analogger/AnaloggerProtocol.rb
|
91
90
|
- lib/swiftcore/Analogger/Client.rb
|
92
91
|
- lib/swiftcore/Analogger/EMClient.rb
|
@@ -1,291 +0,0 @@
|
|
1
|
-
require 'socket'
|
2
|
-
begin
|
3
|
-
load_attempted ||= false
|
4
|
-
require 'eventmachine'
|
5
|
-
require 'benchmark'
|
6
|
-
rescue LoadError => e
|
7
|
-
unless load_attempted
|
8
|
-
load_attempted = true
|
9
|
-
require 'rubygems'
|
10
|
-
retry
|
11
|
-
end
|
12
|
-
raise e
|
13
|
-
end
|
14
|
-
|
15
|
-
module Swiftcore
|
16
|
-
class Analogger
|
17
|
-
C_colon = ':'.freeze
|
18
|
-
C_bar = '|'.freeze
|
19
|
-
Ccull = 'cull'.freeze
|
20
|
-
Cdaemonize = 'daemonize'.freeze
|
21
|
-
Cdefault = 'default'.freeze
|
22
|
-
Cdefault_log = 'default_log'.freeze
|
23
|
-
Cepoll = 'epoll'.freeze
|
24
|
-
Chost = 'host'.freeze
|
25
|
-
Cinterval = 'interval'.freeze
|
26
|
-
Ckey = 'key'.freeze
|
27
|
-
Ckqueue = 'kqueue'.freeze
|
28
|
-
Clevels = 'levels'.freeze
|
29
|
-
Clogfile = 'logfile'.freeze
|
30
|
-
Clogs = 'logs'.freeze
|
31
|
-
Cpidfile = 'pidfile'.freeze
|
32
|
-
Cport = 'port'.freeze
|
33
|
-
Croll = 'roll'.freeze
|
34
|
-
Croll_age = 'roll_age'.freeze
|
35
|
-
Croll_size = 'roll_size'.freeze
|
36
|
-
Croll_interval = 'roll_interval'.freeze
|
37
|
-
Csecret = 'secret'.freeze
|
38
|
-
Cservice = 'service'.freeze
|
39
|
-
Csyncinterval = 'syncinterval'.freeze
|
40
|
-
DefaultSeverityLevels = ['debug','info','warn','error','fatal'].inject({}){|h,k|h[k]=true;h}
|
41
|
-
TimeFormat = '%Y/%m/%d %H:%M:%S'.freeze
|
42
|
-
|
43
|
-
class NoPortProvided < Exception; def to_s; "The port to bind to was not provided."; end; end
|
44
|
-
class BadPort < Exception
|
45
|
-
def initialize(port)
|
46
|
-
@port = port
|
47
|
-
end
|
48
|
-
|
49
|
-
def to_s; "The port provided (#{@port}) is invalid."; end
|
50
|
-
end
|
51
|
-
|
52
|
-
EXIT_SIGNALS = %w[INT TERM]
|
53
|
-
RELOAD_SIGNALS = %w[HUP]
|
54
|
-
|
55
|
-
class << self
|
56
|
-
def safe_trap(siglist, &operation)
|
57
|
-
(Signal.list.keys & siglist).each {|sig| trap(sig, &operation)}
|
58
|
-
end
|
59
|
-
|
60
|
-
def start(config,protocol = AnaloggerProtocol)
|
61
|
-
@config = config
|
62
|
-
daemonize if @config[Cdaemonize]
|
63
|
-
File.open(@config[Cpidfile],'w+') {|fh| fh.puts $$} if @config[Cpidfile]
|
64
|
-
@logs = Hash.new {|h,k| h[k] = new_log(k)}
|
65
|
-
@queue = Hash.new {|h,k| h[k] = []}
|
66
|
-
postprocess_config_load
|
67
|
-
check_config_settings
|
68
|
-
populate_logs
|
69
|
-
set_config_defaults
|
70
|
-
@rcount = 0
|
71
|
-
@wcount = 0
|
72
|
-
safe_trap(EXIT_SIGNALS) {cleanup;exit}
|
73
|
-
safe_trap(RELOAD_SIGNALS) {cleanup;throw :hup}
|
74
|
-
|
75
|
-
if @config[Cepoll] or @config[Ckqueue]
|
76
|
-
EventMachine.epoll if @config[Cepoll]
|
77
|
-
EventMachine.kqueue if @config[Ckqueue]
|
78
|
-
|
79
|
-
EventMachine.set_descriptor_table_size(4096)
|
80
|
-
end
|
81
|
-
|
82
|
-
EventMachine.run {
|
83
|
-
EventMachine.start_server @config[Chost], @config[Cport], protocol
|
84
|
-
EventMachine.add_periodic_timer(1) {Analogger.update_now}
|
85
|
-
EventMachine.add_periodic_timer(@config[Cinterval]) {write_queue}
|
86
|
-
EventMachine.add_periodic_timer(@config[Csyncinterval]) {flush_queue}
|
87
|
-
EventMachine.add_periodic_timer(@config[Crollinterval]) {roll logs}
|
88
|
-
}
|
89
|
-
end
|
90
|
-
|
91
|
-
def daemonize
|
92
|
-
if (child_pid = fork)
|
93
|
-
puts "PID #{child_pid}" unless @config[Cpidfile]
|
94
|
-
exit!
|
95
|
-
end
|
96
|
-
Process.setsid
|
97
|
-
|
98
|
-
rescue Exception
|
99
|
-
puts "Platform (#{RUBY_PLATFORM}) does not appear to support fork/setsid; skipping"
|
100
|
-
end
|
101
|
-
|
102
|
-
def new_log(facility = Cdefault, levels = @config[Clevels] || DefaultSeverityLevels, log = @config[Cdefault_log], cull = true, roll = @config[Croll], roll_age = @config[Croll_age], roll_size = @config[Croll_size])
|
103
|
-
Log.new({Cservice => facility, Clevels => levels, Clogfile => log, Ccull => cull, Croll => roll, Croll_age => roll_age, Croll_size => roll_size})
|
104
|
-
end
|
105
|
-
|
106
|
-
def cleanup
|
107
|
-
@logs.each do |service,l|
|
108
|
-
l.logfile.fsync if !l.logfile.closed? and l.logfile.fileno > 2
|
109
|
-
l.logfile.close unless l.logfile.closed? or l.logfile.fileno < 3
|
110
|
-
end
|
111
|
-
end
|
112
|
-
|
113
|
-
def roll_logs
|
114
|
-
@logs.each do |service,l|
|
115
|
-
end
|
116
|
-
end
|
117
|
-
|
118
|
-
def update_now
|
119
|
-
@now = Time.now.strftime(TimeFormat)
|
120
|
-
end
|
121
|
-
|
122
|
-
def config
|
123
|
-
@config
|
124
|
-
end
|
125
|
-
|
126
|
-
def config=(conf)
|
127
|
-
@config = conf
|
128
|
-
end
|
129
|
-
|
130
|
-
def populate_logs
|
131
|
-
@config[Clogs].each do |log|
|
132
|
-
next unless log[Cservice]
|
133
|
-
roll = log[Croll] || log[Croll_age] || log[Croll_size] ? true : false
|
134
|
-
if Array === log[Cservice]
|
135
|
-
log[Cservice].each do |loglog|
|
136
|
-
@logs[loglog] = new_log(loglog,log[Clevels],logfile_destination(log[Clogfile]),log[Ccull],roll,log[Croll_age],log[Croll_size])
|
137
|
-
end
|
138
|
-
else
|
139
|
-
@logs[log[Cservice]] = new_log(log[Cservice],log[Clevels],logfile_destination(log[Clogfile]),log[Ccull],roll,log[Croll_age],log[Croll_size])
|
140
|
-
end
|
141
|
-
end
|
142
|
-
end
|
143
|
-
|
144
|
-
def postprocess_config_load
|
145
|
-
@config[Clogs] ||= []
|
146
|
-
if @config[Clevels]
|
147
|
-
@config[Clevels] = normalize_levels(@config[Clevels])
|
148
|
-
end
|
149
|
-
|
150
|
-
@config[Clogs].each do |log|
|
151
|
-
log[Clevels] = normalize_levels(log[Clevels])
|
152
|
-
end
|
153
|
-
end
|
154
|
-
|
155
|
-
def normalize_levels(levels)
|
156
|
-
if String === levels and levels =~ /,/
|
157
|
-
levels.split(/,/).inject({}) {|h,k| h[k.to_s] = true; h}
|
158
|
-
elsif Array === levels
|
159
|
-
levels.inject({}) {|h,k| h[k.to_s] = true; h}
|
160
|
-
elsif levels.nil?
|
161
|
-
DefaultSeverityLevels
|
162
|
-
elsif !(Hash === levels)
|
163
|
-
[levels.to_s => true]
|
164
|
-
else
|
165
|
-
levels
|
166
|
-
end
|
167
|
-
end
|
168
|
-
|
169
|
-
def check_config_settings
|
170
|
-
raise NoPortProvided unless @config[Cport]
|
171
|
-
raise BadPort.new(@config[Cport]) unless @config[Cport].to_i > 0
|
172
|
-
end
|
173
|
-
|
174
|
-
def set_config_defaults
|
175
|
-
@config[Chost] ||= '127.0.0.1'
|
176
|
-
@config[Cinterval] ||= 1
|
177
|
-
@config[Csyncinterval] ||= 60
|
178
|
-
@config[Csyncinterval] = nil if @config[Csyncinterval] == 0
|
179
|
-
@config[Cdefault_log] = @config[Cdefault_log].nil? || @config[Cdefault_log] == '-' ? 'STDOUT' : @config[Cdefault_log]
|
180
|
-
@config[Cdefault_log] = logfile_destination(@config[Cdefault_log])
|
181
|
-
@logs['default'] = new_log
|
182
|
-
end
|
183
|
-
|
184
|
-
def logfile_destination(logfile)
|
185
|
-
# We're reloading if it's already an IO.
|
186
|
-
if logfile.is_a?(IO)
|
187
|
-
return $stdout if logfile == $stdout
|
188
|
-
return $stderr if logfile == $stderr
|
189
|
-
return logfile.reopen(logfile.path, 'ab+')
|
190
|
-
end
|
191
|
-
|
192
|
-
if logfile =~ /^STDOUT$/i
|
193
|
-
$stdout
|
194
|
-
elsif logfile =~ /^STDERR$/i
|
195
|
-
$stderr
|
196
|
-
else
|
197
|
-
File.open(logfile,'ab+')
|
198
|
-
end
|
199
|
-
end
|
200
|
-
|
201
|
-
def add_log(log)
|
202
|
-
@queue[log.first] << log
|
203
|
-
@rcount += 1
|
204
|
-
end
|
205
|
-
|
206
|
-
def write_queue
|
207
|
-
@queue.each do |service,q|
|
208
|
-
last_sv = nil
|
209
|
-
last_m = nil
|
210
|
-
last_count = 0
|
211
|
-
next unless log = @logs[service]
|
212
|
-
lf = log.logfile
|
213
|
-
cull = log.cull
|
214
|
-
levels = log.levels
|
215
|
-
q.each do |m|
|
216
|
-
next unless levels.has_key?(m[1])
|
217
|
-
if cull
|
218
|
-
if m.last == last_m and m[0..1] == last_sv
|
219
|
-
last_count += 1
|
220
|
-
next
|
221
|
-
elsif last_count > 0
|
222
|
-
lf.write_nonblock "#{@now}|#{last_sv.join(C_bar)}|Last message repeated #{last_count} times\n"
|
223
|
-
last_sv = last_m = nil
|
224
|
-
last_count = 0
|
225
|
-
end
|
226
|
-
lf.write_nonblock "#{@now}|#{m.join(C_bar)}\n"
|
227
|
-
last_m = m.last
|
228
|
-
last_sv = m[0..1]
|
229
|
-
else
|
230
|
-
lf.write_nonblock "#{@now}|#{m.join(C_bar)}\n"
|
231
|
-
end
|
232
|
-
@wcount += 1
|
233
|
-
end
|
234
|
-
lf.write_nonblock "#{@now}|#{last_sv.join(C_bar)}|Last message repeated #{last_count} times\n" if cull and last_count > 0
|
235
|
-
end
|
236
|
-
@queue.each {|service,q| q.clear}
|
237
|
-
end
|
238
|
-
|
239
|
-
def flush_queue
|
240
|
-
@logs.each_value {|l| l.logfile.fsync if l.logfile.fileno > 2}
|
241
|
-
end
|
242
|
-
|
243
|
-
def key
|
244
|
-
@config[Ckey].to_s
|
245
|
-
end
|
246
|
-
|
247
|
-
end
|
248
|
-
|
249
|
-
end
|
250
|
-
|
251
|
-
class Log
|
252
|
-
attr_reader :service, :levels, :logfile, :cull, :roll, :roll_age, :roll_size
|
253
|
-
|
254
|
-
def initialize(spec)
|
255
|
-
@service = spec[Analogger::Cservice]
|
256
|
-
@levels = spec[Analogger::Clevels]
|
257
|
-
@logfile = spec[Analogger::Clogfile]
|
258
|
-
@cull = spec[Analogger::Ccull]
|
259
|
-
@roll = spec[Analogger::Croll]
|
260
|
-
@roll_inteval = spec[Analogger::Croll_age]
|
261
|
-
@roll_size = spec[Analogger::Croll_size]
|
262
|
-
end
|
263
|
-
|
264
|
-
def to_s
|
265
|
-
"service: #{@service}\nlevels: #{@levels.inspect}\nlogfile: #{@logfile}\ncull: #{@cull}\n"
|
266
|
-
end
|
267
|
-
end
|
268
|
-
|
269
|
-
class AnaloggerProtocol < EventMachine::Connection
|
270
|
-
Ci = 'i'.freeze
|
271
|
-
Rcolon = /:/
|
272
|
-
MaxMessageLength = 8192
|
273
|
-
|
274
|
-
LoggerClass = Analogger
|
275
|
-
|
276
|
-
def post_init
|
277
|
-
setup
|
278
|
-
end
|
279
|
-
|
280
|
-
end
|
281
|
-
end
|
282
|
-
|
283
|
-
case RUBY_VERSION
|
284
|
-
when /^1.8/
|
285
|
-
require 'swiftcore/Analogger/receive_data_18.rb'
|
286
|
-
when /^1.9/
|
287
|
-
require 'swiftcore/Analogger/receive_data_19.rb'
|
288
|
-
else
|
289
|
-
raise "We're sorry, but Analogger is not supported for ruby versions prior to 1.8.x (and is untested below 1.8.5)."
|
290
|
-
end
|
291
|
-
|