logstash_writer 0.0.11 → 0.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/logstash_writer.rb +29 -27
- data/logstash_writer.gemspec +3 -6
- metadata +7 -36
- data/.rubocop.yml +0 -116
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b4f17ecaf181fbd3bf0160ecbd3a0a1fb156b5965be340e4bda05713edafa8fa
|
4
|
+
data.tar.gz: c279de10eb157ab271e88117b78d46fe345b8612648c9a5a945fe01c808752d3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f3de8d9133af36ed73dcf965db8d8004d2cd92df054d23a59930a2ea444711d579f6877740924860d265fdb30eb2289ca6b5506331685cac8606d1e6ff13f170
|
7
|
+
data.tar.gz: 9d40ca34dd0622593b48ef73c613d278a3f6ab7241abb0f5596e19745094a9791a15e5b9a163167d48b7bb3cd8256d8b5f80616a9b695d7b178d9f3f65cd74e1
|
data/lib/logstash_writer.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'ipaddr'
|
2
4
|
require 'json'
|
3
5
|
require 'resolv'
|
@@ -74,25 +76,25 @@ class LogstashWriter
|
|
74
76
|
@server_name, @logger, @backlog = server_name, logger, backlog
|
75
77
|
|
76
78
|
@metrics = {
|
77
|
-
received: metrics_registry.counter(:"#{metrics_prefix}_events_received_total", "The number of logstash events which have been submitted for delivery"),
|
78
|
-
sent: metrics_registry.counter(:"#{metrics_prefix}_events_written_total", "The number of logstash events which have been delivered to the logstash server"),
|
79
|
-
queue_size: metrics_registry.gauge(:"#{metrics_prefix}_queue_size", "The number of events currently in the queue to be sent"),
|
80
|
-
dropped: metrics_registry.counter(:"#{metrics_prefix}_events_dropped_total", "The number of events which have been dropped from the queue"),
|
79
|
+
received: metrics_registry.counter(:"#{metrics_prefix}_events_received_total", docstring: "The number of logstash events which have been submitted for delivery"),
|
80
|
+
sent: metrics_registry.counter(:"#{metrics_prefix}_events_written_total", docstring: "The number of logstash events which have been delivered to the logstash server", labels: %i{server}),
|
81
|
+
queue_size: metrics_registry.gauge(:"#{metrics_prefix}_queue_size", docstring: "The number of events currently in the queue to be sent"),
|
82
|
+
dropped: metrics_registry.counter(:"#{metrics_prefix}_events_dropped_total", docstring: "The number of events which have been dropped from the queue"),
|
81
83
|
|
82
|
-
lag: metrics_registry.gauge(:"#{metrics_prefix}_last_sent_event_time_seconds", "When the last event successfully sent to logstash was originally received"),
|
84
|
+
lag: metrics_registry.gauge(:"#{metrics_prefix}_last_sent_event_time_seconds", docstring: "When the last event successfully sent to logstash was originally received"),
|
83
85
|
|
84
|
-
connected: metrics_registry.gauge(:"#{metrics_prefix}_connected_to_server", "Boolean flag indicating whether we are currently connected to a logstash server"),
|
85
|
-
connect_exception: metrics_registry.counter(:"#{metrics_prefix}_connect_exceptions_total", "The number of exceptions that have occurred whilst attempting to connect to a logstash server"),
|
86
|
-
write_exception: metrics_registry.counter(:"#{metrics_prefix}_write_exceptions_total", "The number of exceptions that have occurred whilst attempting to write an event to a logstash server"),
|
86
|
+
connected: metrics_registry.gauge(:"#{metrics_prefix}_connected_to_server", docstring: "Boolean flag indicating whether we are currently connected to a logstash server", labels: %i{server}),
|
87
|
+
connect_exception: metrics_registry.counter(:"#{metrics_prefix}_connect_exceptions_total", docstring: "The number of exceptions that have occurred whilst attempting to connect to a logstash server", labels: %i{server class}),
|
88
|
+
write_exception: metrics_registry.counter(:"#{metrics_prefix}_write_exceptions_total", docstring: "The number of exceptions that have occurred whilst attempting to write an event to a logstash server", labels: %i{server class}),
|
87
89
|
|
88
|
-
write_loop_exception: metrics_registry.counter(:"#{metrics_prefix}_write_loop_exceptions_total", "The number of exceptions that have occurred in the writing loop"),
|
89
|
-
write_loop_ok: metrics_registry.gauge(:"#{metrics_prefix}_write_loop_ok", "Boolean flag indicating whether the writing loop is currently operating correctly, or is in a post-apocalyptic hellscape of never-ending exceptions"),
|
90
|
+
write_loop_exception: metrics_registry.counter(:"#{metrics_prefix}_write_loop_exceptions_total", docstring: "The number of exceptions that have occurred in the writing loop", labels: %i{class}),
|
91
|
+
write_loop_ok: metrics_registry.gauge(:"#{metrics_prefix}_write_loop_ok", docstring: "Boolean flag indicating whether the writing loop is currently operating correctly, or is in a post-apocalyptic hellscape of never-ending exceptions"),
|
90
92
|
}
|
91
93
|
|
92
|
-
@metrics[:lag].set(
|
93
|
-
@metrics[:queue_size].set(
|
94
|
+
@metrics[:lag].set(0)
|
95
|
+
@metrics[:queue_size].set(0)
|
94
96
|
|
95
|
-
metrics_registry.gauge(:"#{metrics_prefix}_queue_max", "The maximum size of the event queue").set(
|
97
|
+
metrics_registry.gauge(:"#{metrics_prefix}_queue_max", docstring: "The maximum size of the event queue").set(backlog)
|
96
98
|
|
97
99
|
# We can't use a stdlib Queue object because we need to re-push items
|
98
100
|
# onto the front of the queue in case of error
|
@@ -259,15 +261,15 @@ class LogstashWriter
|
|
259
261
|
current_target do |t|
|
260
262
|
t.socket.puts event[:content].to_json
|
261
263
|
stat_sent(t.to_s, event[:arrival_timestamp])
|
262
|
-
@metrics[:write_loop_ok].set(
|
264
|
+
@metrics[:write_loop_ok].set(1)
|
263
265
|
error_wait = INITIAL_RETRY_WAIT
|
264
266
|
end
|
265
267
|
end
|
266
268
|
rescue StandardError => ex
|
267
269
|
@logger.error("LogstashWriter") { (["Exception in write_loop: #{ex.message} (#{ex.class})"] + ex.backtrace).join("\n ") }
|
268
270
|
@queue_mutex.synchronize { @queue.unshift(event) if event }
|
269
|
-
@metrics[:write_loop_exception].increment(class: ex.class.to_s)
|
270
|
-
@metrics[:write_loop_ok].set(
|
271
|
+
@metrics[:write_loop_exception].increment(labels: { class: ex.class.to_s })
|
272
|
+
@metrics[:write_loop_ok].set(0)
|
271
273
|
sleep error_wait
|
272
274
|
# Increase the error wait timeout for next time, up to a maximum
|
273
275
|
# interval of about 60 seconds
|
@@ -309,13 +311,13 @@ class LogstashWriter
|
|
309
311
|
raise Errno::ENOTCONN unless IO.select([@current_target.socket], [], [], 0).nil?
|
310
312
|
|
311
313
|
yield @current_target
|
312
|
-
@metrics[:connected].set({ server: @current_target.describe_peer }
|
314
|
+
@metrics[:connected].set(1, labels: { server: @current_target.describe_peer })
|
313
315
|
done = true
|
314
316
|
rescue SystemCallError => ex
|
315
317
|
# Something went wrong during the send; disconnect from this
|
316
318
|
# server and recycle
|
317
|
-
@metrics[:write_exception].increment(server: @current_target.describe_peer, class: ex.class.to_s)
|
318
|
-
@metrics[:connected].set({ server: @current_target.describe_peer }
|
319
|
+
@metrics[:write_exception].increment(labels: { server: @current_target.describe_peer, class: ex.class.to_s })
|
320
|
+
@metrics[:connected].set(0, labels: { server: @current_target.describe_peer })
|
319
321
|
@logger.error("LogstashWriter") { "Error while writing to current server #{@current_target.describe_peer}: #{ex.message} (#{ex.class})" }
|
320
322
|
@current_target.close
|
321
323
|
@current_target = nil
|
@@ -358,7 +360,7 @@ class LogstashWriter
|
|
358
360
|
end
|
359
361
|
rescue SystemCallError => ex
|
360
362
|
# Connection failed for any number of reasons; try the next one in the list
|
361
|
-
@metrics[:connect_exception].increment(server: next_server.to_s, class: ex.class.to_s)
|
363
|
+
@metrics[:connect_exception].increment(labels: { server: next_server.to_s, class: ex.class.to_s })
|
362
364
|
@logger.error("LogstashWriter") { "Failed to connect to #{next_server.to_s}: #{ex.message} (#{ex.class})" }
|
363
365
|
sleep INITIAL_RETRY_WAIT
|
364
366
|
retry
|
@@ -450,19 +452,19 @@ class LogstashWriter
|
|
450
452
|
end
|
451
453
|
|
452
454
|
def stat_received
|
453
|
-
@metrics[:received].increment
|
454
|
-
@metrics[:queue_size].increment
|
455
|
+
@metrics[:received].increment
|
456
|
+
@metrics[:queue_size].increment
|
455
457
|
end
|
456
458
|
|
457
459
|
def stat_sent(peer, arrived_time)
|
458
|
-
@metrics[:sent].increment(server: peer)
|
459
|
-
@metrics[:queue_size].decrement
|
460
|
-
@metrics[:lag].set(
|
460
|
+
@metrics[:sent].increment(labels: { server: peer })
|
461
|
+
@metrics[:queue_size].decrement
|
462
|
+
@metrics[:lag].set(arrived_time.to_f)
|
461
463
|
end
|
462
464
|
|
463
465
|
def stat_dropped
|
464
|
-
@metrics[:queue_size].decrement
|
465
|
-
@metrics[:dropped].increment
|
466
|
+
@metrics[:queue_size].decrement
|
467
|
+
@metrics[:dropped].increment
|
466
468
|
end
|
467
469
|
|
468
470
|
# An individual target for logstash messages
|
data/logstash_writer.gemspec
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
begin
|
2
4
|
require 'git-version-bump'
|
3
5
|
rescue LoadError
|
@@ -27,21 +29,16 @@ Gem::Specification.new do |s|
|
|
27
29
|
|
28
30
|
s.required_ruby_version = ">= 2.3.0"
|
29
31
|
|
30
|
-
|
31
|
-
# and in fact happily breaks things with no notice, so we're stuck
|
32
|
-
# with hard-coding a specific version to avoid unexpected disaster.
|
33
|
-
s.add_runtime_dependency "prometheus-client", "0.8.0"
|
32
|
+
s.add_runtime_dependency "prometheus-client", "~> 2.0"
|
34
33
|
|
35
34
|
s.add_development_dependency 'bundler'
|
36
35
|
s.add_development_dependency 'github-release'
|
37
36
|
s.add_development_dependency 'git-version-bump'
|
38
37
|
s.add_development_dependency 'guard-rspec'
|
39
|
-
s.add_development_dependency 'guard-rubocop'
|
40
38
|
s.add_development_dependency 'rack-test'
|
41
39
|
s.add_development_dependency 'rake', "~> 12.0"
|
42
40
|
s.add_development_dependency 'redcarpet'
|
43
41
|
s.add_development_dependency 'rspec'
|
44
|
-
s.add_development_dependency 'rubocop'
|
45
42
|
s.add_development_dependency 'simplecov'
|
46
43
|
s.add_development_dependency 'yard'
|
47
44
|
end
|
metadata
CHANGED
@@ -1,29 +1,29 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash_writer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.12
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Matt Palmer
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-05-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: prometheus-client
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- -
|
17
|
+
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
19
|
+
version: '2.0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- -
|
24
|
+
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version:
|
26
|
+
version: '2.0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: bundler
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -80,20 +80,6 @@ dependencies:
|
|
80
80
|
- - ">="
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '0'
|
83
|
-
- !ruby/object:Gem::Dependency
|
84
|
-
name: guard-rubocop
|
85
|
-
requirement: !ruby/object:Gem::Requirement
|
86
|
-
requirements:
|
87
|
-
- - ">="
|
88
|
-
- !ruby/object:Gem::Version
|
89
|
-
version: '0'
|
90
|
-
type: :development
|
91
|
-
prerelease: false
|
92
|
-
version_requirements: !ruby/object:Gem::Requirement
|
93
|
-
requirements:
|
94
|
-
- - ">="
|
95
|
-
- !ruby/object:Gem::Version
|
96
|
-
version: '0'
|
97
83
|
- !ruby/object:Gem::Dependency
|
98
84
|
name: rack-test
|
99
85
|
requirement: !ruby/object:Gem::Requirement
|
@@ -150,20 +136,6 @@ dependencies:
|
|
150
136
|
- - ">="
|
151
137
|
- !ruby/object:Gem::Version
|
152
138
|
version: '0'
|
153
|
-
- !ruby/object:Gem::Dependency
|
154
|
-
name: rubocop
|
155
|
-
requirement: !ruby/object:Gem::Requirement
|
156
|
-
requirements:
|
157
|
-
- - ">="
|
158
|
-
- !ruby/object:Gem::Version
|
159
|
-
version: '0'
|
160
|
-
type: :development
|
161
|
-
prerelease: false
|
162
|
-
version_requirements: !ruby/object:Gem::Requirement
|
163
|
-
requirements:
|
164
|
-
- - ">="
|
165
|
-
- !ruby/object:Gem::Version
|
166
|
-
version: '0'
|
167
139
|
- !ruby/object:Gem::Dependency
|
168
140
|
name: simplecov
|
169
141
|
requirement: !ruby/object:Gem::Requirement
|
@@ -204,7 +176,6 @@ extra_rdoc_files: []
|
|
204
176
|
files:
|
205
177
|
- ".editorconfig"
|
206
178
|
- ".gitignore"
|
207
|
-
- ".rubocop.yml"
|
208
179
|
- ".travis.yml"
|
209
180
|
- ".yardopts"
|
210
181
|
- CODE_OF_CONDUCT.md
|
@@ -231,7 +202,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
231
202
|
- !ruby/object:Gem::Version
|
232
203
|
version: '0'
|
233
204
|
requirements: []
|
234
|
-
rubygems_version: 3.0.
|
205
|
+
rubygems_version: 3.0.3
|
235
206
|
signing_key:
|
236
207
|
specification_version: 4
|
237
208
|
summary: Opinionated logstash event pump
|
data/.rubocop.yml
DELETED
@@ -1,116 +0,0 @@
|
|
1
|
-
AllCops:
|
2
|
-
TargetRubyVersion: 2.4
|
3
|
-
DisabledByDefault: true
|
4
|
-
Exclude:
|
5
|
-
- 'db/schema.rb'
|
6
|
-
- 'bundle/**/*'
|
7
|
-
- 'vendor/**/*'
|
8
|
-
- 'node_modules/**/*'
|
9
|
-
- 'public/**/*'
|
10
|
-
|
11
|
-
# Prefer &&/|| over and/or.
|
12
|
-
Style/AndOr:
|
13
|
-
Enabled: true
|
14
|
-
|
15
|
-
# Do not use braces for hash literals when they are the last argument of a
|
16
|
-
# method call.
|
17
|
-
Style/BracesAroundHashParameters:
|
18
|
-
Enabled: true
|
19
|
-
|
20
|
-
# Align `when` with `case`.
|
21
|
-
Layout/CaseIndentation:
|
22
|
-
Enabled: true
|
23
|
-
|
24
|
-
# Align comments with method definitions.
|
25
|
-
Layout/CommentIndentation:
|
26
|
-
Enabled: true
|
27
|
-
|
28
|
-
# No extra empty lines.
|
29
|
-
Layout/EmptyLines:
|
30
|
-
Enabled: true
|
31
|
-
|
32
|
-
# Use Ruby >= 1.9 syntax for hashes. Prefer { a: :b } over { :a => :b }.
|
33
|
-
Style/HashSyntax:
|
34
|
-
Enabled: true
|
35
|
-
|
36
|
-
# Two spaces, no tabs (for indentation).
|
37
|
-
Layout/IndentationWidth:
|
38
|
-
Enabled: true
|
39
|
-
|
40
|
-
Layout/SpaceAfterColon:
|
41
|
-
Enabled: true
|
42
|
-
|
43
|
-
Layout/SpaceAfterComma:
|
44
|
-
Enabled: true
|
45
|
-
|
46
|
-
Layout/SpaceAroundEqualsInParameterDefault:
|
47
|
-
Enabled: true
|
48
|
-
|
49
|
-
Layout/SpaceAroundKeyword:
|
50
|
-
Enabled: true
|
51
|
-
|
52
|
-
Layout/SpaceAroundOperators:
|
53
|
-
Enabled: true
|
54
|
-
|
55
|
-
Layout/SpaceBeforeFirstArg:
|
56
|
-
Enabled: true
|
57
|
-
|
58
|
-
# Defining a method with parameters needs parentheses.
|
59
|
-
Style/MethodDefParentheses:
|
60
|
-
Enabled: true
|
61
|
-
|
62
|
-
# Use `foo {}` not `foo{}`.
|
63
|
-
Layout/SpaceBeforeBlockBraces:
|
64
|
-
Enabled: true
|
65
|
-
|
66
|
-
# Use `foo { bar }` not `foo {bar}`.
|
67
|
-
Layout/SpaceInsideBlockBraces:
|
68
|
-
Enabled: true
|
69
|
-
|
70
|
-
# Use `{ a: 1 }` not `{a:1}`.
|
71
|
-
Layout/SpaceInsideHashLiteralBraces:
|
72
|
-
Enabled: true
|
73
|
-
|
74
|
-
Layout/SpaceInsideParens:
|
75
|
-
Enabled: true
|
76
|
-
|
77
|
-
# Detect hard tabs, no hard tabs.
|
78
|
-
Layout/Tab:
|
79
|
-
Enabled: true
|
80
|
-
|
81
|
-
# Blank lines should not have any spaces.
|
82
|
-
Layout/TrailingBlankLines:
|
83
|
-
Enabled: true
|
84
|
-
|
85
|
-
# No trailing whitespace.
|
86
|
-
Layout/TrailingWhitespace:
|
87
|
-
Enabled: true
|
88
|
-
|
89
|
-
Lint/Debugger:
|
90
|
-
Enabled: true
|
91
|
-
|
92
|
-
Layout/BlockAlignment:
|
93
|
-
Enabled: true
|
94
|
-
|
95
|
-
# Align `end` with the matching keyword or starting expression except for
|
96
|
-
# assignments, where it should be aligned with the LHS.
|
97
|
-
Layout/EndAlignment:
|
98
|
-
Enabled: true
|
99
|
-
EnforcedStyleAlignWith: variable
|
100
|
-
|
101
|
-
# Use my_method(my_arg) not my_method( my_arg ) or my_method my_arg.
|
102
|
-
Lint/RequireParentheses:
|
103
|
-
Enabled: true
|
104
|
-
|
105
|
-
Lint/ShadowingOuterLocalVariable:
|
106
|
-
Enabled: true
|
107
|
-
|
108
|
-
Layout/MultilineMethodCallIndentation:
|
109
|
-
Enabled: true
|
110
|
-
EnforcedStyle: indented
|
111
|
-
|
112
|
-
Layout/AlignHash:
|
113
|
-
Enabled: true
|
114
|
-
|
115
|
-
Bundler/OrderedGems:
|
116
|
-
Enabled: false
|