logstash_writer 0.0.7 → 0.0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9254bae4e4a6f114fd683a3509b576d272256e162ecc5722c8d1c7a78a3e25e3
-  data.tar.gz: 9e61badc96a9bc3028a15311117eb95b6d3f37c7c59d950d66fe76d223160363
+  metadata.gz: b4f17ecaf181fbd3bf0160ecbd3a0a1fb156b5965be340e4bda05713edafa8fa
+  data.tar.gz: c279de10eb157ab271e88117b78d46fe345b8612648c9a5a945fe01c808752d3
 SHA512:
-  metadata.gz: 97ebc1f452ecc063b81e534ffb87e62ac4f225101454494ad8058abc18cef6427986b93a8643bc87631fdbde9d629dbebb35a51936f403a20e2d84b5517086b5
-  data.tar.gz: 01f8be94cee73a68b228e69f55f4f2d8e6448f8d37d885427a741cb8786f4c8fc37e1a99567ee92c8133f27e3faded0f6b3b1ca19cfe0bc382696682b281b6c7
+  metadata.gz: f3de8d9133af36ed73dcf965db8d8004d2cd92df054d23a59930a2ea444711d579f6877740924860d265fdb30eb2289ca6b5506331685cac8606d1e6ff13f170
+  data.tar.gz: 9d40ca34dd0622593b48ef73c613d278a3f6ab7241abb0f5596e19745094a9791a15e5b9a163167d48b7bb3cd8256d8b5f80616a9b695d7b178d9f3f65cd74e1

data/.editorconfig

@@ -0,0 +1,7 @@
+[*.rb]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+insert_final_newline = true
+charset = utf-8
+trim_trailing_whitespace = true

data/README.md

@@ -186,7 +186,7 @@ conduct](CODE_OF_CONDUCT.md).
 Unless otherwise stated, everything in this repo is covered by the following
 copyright notice:
 
-    Copyright (C) 2015  Civilized Discourse Construction Kit, Inc.
+    Copyright (C) 2016,2017,2018  Civilized Discourse Construction Kit, Inc.
 
     This program is free software: you can redistribute it and/or modify it
     under the terms of the GNU General Public License version 3, as

data/lib/logstash_writer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'ipaddr'
 require 'json'
 require 'resolv'
@@ -8,8 +10,8 @@ require 'prometheus/client'
 #
 # Flings events, represented as JSON objects, to logstash using the
 # `json_lines` codec (over TCP).  Doesn't do any munging or modification of
-# the event data given to it, other than adding `@timestamp` and
-# `[@metadata][_id]` fields if they do not already exist.
+# the event data given to it, other than adding a `@timestamp` field if
+# it doesn't already exist.
 #
 # We support highly-available logstash installations by means of multiple
 # address records, or via SRV records.  See the docs for .new for details
@@ -74,27 +76,29 @@ class LogstashWriter
     @server_name, @logger, @backlog = server_name, logger, backlog
 
     @metrics = {
-      received:            metrics_registry.counter(:"#{metrics_prefix}_events_received_total", "The number of logstash events which have been submitted for delivery"),
-      sent:                metrics_registry.counter(:"#{metrics_prefix}_events_written_total", "The number of logstash events which have been delivered to the logstash server"),
-      queue_size:          metrics_registry.gauge(:"#{metrics_prefix}_queue_size", "The number of events currently in the queue to be sent"),
-      dropped:             metrics_registry.counter(:"#{metrics_prefix}_events_dropped_total", "The number of events which have been dropped from the queue"),
+      received: metrics_registry.counter(:"#{metrics_prefix}_events_received_total", docstring: "The number of logstash events which have been submitted for delivery"),
+      sent: metrics_registry.counter(:"#{metrics_prefix}_events_written_total", docstring: "The number of logstash events which have been delivered to the logstash server", labels: %i{server}),
+      queue_size: metrics_registry.gauge(:"#{metrics_prefix}_queue_size", docstring: "The number of events currently in the queue to be sent"),
+      dropped: metrics_registry.counter(:"#{metrics_prefix}_events_dropped_total", docstring: "The number of events which have been dropped from the queue"),
 
-      lag:                 metrics_registry.gauge(:"#{metrics_prefix}_last_sent_event_time_seconds", "When the last event successfully sent to logstash was originally received"),
+      lag: metrics_registry.gauge(:"#{metrics_prefix}_last_sent_event_time_seconds", docstring: "When the last event successfully sent to logstash was originally received"),
 
-      connected:           metrics_registry.gauge(:"#{metrics_prefix}_connected_to_server", "Boolean flag indicating whether we are currently connected to a logstash server"),
-      connect_exception:   metrics_registry.counter(:"#{metrics_prefix}_connect_exceptions_total", "The number of exceptions that have occurred whilst attempting to connect to a logstash server"),
-      write_exception:     metrics_registry.counter(:"#{metrics_prefix}_write_exceptions_total", "The number of exceptions that have occurred whilst attempting to write an event to a logstash server"),
+      connected: metrics_registry.gauge(:"#{metrics_prefix}_connected_to_server", docstring: "Boolean flag indicating whether we are currently connected to a logstash server", labels: %i{server}),
+      connect_exception: metrics_registry.counter(:"#{metrics_prefix}_connect_exceptions_total", docstring: "The number of exceptions that have occurred whilst attempting to connect to a logstash server", labels: %i{server class}),
+      write_exception: metrics_registry.counter(:"#{metrics_prefix}_write_exceptions_total", docstring: "The number of exceptions that have occurred whilst attempting to write an event to a logstash server", labels: %i{server class}),
 
-      write_loop_exception: metrics_registry.counter(:"#{metrics_prefix}_write_loop_exceptions_total", "The number of exceptions that have occurred in the writing loop"),
-      write_loop_ok:        metrics_registry.gauge(:"#{metrics_prefix}_write_loop_ok", "Boolean flag indicating whether the writing loop is currently operating correctly, or is in a post-apocalyptic hellscape of never-ending exceptions"),
+      write_loop_exception: metrics_registry.counter(:"#{metrics_prefix}_write_loop_exceptions_total", docstring: "The number of exceptions that have occurred in the writing loop", labels: %i{class}),
+      write_loop_ok: metrics_registry.gauge(:"#{metrics_prefix}_write_loop_ok", docstring: "Boolean flag indicating whether the writing loop is currently operating correctly, or is in a post-apocalyptic hellscape of never-ending exceptions"),
     }
 
-    @metrics[:lag].set({}, 0)
-    @metrics[:queue_size].set({}, 0)
+    @metrics[:lag].set(0)
+    @metrics[:queue_size].set(0)
 
-    metrics_registry.gauge(:"#{metrics_prefix}_queue_max", "The maximum size of the event queue").set({}, backlog)
+    metrics_registry.gauge(:"#{metrics_prefix}_queue_max", docstring: "The maximum size of the event queue").set(backlog)
 
-    @queue = []
+    # We can't use a stdlib Queue object because we need to re-push items
+    # onto the front of the queue in case of error
+    @queue       = []
     @queue_mutex = Mutex.new
     @queue_cv    = ConditionVariable.new
 
@@ -104,8 +108,8 @@ class LogstashWriter
 
   # Add an event to the queue, to be sent to logstash.  Actual event
   # delivery will happen in a worker thread that is started with
-  # #run.  If the event does not have a `@timestamp` or `[@metadata][_id]`
-  # element, they will be added with appropriate values.
+  # #run.  If the event does not have a `@timestamp` field, it will
+  # be added set to the current time.
   #
   # @param e [Hash] the event data to be sent.
   #
@@ -120,22 +124,6 @@ class LogstashWriter
       e[:@timestamp] = Time.now.utc.strftime("%FT%T.%NZ")
     end
 
-    if e.has_key?("@metadata")
-      e[:@metadata] = (e[:@metadata] || {}).merge(e.delete("@metadata"))
-    end
-
-    unless e.has_key?(:@metadata)
-      e[:@metadata] = {}
-    end
-
-    unless e[:@metadata].has_key?(:_id) || e.has_key?("_id")
-      # This is the quickest way I've found to get a long, random string.
-      # We don't need any sort of cryptographic or unpredictability
-      # guarantees for what we're doing here, so SecureRandom is unnecessary
-      # overhead.
-      e[:@metadata][:_id] = rand(0x1000_0000_0000_0000_0000_0000_0000_0000).to_s(36)
-    end
-
     @queue_mutex.synchronize do
       @queue << { content: e, arrival_timestamp: Time.now }
       while @queue.length > @backlog
@@ -150,6 +138,33 @@ class LogstashWriter
     nil
   end
 
+  # Send events.
+  #
+  # Does not return until `#shutdown` is called (in another thread).
+  #
+  def run
+    @queue_mutex.synchronize do
+      @terminate = false
+    end
+
+    write_loop
+  end
+
+  # Tell the LogstashWriter to flush its queue and terminate operation.
+  #
+  # Returns immediately.
+  #
+  def shutdown
+    #:nocov:
+    @worker_mutex.synchronize do
+      @queue_mutex.synchronize do
+        @terminate = true
+        @queue_cv.signal
+      end
+    end
+    #:nocov:
+  end
+
   # Start sending events.
   #
   # This method will return almost immediately, and actual event
@@ -157,15 +172,16 @@ class LogstashWriter
   #
   # @return [NilClass]
   #
-  def run
+  def start!
     @worker_mutex.synchronize do
       if @worker_thread.nil?
-        m, cv = Mutex.new, ConditionVariable.new
-
-        @worker_thread = Thread.new { cv.signal; write_loop }
+        @queue_mutex.synchronize do
+          @terminate = false
+        end
 
-        # Don't return until the thread has *actually* started
-        m.synchronize { cv.wait(m) }
+        @worker_thread = Thread.new do
+          write_loop
+        end
       end
     end
 
@@ -180,18 +196,19 @@ class LogstashWriter
   #
   # @return [NilClass]
   #
-  def stop
+  def stop!
     @worker_mutex.synchronize do
       if @worker_thread
-        @terminate = true
-        @queue_cv.signal
+        @queue_mutex.synchronize do
+          @terminate = true
+          @queue_cv.signal
+        end
         begin
-          @worker_thread.join
+          @worker_thread.join unless @worker_thread == Thread.current
         rescue Exception => ex
           @logger.error("LogstashWriter") { (["Worker thread terminated with exception: #{ex.message} (#{ex.class})"] + ex.backtrace).join("\n  ") }
         end
         @worker_thread = nil
-        @socket_mutex.synchronize { (@current_target.close; @current_target = nil) if @current_target }
       end
     end
 
@@ -226,45 +243,43 @@ class LogstashWriter
   #
   def write_loop
     error_wait = INITIAL_RETRY_WAIT
+    Thread.current.name = "LogstashWriter"
 
-    catch :terminate do
-      loop do
-        event = nil
+    until @terminate do
+      event = nil
 
-        begin
-          @queue_mutex.synchronize do
-            while @queue.empty? && !@terminate
-              @queue_cv.wait(@queue_mutex)
-            end
-
-            if @queue.empty? && @terminate
-              @terminate = false
-              throw :terminate
-            end
-
-            event = @queue.shift
+      begin
+        @queue_mutex.synchronize do
+          while @queue.empty? && !@terminate
+            @queue_cv.wait(@queue_mutex)
           end
 
+          event = @queue.shift
+        end
+
+        if event
           current_target do |t|
             t.socket.puts event[:content].to_json
             stat_sent(t.to_s, event[:arrival_timestamp])
-            @metrics[:write_loop_ok].set({}, 1)
+            @metrics[:write_loop_ok].set(1)
             error_wait = INITIAL_RETRY_WAIT
           end
-        rescue StandardError => ex
-          @logger.error("LogstashWriter") { (["Exception in write_loop: #{ex.message} (#{ex.class})"] + ex.backtrace).join("\n  ") }
-          @queue_mutex.synchronize { @queue.unshift(event) if event }
-          @metrics[:write_loop_exception].increment(class: ex.class.to_s)
-          @metrics[:write_loop_ok].set({}, 0)
-          sleep error_wait
-          # Increase the error wait timeout for next time, up to a maximum
-          # interval of about 60 seconds
-          error_wait *= 1.1
-          error_wait = 60 if error_wait > 60
-          error_wait += rand / 0.5
         end
+      rescue StandardError => ex
+        @logger.error("LogstashWriter") { (["Exception in write_loop: #{ex.message} (#{ex.class})"] + ex.backtrace).join("\n  ") }
+        @queue_mutex.synchronize { @queue.unshift(event) if event }
+        @metrics[:write_loop_exception].increment(labels: { class: ex.class.to_s })
+        @metrics[:write_loop_ok].set(0)
+        sleep error_wait
+        # Increase the error wait timeout for next time, up to a maximum
+        # interval of about 60 seconds
+        error_wait *= 1.1
+        error_wait = 60 if error_wait > 60
+        error_wait += rand / 0.5
       end
     end
+
+    force_disconnect!
   end
 
   # Yield a Target connected to the server we currently believe to be
@@ -296,14 +311,14 @@ class LogstashWriter
             raise Errno::ENOTCONN unless IO.select([@current_target.socket], [], [], 0).nil?
 
             yield @current_target
-            @metrics[:connected].set({ server: @current_target.describe_peer }, 1)
+            @metrics[:connected].set(1, labels: { server: @current_target.describe_peer })
             done = true
           rescue SystemCallError => ex
             # Something went wrong during the send; disconnect from this
             # server and recycle
-            @metrics[:write_exception].increment(server: @current_target.describe_peer, class: ex.class.to_s)
-            @metrics[:connected].set({ server: @current_target.describe_peer }, 0)
-            @logger.info("LogstashWriter") { "Error while writing to current server #{@current_target.describe_peer}: #{ex.message} (#{ex.class})" }
+            @metrics[:write_exception].increment(labels: { server: @current_target.describe_peer, class: ex.class.to_s })
+            @metrics[:connected].set(0, labels: { server: @current_target.describe_peer })
+            @logger.error("LogstashWriter") { "Error while writing to current server #{@current_target.describe_peer}: #{ex.message} (#{ex.class})" }
             @current_target.close
             @current_target = nil
 
@@ -328,6 +343,7 @@ class LogstashWriter
                 @current_target = next_server
                 # Trigger a connection attempt
                 @current_target.socket
+                @logger.info("LogstashWriter") { "Connected to #{@current_target.describe_peer}" }
               else
                 @logger.debug("LogstashWriter") { "Could not connect to any server; pausing before trying again" }
                 @current_target = nil
@@ -344,7 +360,7 @@ class LogstashWriter
               end
             rescue SystemCallError => ex
               # Connection failed for any number of reasons; try the next one in the list
-              @metrics[:connect_exception].increment(server: next_server.to_s, class: ex.class.to_s)
+              @metrics[:connect_exception].increment(labels: { server: next_server.to_s, class: ex.class.to_s })
               @logger.error("LogstashWriter") { "Failed to connect to #{next_server.to_s}: #{ex.message} (#{ex.class})" }
               sleep INITIAL_RETRY_WAIT
               retry
@@ -436,19 +452,19 @@ class LogstashWriter
   end
 
   def stat_received
-    @metrics[:received].increment({})
-    @metrics[:queue_size].increment({})
+    @metrics[:received].increment
+    @metrics[:queue_size].increment
   end
 
   def stat_sent(peer, arrived_time)
-    @metrics[:sent].increment(server: peer)
-    @metrics[:queue_size].decrement({})
-    @metrics[:lag].set({}, arrived_time.to_f)
+    @metrics[:sent].increment(labels: { server: peer })
+    @metrics[:queue_size].decrement
+    @metrics[:lag].set(arrived_time.to_f)
   end
 
   def stat_dropped
-    @metrics[:queue_size].decrement({})
-    @metrics[:dropped].increment({})
+    @metrics[:queue_size].decrement
+    @metrics[:dropped].increment
   end
 
   # An individual target for logstash messages

data/logstash_writer.gemspec

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 begin
   require 'git-version-bump'
 rescue LoadError
@@ -27,21 +29,16 @@ Gem::Specification.new do |s|
 
   s.required_ruby_version = ">= 2.3.0"
 
-  # prometheus-client provides no guaranteed backwards compatibility,
-  # and in fact happily breaks things with no notice, so we're stuck
-  # with hard-coding a specific version to avoid unexpected disaster.
-  s.add_runtime_dependency "prometheus-client", "0.8.0"
+  s.add_runtime_dependency "prometheus-client", "~> 2.0"
 
   s.add_development_dependency 'bundler'
   s.add_development_dependency 'github-release'
   s.add_development_dependency 'git-version-bump'
   s.add_development_dependency 'guard-rspec'
-  s.add_development_dependency 'guard-rubocop'
   s.add_development_dependency 'rack-test'
   s.add_development_dependency 'rake', "~> 12.0"
   s.add_development_dependency 'redcarpet'
   s.add_development_dependency 'rspec'
-  s.add_development_dependency 'rubocop'
   s.add_development_dependency 'simplecov'
   s.add_development_dependency 'yard'
 end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: logstash_writer
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.12
 platform: ruby
 authors:
 - Matt Palmer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-04 00:00:00.000000000 Z
+date: 2020-05-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: prometheus-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -80,20 +80,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: guard-rubocop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: rack-test
   requirement: !ruby/object:Gem::Requirement
@@ -150,20 +136,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rubocop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -202,8 +174,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".editorconfig"
 - ".gitignore"
-- ".rubocop.yml"
 - ".travis.yml"
 - ".yardopts"
 - CODE_OF_CONDUCT.md
@@ -230,8 +202,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Opinionated logstash event pump

data/.rubocop.yml

@@ -1,116 +0,0 @@
-AllCops:
-  TargetRubyVersion: 2.4
-  DisabledByDefault: true
-  Exclude:
-    - 'db/schema.rb'
-    - 'bundle/**/*'
-    - 'vendor/**/*'
-    - 'node_modules/**/*'
-    - 'public/**/*'
-
-# Prefer &&/|| over and/or.
-Style/AndOr:
-  Enabled: true
-
-# Do not use braces for hash literals when they are the last argument of a
-# method call.
-Style/BracesAroundHashParameters:
-  Enabled: true
-
-# Align `when` with `case`.
-Layout/CaseIndentation:
-  Enabled: true
-
-# Align comments with method definitions.
-Layout/CommentIndentation:
-  Enabled: true
-
-# No extra empty lines.
-Layout/EmptyLines:
-  Enabled: true
-
-# Use Ruby >= 1.9 syntax for hashes. Prefer { a: :b } over { :a => :b }.
-Style/HashSyntax:
-  Enabled: true
-
-# Two spaces, no tabs (for indentation).
-Layout/IndentationWidth:
-  Enabled: true
-
-Layout/SpaceAfterColon:
-  Enabled: true
-
-Layout/SpaceAfterComma:
-  Enabled: true
-
-Layout/SpaceAroundEqualsInParameterDefault:
-  Enabled: true
-
-Layout/SpaceAroundKeyword:
-  Enabled: true
-
-Layout/SpaceAroundOperators:
-  Enabled: true
-
-Layout/SpaceBeforeFirstArg:
-    Enabled: true
-
-# Defining a method with parameters needs parentheses.
-Style/MethodDefParentheses:
-  Enabled: true
-
-# Use `foo {}` not `foo{}`.
-Layout/SpaceBeforeBlockBraces:
-  Enabled: true
-
-# Use `foo { bar }` not `foo {bar}`.
-Layout/SpaceInsideBlockBraces:
-  Enabled: true
-
-# Use `{ a: 1 }` not `{a:1}`.
-Layout/SpaceInsideHashLiteralBraces:
-  Enabled: true
-
-Layout/SpaceInsideParens:
-  Enabled: true
-
-# Detect hard tabs, no hard tabs.
-Layout/Tab:
-  Enabled: true
-
-# Blank lines should not have any spaces.
-Layout/TrailingBlankLines:
-  Enabled: true
-
-# No trailing whitespace.
-Layout/TrailingWhitespace:
-  Enabled: true
-
-Lint/Debugger:
-  Enabled: true
-
-Layout/BlockAlignment:
-  Enabled: true
-
-# Align `end` with the matching keyword or starting expression except for
-# assignments, where it should be aligned with the LHS.
-Layout/EndAlignment:
-  Enabled: true
-  EnforcedStyleAlignWith: variable
-
-# Use my_method(my_arg) not my_method( my_arg ) or my_method my_arg.
-Lint/RequireParentheses:
-  Enabled: true
-
-Lint/ShadowingOuterLocalVariable:
-  Enabled: true
-
-Layout/MultilineMethodCallIndentation:
-  Enabled: true
-  EnforcedStyle: indented
-
-Layout/AlignHash:
-  Enabled: true
-
-Bundler/OrderedGems:
-  Enabled: false