config_skeleton 0.3.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bdaa156de0445e8a32b2d49d55ea11308d153d4c73658f6c4ff648a7c32a1440
-  data.tar.gz: 5dff0715f11dad930b6dd485204d9a1f2b75a47afc9096cb33dd0c4f5a538423
+  metadata.gz: 0b419f026421b513b20f2950609f9b45339d86433b88e8372bd94746f6a72091
+  data.tar.gz: 7e0a5ac1b0dfe9ad7ee989aeb9d0051b06d179190328460ba09c273312b36b7d
 SHA512:
-  metadata.gz: e5c7ffc7621e69441f17349d9e46a9d4f5146ed88a4c5b382524dfd64b7c0aa6b4aedc72bcb3ffac220509299983adff24263f86e8a327072ed34b933095ab3e
-  data.tar.gz: 50e06b7b3fbe40c4baf522b95595bd447ec3e349c2e14a1cbf7da64500f2db5a64d006ec6b9f15a2320fdbc76f879f733d91ff555c26d7eef7432d1fc46ff059
+  metadata.gz: c36398b332b1bfa53a66d11a2c083af4a0f794e53fba518278fcef7ef41dfc63d51b4b010f508eeb898bf99c10ff55a2f813251bdfefe122a70e3254b87da46a
+  data.tar.gz: 53acc41e5da3aea61cda288cfe90f122e8d8871eea594d5e4b0857d127c01ba20a542df7087884c34e0e75589e968820d53b41db62c3f85f91d49eea9dbf88a2

data/.github/workflows/ruby.yml

@@ -43,6 +43,6 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: Release Gem
-        uses: CvX/publish-rubygems-action@master
+        uses: discourse/publish-rubygems-action@main
         env:
           RUBYGEMS_API_KEY: ${{secrets.RUBYGEMS_API_KEY}}

data/config_skeleton.gemspec

@@ -1,14 +1,14 @@
 Gem::Specification.new do |s|
   s.name = "config_skeleton"
 
-  s.version = "0.3.0"
+  s.version = "1.1.0"
 
   s.platform = Gem::Platform::RUBY
 
   s.summary  = "Dynamically generate configs and reload servers"
 
-  s.authors  = ["Matt Palmer"]
-  s.email    = ["matt.palmer@discourse.org"]
+  s.authors  = ["Matt Palmer", "Discourse Team"]
+  s.email    = ["matt.palmer@discourse.org", "team@discourse.org"]
   s.homepage = "https://github.com/discourse/config_skeleton"
 
   s.files = `git ls-files -z`.split("\0").reject { |f| f =~ /^(G|spec|Rakefile)/ }
@@ -16,9 +16,8 @@ Gem::Specification.new do |s|
   s.required_ruby_version = ">= 2.3.0"
 
   s.add_runtime_dependency 'diffy', '~> 3.0'
-  s.add_runtime_dependency 'frankenstein', '~> 1.0'
   s.add_runtime_dependency 'rb-inotify', '~> 0.9'
-  s.add_runtime_dependency 'service_skeleton', '> 0.a'
+  s.add_runtime_dependency 'service_skeleton', "~> 1.0"
 
   s.add_development_dependency 'bundler'
   s.add_development_dependency 'github-release'

data/lib/config_skeleton.rb

@@ -2,9 +2,16 @@ require 'diffy'
 require 'fileutils'
 require 'frankenstein'
 require 'logger'
-require 'rb-inotify'
 require 'service_skeleton'
 require 'tempfile'
+require 'digest/md5'
+
+begin
+  require 'rb-inotify' unless ENV["DISABLE_INOTIFY"]
+rescue FFI::NotFoundError => e
+  STDERR.puts "ERROR: Unable to initialize rb-inotify. To disable, set DISABLE_INOTIFY=1"
+  raise
+end
 
 # Framework for creating config generation systems.
 #
@@ -23,19 +30,18 @@ require 'tempfile'
 #
 # 1. Implement service-specific config generation and reloading code, by
 #    overriding the private methods #config_file, #config_data, and #reload_server
-#    (and also potentially #config_ok? and #sleep_duration).
+#    (and also potentially #config_ok?, #sleep_duration, #before_regenerate_config, and #after_regenerate_config).
 #    See the documentation for those methods for what they need to do.
 #
 # 1. Setup any file watchers you want with .watch and #watch.
 #
-# 1. Instantiate your new class, passing in an environment hash, and then call
-#    #start.  Something like this should do the trick:
+# 1. Use the ServiceSkeleton Runner to start the service. Something like this should do the trick:
 #
 #        class MyConfigGenerator < ConfigSkeleton
 #          # Implement all the necessary methods
 #        end
 #
-#        MyConfigGenerator.new(ENV).start if __FILE__ == $0
+#        ServiceSkeleton::Runner.new(MyConfigGenerator, ENV).run if __FILE__ == $0
 #
 # 1. Sit back and relax.
 #
@@ -135,7 +141,8 @@ require 'tempfile'
 # method, passing one or more strings containing the full path to files or
 # directories to watch.
 #
-class ConfigSkeleton < ServiceSkeleton
+class ConfigSkeleton
+  include ServiceSkeleton
   # All ConfigSkeleton-related errors will be subclasses of this.
   class Error < StandardError; end
 
@@ -154,6 +161,19 @@ class ConfigSkeleton < ServiceSkeleton
     end
   end
 
+  def self.inherited(klass)
+    klass.gauge :"#{klass.service_name}_last_generation_timestamp", docstring: "When the last config generation run was made"
+    klass.gauge :"#{klass.service_name}_last_change_timestamp", docstring: "When the config file was last written to"
+    klass.counter :"#{klass.service_name}_reload_total", docstring: "How many times we've asked the server to reload", labels: [:status]
+    klass.counter :"#{klass.service_name}_signals_total", docstring: "How many signals have been received (and handled)"
+    klass.gauge :"#{klass.service_name}_config_ok", docstring: "Whether the last config change was accepted by the server"
+
+    klass.hook_signal("HUP") do
+      logger.info("SIGHUP") { "received SIGHUP, triggering config regeneration" }
+      @trigger_regen_w << "."
+    end
+  end
+
   # Declare a file watch on all instances of the config generator.
   #
   # When you're looking to watch a file whose path is well-known and never-changing, you
@@ -185,20 +205,8 @@ class ConfigSkeleton < ServiceSkeleton
     @watches || []
   end
 
-  # Create a new config generator.
-  #
-  # @param env [Hash<String, String>] the environment in which this config
-  #   generator runs.  Typically you'll just pass `ENV` in here, but you can
-  #   pass in any hash you like, for testing purposes.
-  #
-  def initialize(env)
+  def initialize(*_)
     super
-
-    hook_signal(:HUP) do
-      logger.info("SIGHUP") { "received SIGHUP, triggering config regeneration" }
-      @trigger_regen_w << "."
-    end
-
     initialize_config_skeleton_metrics
     @trigger_regen_r, @trigger_regen_w = IO.pipe
     @terminate_r, @terminate_w = IO.pipe
@@ -232,11 +240,16 @@ class ConfigSkeleton < ServiceSkeleton
     logger.debug(logloc) { "notifier fd is #{notifier.to_io.inspect}" }
 
     loop do
-      if ios = IO.select(
-          [notifier.to_io, @terminate_r, @trigger_regen_r],
-          [], [],
-          sleep_duration.tap { |d| logger.debug(logloc) { "Sleeping for #{d} seconds" } }
-      )
+      if cooldown_duration > 0
+        logger.debug(logloc) { "Sleeping for #{cooldown_duration} seconds (cooldown)" }
+        IO.select([@terminate_r], [], [], cooldown_duration)
+      end
+
+      timeout = sleep_duration - cooldown_duration
+      logger.debug(logloc) { "Sleeping for #{timeout} seconds unless interrupted" }
+      ios = IO.select([notifier.to_io, @terminate_r, @trigger_regen_r], [], [], timeout)
+
+      if ios
         if ios.first.include?(notifier.to_io)
           logger.debug(logloc) { "inotify triggered" }
           notifier.process
@@ -293,6 +306,7 @@ class ConfigSkeleton < ServiceSkeleton
   # @see .watch for watching files and directories whose path never changes.
   #
   def watch(*files)
+    return if ENV["DISABLE_INOTIFY"]
     files.each do |f|
       if File.directory?(f)
         notifier.watch(f, :recursive, :create, :modify, :delete, :move) { |ev| logger.info("#{logloc} watcher") { "detected #{ev.flags.join(", ")} on #{ev.watcher.path}/#{ev.name}; regenerating config" } }
@@ -304,22 +318,11 @@ class ConfigSkeleton < ServiceSkeleton
 
   private
 
-  # Register metrics in the ServiceSkeleton metrics registry
-  #
-  # @return [void]
-  #
   def initialize_config_skeleton_metrics
-    @config_generation = Frankenstein::Request.new("#{service_name}_generation", outgoing: false, description: "config generation", registry: metrics)
-
-    metrics.gauge(:"#{service_name}_last_generation_timestamp", "When the last config generation run was made")
-    metrics.gauge(:"#{service_name}_last_change_timestamp", "When the config file was last written to")
-    metrics.counter(:"#{service_name}_reload_total", "How many times we've asked the server to reload")
-    metrics.counter(:"#{service_name}_signals_total", "How many signals have been received (and handled)")
-    metrics.gauge(:"#{service_name}_config_ok", "Whether the last config change was accepted by the server")
-
-    metrics.last_generation_timestamp.set({}, 0)
-    metrics.last_change_timestamp.set({}, 0)
-    metrics.config_ok.set({}, 0)
+    @config_generation = Frankenstein::Request.new("#{self.class.service_name}_generation", outgoing: false, description: "config generation", registry: metrics)
+    metrics.last_generation_timestamp.set(0)
+    metrics.last_change_timestamp.set(0)
+    metrics.config_ok.set(0)
   end
 
   # Write out a config file if one doesn't exist, or do an initial regen run
@@ -334,7 +337,7 @@ class ConfigSkeleton < ServiceSkeleton
     else
       logger.info(logloc) { "No existing config file #{config_file} found; writing one" }
       File.write(config_file, instrumented_config_data)
-      metrics.last_change_timestamp.set({}, Time.now.to_f)
+      metrics.last_change_timestamp.set(Time.now.to_f)
     end
   end
 
@@ -362,6 +365,27 @@ class ConfigSkeleton < ServiceSkeleton
     raise NotImplementedError, "config_data must be implemented in subclass."
   end
 
+  # Run code before the config is regenerated and the config_file
+  # is written.
+  #
+  # @param force_reload [Boolean] Whether the regenerate_config was called with force_reload
+  # @param existing_config_hash [String] MD5 hash of the config file before regeneration.
+  #
+  # @note this can optionally be implemented by subclasses.
+  #
+  def before_regenerate_config(force_reload:, existing_config_hash:, existing_config_data:); end
+
+  # Run code after the config is regenerated and potentially a new file is written.
+  #
+  # @param force_reload [Boolean] Whether the regenerate_config was called with force_reload
+  # @param config_was_different [Boolean] Whether the diff of the old and new config was different.
+  # @param config_was_cycled [Boolean] Whether a new config file was cycled in.
+  # @param new_config_hash [String] MD5 hash of the new config file after write.
+  #
+  # @note this can optionally be implemented by subclasses.
+  #
+  def after_regenerate_config(force_reload:, config_was_different:, config_was_cycled:, new_config_hash:); end
+
   # Verify that the currently running config is acceptable.
   #
   # In the event that a generated config is "bad", it may be possible to detect
@@ -401,7 +425,7 @@ class ConfigSkeleton < ServiceSkeleton
   #
   def instrumented_config_data
     begin
-      @config_generation.measure { config_data.tap { metrics.last_generation_timestamp.set({}, Time.now.to_f) } }
+      @config_generation.measure { config_data.tap { metrics.last_generation_timestamp.set(Time.now.to_f) } }
     rescue => ex
       log_exception(ex, logloc) { "Call to config_data raised exception" }
       nil
@@ -425,12 +449,30 @@ class ConfigSkeleton < ServiceSkeleton
     60
   end
 
+  # How long to ignore signals/notifications after a config regeneration
+  #
+  # Hammering a downstream service with reload requests is often a bad idea.
+  # This method exists to allow subclasses to define a 'cooldown' duration.
+  # After each config regeneration, the config generator will sleep for this
+  # duration, regardless of any CONT signals or inotify events. Those events
+  # will be queued up, and processed at the end of the cooldown.
+  #
+  # @return [Integer] the number of seconds to 'cooldown' for.  This *must* be
+  #   greater than zero, and less than sleep_duration
+  #
+  def cooldown_duration
+    5
+  end
+
   # The instance of INotify::Notifier that is holding our file watches.
   #
   # @return [INotify::Notifier]
   #
   def notifier
     @notifier ||= INotify::Notifier.new
+  rescue NameError
+    raise if !ENV["DISABLE_INOTIFY"]
+    @notifier ||= Struct.new(:to_io).new(IO.pipe[1]) # Stub for macOS development
   end
 
   # Do the hard yards of actually regenerating the config and performing the reload.
@@ -443,31 +485,54 @@ class ConfigSkeleton < ServiceSkeleton
   # @return [void]
   #
   def regenerate_config(force_reload: false)
+    data = File.read(config_file)
+    existing_config_hash = Digest::MD5.hexdigest(data)
+    before_regenerate_config(
+      force_reload: force_reload,
+      existing_config_hash: existing_config_hash,
+      existing_config_data: data
+    )
+
     logger.debug(logloc) { "force? #{force_reload.inspect}" }
-    tmpfile = Tempfile.new(service_name, File.dirname(config_file))
+    tmpfile = Tempfile.new(self.class.service_name, File.dirname(config_file))
     logger.debug(logloc) { "Tempfile is #{tmpfile.path}" }
     unless (new_config = instrumented_config_data).nil?
       File.write(tmpfile.path, new_config)
       tmpfile.close
-      logger.debug(logloc) { require 'digest/md5'; "Existing config hash: #{Digest::MD5.hexdigest(File.read(config_file))}, new config hash: #{Digest::MD5.hexdigest(File.read(tmpfile.path))}" }
+
+      new_config_hash = Digest::MD5.hexdigest(File.read(tmpfile.path))
+      logger.debug(logloc) do
+        "Existing config hash: #{existing_config_hash}, new config hash: #{new_config_hash}"
+      end
 
       match_perms(config_file, tmpfile.path)
 
-      diff = Diffy::Diff.new(config_file, tmpfile.path, source: 'files', context: 3, include_diff_info: true)
-      if diff.to_s != ""
-        logger.info(logloc) { "Config has changed.  Diff:\n#{diff.to_s}" }
+      diff = Diffy::Diff.new(config_file, tmpfile.path, source: 'files', context: 3, include_diff_info: true).to_s
+      config_was_different = diff != ""
+
+      if config_was_different
+        logger.info(logloc) { "Config has changed.  Diff:\n#{diff}" }
       end
 
       if force_reload
         logger.debug(logloc) { "Forcing config reload because force_reload == true" }
       end
 
-      if force_reload || diff.to_s != ""
+      config_was_cycled = false
+      if force_reload || config_was_different
         cycle_config(tmpfile.path)
+        config_was_cycled = true
       end
     end
+
+    after_regenerate_config(
+      force_reload: force_reload,
+      config_was_different: config_was_different,
+      config_was_cycled: config_was_cycled,
+      new_config_hash: new_config_hash
+    )
   ensure
-    metrics.last_change_timestamp.set({}, File.stat(config_file).mtime.to_f)
+    metrics.last_change_timestamp.set(File.stat(config_file).mtime.to_f)
     tmpfile.close rescue nil
     tmpfile.unlink rescue nil
   end
@@ -517,7 +582,7 @@ class ConfigSkeleton < ServiceSkeleton
         logger.debug(logloc) { "Restored previous config file" }
         File.rename(old_copy, config_file)
       end
-      metrics.reload_total.increment(status: "failure")
+      metrics.reload_total.increment(labels: { status: "failure" })
 
       return
     end
@@ -525,21 +590,21 @@ class ConfigSkeleton < ServiceSkeleton
     logger.debug(logloc) { "Server reloaded successfully" }
 
     if config_ok?
-      metrics.config_ok.set({}, 1)
+      metrics.config_ok.set(1)
       logger.debug(logloc) { "Configuration successfully updated." }
-      metrics.reload_total.increment(status: "success")
-      metrics.last_change_timestamp.set({}, Time.now.to_f)
+      metrics.reload_total.increment(labels: { status: "success" })
+      metrics.last_change_timestamp.set(Time.now.to_f)
     else
-      metrics.config_ok.set({}, 0)
+      metrics.config_ok.set(0)
       if config_was_ok
         logger.warn(logloc) { "New config file failed config_ok? test; rolling back to previous known-good config" }
         File.rename(old_copy, config_file)
         reload_server
-        metrics.reload_total.increment(status: "bad-config")
+        metrics.reload_total.increment(labels: { status: "bad-config" })
       else
         logger.warn(logloc) { "New config file failed config_ok? test; leaving new config in place because old config is broken too" }
-        metrics.reload_total.increment(status: "everything-is-awful")
-        metrics.last_change_timestamp.set({}, Time.now.to_f)
+        metrics.reload_total.increment(labels: { status: "everything-is-awful" })
+        metrics.last_change_timestamp.set(Time.now.to_f)
       end
     end
   ensure

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: config_skeleton
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Matt Palmer
+- Discourse Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-26 00:00:00.000000000 Z
+date: 2021-03-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: diffy
@@ -24,20 +25,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: frankenstein
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: rb-inotify
   requirement: !ruby/object:Gem::Requirement
@@ -56,16 +43,16 @@ dependencies:
   name: service_skeleton
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.a
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.a
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -195,6 +182,7 @@ dependencies:
 description: 
 email:
 - matt.palmer@discourse.org
+- team@discourse.org
 executables: []
 extensions: []
 extra_rdoc_files: []