prometheus-config-builder 0.0.15

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8afb279bd7ee0d2f9e233c0224bef803b9694054
+  data.tar.gz: 4af2db65c94a80667f8125fd425d5132e627c550
+SHA512:
+  metadata.gz: 87e8783a009b784505acaebbacc977b95d70b9785fa603e1d6acb9274743d1a07b1c166ae016863d13094b13e2fbe7f963cb21d8e2debfc3ae237ea53bd595eb
+  data.tar.gz: c154ff88582bd2270e2f7bb7cbf91fea11c36e5383168580fd87d26d06d9127cfa1445b8889e25f6bf8744724dc735ef8380a069dc45360d20f3cfa8cdafc935

data/.gitignore

@@ -0,0 +1,2 @@
+*.gem
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fx-tftpd.gemspec
+gemspec

data/README.md

@@ -0,0 +1,22 @@
+
+## Description
+
+prometheus-config-builder can be used to build main prometheus.yaml and related rules yaml files from a combination of different sources such as files in S3 buckets.
+
+It also supports some extended ways to build scrape_configs such as fetching AWS ECS service endpoints.
+
+## Quickstart
+
+1) Install via Rubygems:
+
+    $ gem install prometheus-config-builder
+
+2) Create a prometheus.yaml to be used as a base template
+
+3) Create one or more source yaml files (See test/data/test*.yaml as examples) to be included into your prometheus configuration.
+
+4) Start the program: prometheus-config-builder --prometheus-src=test/data/prometheus.yaml  --dst-dir=tmp --path=test/data/test*.yaml
+
+The program will re-create the config files every n seconds. If you set --pgrep=[STRING] it will try to find your Prometheus instance pid and send a SIGHUP to Prometheus so that it reloads new configurations. Try the --pgrep search string with "pgrep -f [STRING]" to be use you will get just one correct result.
+
+

data/Rakefile

@@ -0,0 +1,35 @@
+require 'rake/testtask'
+
+begin
+  require 'rubygems/tasks'
+  Gem::Tasks.new
+rescue LoadError => e
+  warn e.message
+end
+
+begin
+  require 'yard'
+  YARD::Rake::YardocTask.new
+  task :doc => :yard
+rescue LoadError => e
+  warn e.message
+end
+
+task :default => :test
+
+Rake::TestTask.new do |t|
+  t.libs = ['lib', 'test']
+  t.name = 'test'
+  t.warning = true
+  t.test_files = FileList['test/*.rb']
+end
+
+FileList['test/*.rb'].each do |p|
+  name = p.split('/').last.split('.').first
+  Rake::TestTask.new do |t|
+    t.libs = ['lib', 'test']
+    t.name = "test:#{name}"
+    t.warning = true
+    t.test_files = [p]
+  end
+end

data/bin/prometheus-config-builder

@@ -0,0 +1,128 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'logger'
+require 'optparse'
+require 'prometheus-config-builder'
+require 'pp'
+require 'yaml'
+require 'json'
+require 'prometheus_exporter'
+require 'prometheus_exporter/client'
+require 'prometheus_exporter/server'
+
+config = {}
+config[:paths] ||= []
+config[:root] ||= 'test/data'
+config[:every] = 60
+config[:pgrep] = nil
+
+# Defaults
+op = OptionParser.new do |o|
+  o.banner = "Usage: #{$PROGRAM_NAME} [OPTIONS]"
+  o.on('--prometheus-src=[FILE]', 'Location of prometheus.yaml source config') do |arg|
+    config[:prometheus_src] = arg
+  end
+  o.on('--dst-dir=[FILE]', 'Destination directory') do |arg|
+    config[:dst_dir] = arg
+  end
+  o.on('--path=[PATH]', 'Directory or S3 location for individual config yaml files. Can be set multiple times') do |arg|
+    config[:paths] << arg
+  end
+  o.on('--http PORT', Integer,
+    'Port to listen on for http for Prometheus /metrics api') do |port|
+    abort 'Invalid port' if port < 1 || port > 65535
+    config[:http] = http
+  end
+  o.on('-v', '--verbose', 'Enable verbose output') do
+    config[:verbose] = true
+  end
+  o.on('--every=[seconds]', 'Rerun every n seconds') do |seconds|
+    config[:every] = seconds
+  end
+  o.on('--pgrep=[string]', 'pgrep -f [string] for finding Prometheus process for SIGHUP') do |str|
+    config[:pgrep] = str
+  end
+end
+op.parse!
+
+if config[:verbose]
+  PP.pp(config, STDERR)
+end
+
+abort "You need to set --prometheus-src" if !config[:prometheus_src]
+abort "You need to set --dst_dir" if !config[:dst_dir]
+
+log = Logger.new(STDOUT)
+log.level = config[:verbose] ? Logger::DEBUG : Logger::INFO
+log.formatter = lambda do |s, d, p, m|
+  "#{d.strftime('%Y-%m-%d %H:%M:%S.%3N')} | #{s.ljust(5)} | #{m}\n"
+end
+config[:logger] = log
+PrometheusConfigBuilderLogger.logger = log
+
+if !File.exist?(config[:dst_dir])
+  log.fatal("The --dst-dir #{config[:dst_dir]} does not exists. Create it first.")
+  exit!(1)
+end
+
+server = PrometheusExporter::Server::WebServer.new port: 12345
+PrometheusExporter::Client.default = PrometheusExporter::LocalClient.new(collector: server.collector)
+server.start
+
+something_changed_count = PrometheusExporter::Client.default.register(:counter, "prometheusconfigbuilder_changes_count", "Number of times configuration has changed.")
+error_count = PrometheusExporter::Client.default.register(:counter, "prometheusconfigbuilder_errors_count", "Number of exceptions during evaluation.")
+iteration_count = PrometheusExporter::Client.default.register(:counter, "prometheusconfigbuilder_iteration_count", "Number of times configuration has changed.")
+iteration_duration = PrometheusExporter::Client.default.register(:gauge, "prometheusconfigbuilder_last_iteration_duration", "Duration (in seconds) of the last full config iteration")
+
+error_count.observe(0)
+
+builder = PrometheusConfigBuilder::Builder.new(config[:prometheus_src], config[:dst_dir])
+
+config[:paths].each do |path|
+  log.info("Adding path #{path} to config discovery list")
+  builder.add_path(path)
+end
+
+
+loop do
+  starting = Time.now
+  begin
+    something_changed = builder.write_out()
+  rescue Exception => e
+    log.warn("Error while building config: #{e}")
+    puts e.backtrace
+    error_count.increment
+    sleep(config[:every])
+    next
+  end
+
+  something_changed_count.increment if something_changed
+  iteration_count.increment
+
+  # Send a SIGHUP signal to Prometheus so that it knows to reload config files
+  begin
+    if something_changed && config[:pgrep] != nil
+      pid = `pgrep -f "#{config[:pgrep]}"`.split("\n").first.to_i
+      if pid
+        log.info("Sending SIGHUP signal to Prometheus at pid #{pid}.")
+        Process.kill "HUP", pid
+      end
+    end
+  rescue Exception => e
+    log.warn("Unable to send SIGHUP signal to Prometheus: #{e}")
+    puts e.backtrace
+  end
+
+  log.info("Sleeping for #{config[:every]} second and starting then again.")
+  if log.level == Logger::INFO
+    log.info("Setting log level to WARN for the rest of the program execution. Use -v to overwrite this.")
+    log.level = Logger::WARN
+  end
+
+  ending = Time.now
+  iteration_duration.observe(ending-starting)
+
+  sleep(config[:every])
+end

data/lib/prometheus-config-builder.rb

@@ -0,0 +1 @@
+require 'prometheus-config-builder/prometheus-config-builder'

data/lib/prometheus-config-builder/logger.rb

@@ -0,0 +1,15 @@
+
+module PrometheusConfigBuilderLogger
+  def logger
+    PrometheusConfigBuilderLogger.logger
+  end
+
+  def self.logger=(logger)
+    @logger = logger
+  end
+
+  def self.logger
+    @logger ||= Logger.new(STDOUT)
+  end
+
+end

data/lib/prometheus-config-builder/prometheus-config-builder.rb

@@ -0,0 +1,252 @@
+# After https://www.ietf.org/rfc/rfc1350.txt
+#
+
+require 'erb'
+require 'net/http'
+require 'uri'
+require 'json'
+require 'pp'
+require 'yaml'
+require 'time'
+require 'fileutils'
+require 'digest'
+require 'logger'
+require_relative './logger.rb'
+require_relative './scrape_passthrough.rb'
+require_relative './scrape_ecs.rb'
+require 'prometheus_exporter'
+require 'prometheus_exporter/client'
+
+module PrometheusConfigBuilder
+
+
+  class ConfigFile
+    include PrometheusConfigBuilderLogger
+    @data = {}
+
+    attr_reader :basename
+
+    def open(filename)
+      filename = File.expand_path(filename)
+      @source = "file://#{filename}"
+      @basename = File.basename(filename)
+      @data = YAML.load_file(filename)
+    end
+
+    def load(file)
+      @source = "file://#{file[:filename]}"
+      @basename = file[:filename]
+      @data = YAML.load(file[:contents])
+    end
+
+    def config_rules
+      @data['config_rules']
+    end
+
+    def scrape_configs
+      @data['scrape_configs']
+    end
+
+    def get_scrape_configs(dst_prefix)
+      configs = []
+
+      @data['scrape_configs'].each do |config|
+        case config["type"]
+        when "passthrough"
+          configs << ScrapeConfigPassthrough::handle(config)
+        when "ecs-tasks"
+          configs << ScrapeConfigECS::handle(@basename, config, dst_prefix)
+        else
+          raise "Unknown scrape_config type #{config["type"]}"
+        end
+      end
+
+      return configs
+    end
+
+    def write_rules(path)
+      filename = path + "/" + @basename
+      File.open(filename, "w") do |file|
+        file.write("# AUTOGENERATED FILE, DO NOT MODIFY\n")
+        file.write("#\n")
+        file.write("# This file was automatically generated by prometheus-config-from-s3 at #{Time.now.utc.iso8601}\n")
+        file.write("# based on #{@source}\n")
+        file.write("#\n\n")
+        file.write(config_rules.to_yaml)
+      end
+
+      return Digest::SHA256.digest config_rules.to_yaml
+    end
+  end
+
+  class ConfigDiscover
+    include PrometheusConfigBuilderLogger
+    def discover(glob)
+
+      files = []
+      if m = glob.match(/s3:\/\/([^\/]+)\/(.*)/)
+        bucket_name = m[1]
+        prefix = m[2]
+
+        x = $VERBOSE
+        $VERBOSE = nil
+        require 'aws-sdk'
+        s3 = Aws::S3::Client.new
+        $VERBOSE = x
+        key = []
+        resp = s3.list_objects_v2({
+          bucket: bucket_name,
+          prefix: prefix
+        })
+
+        resp.contents.each do |file|
+          contents = s3.get_object({
+            key: file.key,
+            bucket: bucket_name
+          })
+
+          logger.debug("Found file \"#{file.key}\" from S3 bucket #{bucket_name}")
+          files << {
+            filename: File.basename(file.key),
+            contents: contents.body.string,
+          }
+        end
+      else
+        Dir.glob(glob).each do |file|
+          files << {
+            filename: File.basename(file),
+            contents: File.read(file)
+          }
+        end
+
+      end
+
+      return files
+    end
+  end
+
+
+  class ConfigFiles
+
+    def initialize(prometheus_yaml_filename)
+      @files = []
+      @prometheus_yaml_filename = prometheus_yaml_filename
+      @rules_dir = nil
+      @scrape_files_dir = nil
+    end
+
+    def add(file)
+      @files << file
+    end
+
+    def write_rules(path)
+
+      hash = ""
+      @files.each do |file|
+        hash += file.write_rules(path)
+      end
+
+      return hash
+    end
+
+    def set_rules_dir(rules_dir)
+      @rules_dir = rules_dir
+    end
+
+    def set_scrape_files_dir(scrape_files_dir)
+      @scrape_files_dir = scrape_files_dir
+    end
+
+    def write_prometheus_yaml(destination, scrape_files_dir)
+      data = YAML.load_file(@prometheus_yaml_filename)
+
+      if !data["scrape_configs"]
+        data["scrape_configs"] = []
+      end
+
+      FileUtils.mkdir_p(scrape_files_dir) if !File.exist?(scrape_files_dir)
+
+      @files.each do |file|
+        configs = file.get_scrape_configs(scrape_files_dir + "/" + File.basename(file.basename, ".*"))
+        data["scrape_configs"].push(*configs)
+      end
+
+      if !data["rule_files"] or data["rule_files"].class != Array
+        data["rule_files"] = []
+      end
+
+      if @rules_dir != nil && !data["rule_files"].include?(@rules_dir)
+        data["rule_files"] << File.expand_path(@rules_dir) + "/*.yaml"
+      end
+
+      File.open(destination, "w") do |file|
+        file.write("# AUTOGENERATED FILE, DO NOT MODIFY\n")
+        file.write("#\n")
+        file.write("# This file was automatically generated by prometheus-config-from-s3 at #{Time.now.utc.iso8601}\n")
+        file.write("# based on #{@prometheus_yaml_filename}\n")
+        file.write("#\n\n")
+        file.write(data.to_yaml)
+      end
+
+      return Digest::SHA256.digest data.to_yaml
+    end
+  end
+
+  class Builder
+    include PrometheusConfigBuilderLogger
+
+    def initialize(prometheus_src, dst_dir)
+      @prometheus_src = prometheus_src
+      @dst_dir = dst_dir
+
+      @discoverer = ConfigDiscover.new
+      @paths = []
+      @last_hash = ""
+
+      @@config_files = PrometheusExporter::Client.default.register(:gauge, "prometheusconfigbuilder_config_files", "Number of found config files")
+
+    end
+
+    def add_path(path)
+      @paths << path
+    end
+
+    def write_out
+      hash = ""
+      cfs = ConfigFiles.new(@prometheus_src)
+      files = []
+      @paths.each do |path|
+        found_files = @discoverer.discover(path)
+        logger.info("Found #{files.length} files from path #{path}")
+        files.push(*found_files)
+      end
+
+      files.each do |file|
+        cf = ConfigFile.new
+        cf.load(file)
+        cfs.add(cf)
+      end
+
+      @@config_files.observe(files.length)
+
+      rules_dir = @dst_dir + "/rules"
+      scrape_files_dir = @dst_dir + "/scrape_files"
+      cfs.set_rules_dir(rules_dir)
+      cfs.set_scrape_files_dir(scrape_files_dir)
+      hash += cfs.write_prometheus_yaml(@dst_dir + "/prometheus.yaml", scrape_files_dir)
+
+      FileUtils.mkdir_p(rules_dir) unless File.exist?(rules_dir)
+      FileUtils.mkdir_p(scrape_files_dir) unless File.exist?(scrape_files_dir)
+      hash += cfs.write_rules(rules_dir)
+
+      # Determine if something had changed since the last write_out based on the generated hashes
+      if hash != @last_hash
+        @last_hash = hash
+        return true
+      else
+        return false
+      end
+    end
+  end
+
+end

data/lib/prometheus-config-builder/scrape_ecs.rb

@@ -0,0 +1,123 @@
+require 'pp'
+require_relative './logger.rb'
+
+module PrometheusConfigBuilder
+
+  class ScrapeConfigECS
+    extend PrometheusConfigBuilderLogger
+
+    def self.handle(basename, config, dst_prefix)
+
+      x = $VERBOSE
+      $VERBOSE = nil
+      require 'aws-sdk'
+
+      if !config["region"]
+        logger.warn("File #{basename}: the scrape_configs of type:ecs-tasks doesn't have \"region\" field set. Ignoring!")
+        return nil
+
+      end
+
+      @@ecs = Aws::ECS::Client.new({
+        region:config["region"]
+      })
+      $VERBOSE = x
+      tasks = get_tasks(config["cluster"], config["service"])
+      endpoints = get_task_endpoints(config["cluster"], tasks, config["labels"], config["metrics_port"])
+
+      if !config["job_name"]
+        logger.warn("File #{basename}: the scrape_configs of type:ecs-tasks doesn't have \"job_name\" field set. Ignoring!")
+        return nil
+      end
+
+      file = File.expand_path(dst_prefix + "_" + config["job_name"] + ".json")
+      File.open(file, "w") do |file|
+        file.write(endpoints.to_json)
+      end
+
+      # Make copy of the settings and remove our custom properties from it.
+      # The rest user can set just as he wants according to the Prometheus schema.
+      # See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config
+      settings = config.clone
+      settings.delete("type")
+      settings.delete("cluster")
+      settings.delete("service")
+      settings.delete("metrics_port")
+      settings.delete("labels")
+      settings.delete("region")
+      settings["file_sd_configs"] = [
+        "files" => [
+          file
+        ]
+      ]
+
+      return settings
+    end
+
+    def self.get_tasks(cluster, service_name)
+      tasks = []
+      last_result = Aws::ECS::Types::ListTasksResponse.new
+      last_result.next_token = nil
+      loop do
+        begin
+          options = {
+            cluster: cluster,
+            service_name: service_name,
+            next_token: last_result.next_token
+          }
+          last_result = @@ecs.list_tasks(options)
+          if last_result && last_result.task_arns
+            tasks.push(*last_result.task_arns)
+          end
+        rescue Aws::ECS::Errors::ServiceError => e
+          print "Error listing ecs tasks: #{e}"
+          sleep(1)
+          next
+        end
+        break if !last_result.next_token
+      end
+
+      return tasks
+    end
+
+    def self.get_task_endpoints(cluster, tasks, common_labels, metrics_port)
+      endpoints = []
+      task_chunk = tasks.pop(100)
+      loop do
+        begin
+          options = {
+            cluster: cluster,
+            tasks: task_chunk,
+          }
+          result = @@ecs.describe_tasks(options)
+          if result && result.tasks
+            result.tasks.each do |task|
+              # FIXME: This assumes somewhat on the ip structure.
+              ip = task.containers[0].network_interfaces[0].private_ipv_4_address
+
+              if metrics_port
+                ip = ip + ":" + metrics_port.to_s
+              end
+              labels = common_labels.clone
+              labels["ecs_arn"] = task.task_arn
+              endpoints << {
+                "targets" => [ip],
+                "labels" => labels
+              }
+            end
+          end
+        rescue Aws::ECS::Errors::ServiceError => e
+          print "Error listing ecs tasks: #{e}\n"
+          sleep(1)
+          next
+        end
+        break if tasks.length == 0
+        task_chunk = tasks.pop(100)
+      end
+      return endpoints
+    end
+
+
+
+  end
+end

data/lib/prometheus-config-builder/scrape_passthrough.rb

@@ -0,0 +1,11 @@
+module PrometheusConfigBuilder
+
+  class ScrapeConfigPassthrough
+    def self.handle(config)
+      settings = config.clone
+      settings.delete("type")
+      return settings
+    end
+  end
+
+end

data/lib/prometheus-config-builder/version.rb

@@ -0,0 +1,4 @@
+module TFTP
+  # Current version string.
+  VERSION = '0.1'
+end

data/prometheus-config-builder.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+#
+
+Gem::Specification.new do |s|
+  s.name          = 'prometheus-config-builder'
+  s.version       = '0.0.15'
+  s.date          = Time.now
+
+  s.summary       = %q{Template based config generation}
+  s.files         = `git ls-files`.split("\n")
+  s.executables   = ['prometheus-config-builder']
+  s.test_files    = s.files.grep(%r{^test/})
+  s.require_paths = ['lib']
+  s.authors       = "Juho Mäkinen juho.makinen@gmail.com"
+
+  s.required_ruby_version = '>= 2.0.0'
+
+  s.add_dependency 'aws-sdk', '~> 3'
+  s.add_dependency 'prometheus_exporter', '>= 0.4.13'
+
+  s.add_development_dependency 'rubygems-tasks', '~> 0.2'
+  s.add_development_dependency 'minitest', '~> 5.4'
+  s.add_development_dependency 'minitest-reporters'
+  s.add_development_dependency 'rake', '~> 10.0'
+end

data/test/configdiscovery.rb

@@ -0,0 +1,121 @@
+require 'minitest/autorun'
+
+require 'prometheus-config-builder'
+require 'fileutils'
+
+class ConfigDiscoveryTest < Minitest::Test
+
+  class MockDiscovery < PrometheusConfigBuilder::ConfigFile
+
+    attr_accessor :files
+    @files = []
+
+    def get_files()
+      @files
+    end
+  end
+
+  def test_can_get_rules_from_file
+    f = PrometheusConfigBuilder::ConfigFile.new()
+    f.open('test/data/test1.yaml')
+
+    assert_equal "from_terraform.generic.rules", f.config_rules[0]['name']
+    assert_equal "passthrough", f.scrape_configs[0]['type']
+  end
+
+  def test_scrape_config
+    f = PrometheusConfigBuilder::ConfigFile.new()
+    f.open('test/data/test1.yaml')
+
+    configs = f.get_scrape_configs('tmp/test')
+
+    assert_equal "discovery 1", configs[0]["job_name"]
+    assert_equal "localhost:9090", configs[0]["static_configs"][0]["targets"][0]
+
+  end
+
+  def test_can_write_rules_to_file
+    f = PrometheusConfigBuilder::ConfigFile.new()
+    f.open('test/data/test1.yaml')
+
+    filename = './tmp/test1.yaml'
+    FileUtils.mkdir('tmp') if not File.directory?('tmp')
+    File.delete(filename) if File.exist?(filename)
+    f.write_rules('./tmp')
+
+    contents = YAML.load_file(filename)
+
+    assert_equal "from_terraform.generic.rules",  contents[0]["name"]
+
+  end
+
+  def test_can_get_all_files_from_mock
+
+      f1 = PrometheusConfigBuilder::ConfigFile.new
+      f2 = PrometheusConfigBuilder::ConfigFile.new
+      f1.open('test/data/test1.yaml')
+      f2.open('test/data/test2.yaml')
+
+      d = PrometheusConfigBuilder::ConfigFiles.new('test/data/prometheus.yaml')
+      d.add(f1)
+      d.add(f2)
+
+      FileUtils.rm_r('tmp/test_can_get_all_files_from_mock') if File.exist?('tmp/test_can_get_all_files_from_mock')
+      FileUtils.mkdir_p('tmp/test_can_get_all_files_from_mock')
+
+      d.write_rules('./tmp/test_can_get_all_files_from_mock')
+
+      # Test over all files
+      filename = './tmp/test_can_get_all_files_from_mock/test1.yaml'
+      contents = YAML.load_file(filename)
+      assert_equal "from_terraform.generic.rules",  contents[0]["name"]
+
+      filename = './tmp/test_can_get_all_files_from_mock/test2.yaml'
+      contents = YAML.load_file(filename)
+      assert_equal "something_else",  contents[0]["name"]
+
+      # Write main prometheus.yaml
+      filename = './tmp/test_can_get_all_files_from_mock/prometheus.yaml'
+      d.write_prometheus_yaml(filename, 'tmp/')
+
+      contents = YAML.load_file(filename)
+      assert_equal "15s", contents["global"]["scrape_interval"]
+      assert_equal "prometheus", contents["scrape_configs"][0]["job_name"]
+      assert_equal "alertmanager", contents["scrape_configs"][1]["job_name"]
+
+      assert_equal "discovery 1", contents["scrape_configs"][2]["job_name"]
+
+  end
+
+  def test_discover_config_files
+    x = PrometheusConfigBuilder::ConfigDiscover.new
+
+    files = x.discover("./test/data/test*.yaml")
+    assert_equal 2, files.length
+
+  end
+
+  def test_config_builder
+    x = PrometheusConfigBuilder::Builder.new("./test/data/prometheus.yaml", "tmp/")
+    x.add_path("./test/data/test*.yaml")
+
+    ret = x.write_out()
+    assert_equal true, ret # true as in config files were modified or at least rewritten
+
+    assert_equal true, File.exist?("tmp/prometheus.yaml")
+    assert_equal true, File.exist?("tmp/rules/test1.yaml")
+    assert_equal true, File.exist?("tmp/rules/test2.yaml")
+
+  end
+
+  def test_config_builder_knows_when_to_refresh
+    x = PrometheusConfigBuilder::Builder.new("./test/data/prometheus.yaml", "tmp/")
+    x.add_path("./test/data/test*.yaml")
+
+    x.write_out()
+    ret = x.write_out()
+    assert_equal false, ret # Nothing had changed, no need to rewrite
+
+  end
+
+end

data/test/data/prometheus.yaml

@@ -0,0 +1,22 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+alerting:
+  alertmanagers:
+  - static_configs:
+    - targets:
+      - localhost:9093
+
+rule_files:
+- "/mnt/data/prometheus/rules/*.yaml"
+
+scrape_configs:
+- job_name: prometheus
+  static_configs:
+  - targets:
+    - localhost:9090
+- job_name: alertmanager
+  static_configs:
+  - targets:
+    - localhost:9093

data/test/data/test1.yaml

@@ -0,0 +1,17 @@
+scrape_configs:
+  - type: passthrough
+    job_name: discovery 1
+    static_configs:
+      - targets: ['localhost:9090']
+
+
+config_rules:
+  - name: from_terraform.generic.rules
+    rules:
+    - alert: DeadMansSwitch
+      annotations:
+        description: This is a DeadMansSwitch meant to ensure that the entire Alerting pipeline is functional.
+        summary: Alerting DeadMansSwitch
+      expr: vector(1)
+      labels:
+        severity: none

data/test/data/test2.yaml

@@ -0,0 +1,17 @@
+scrape_configs:
+  - type: passthrough
+    job_name: discovery 2
+    static_configs:
+      - targets: ['localhost:9090']
+
+
+config_rules:
+  - name: something_else
+    rules:
+    - alert: DeadMansSwitch
+      annotations:
+        description: This is a DeadMansSwitch meant to ensure that the entire Alerting pipeline is functional.
+        summary: Alerting DeadMansSwitch
+      expr: vector(1)
+      labels:
+        severity: none

data/test/test_helper.rb

@@ -0,0 +1,4 @@
+require "minitest/reporters"
+
+Minitest::Reporters.use! Minitest::Reporters::SpecReporter.new
+

metadata

@@ -0,0 +1,149 @@
+--- !ruby/object:Gem::Specification
+name: prometheus-config-builder
+version: !ruby/object:Gem::Version
+  version: 0.0.15
+platform: ruby
+authors:
+- Juho Mäkinen juho.makinen@gmail.com
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-09-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: prometheus_exporter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.13
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.13
+- !ruby/object:Gem::Dependency
+  name: rubygems-tasks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.4'
+- !ruby/object:Gem::Dependency
+  name: minitest-reporters
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: 
+email: 
+executables:
+- prometheus-config-builder
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- Rakefile
+- bin/prometheus-config-builder
+- lib/prometheus-config-builder.rb
+- lib/prometheus-config-builder/logger.rb
+- lib/prometheus-config-builder/prometheus-config-builder.rb
+- lib/prometheus-config-builder/scrape_ecs.rb
+- lib/prometheus-config-builder/scrape_passthrough.rb
+- lib/prometheus-config-builder/version.rb
+- prometheus-config-builder.gemspec
+- test/configdiscovery.rb
+- test/data/prometheus.yaml
+- test/data/test1.yaml
+- test/data/test2.yaml
+- test/test_helper.rb
+homepage: 
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: Template based config generation
+test_files:
+- test/configdiscovery.rb
+- test/data/prometheus.yaml
+- test/data/test1.yaml
+- test/data/test2.yaml
+- test/test_helper.rb