prometheus-config-builder 0.0.15

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 8afb279bd7ee0d2f9e233c0224bef803b9694054
4
+ data.tar.gz: 4af2db65c94a80667f8125fd425d5132e627c550
5
+ SHA512:
6
+ metadata.gz: 87e8783a009b784505acaebbacc977b95d70b9785fa603e1d6acb9274743d1a07b1c166ae016863d13094b13e2fbe7f963cb21d8e2debfc3ae237ea53bd595eb
7
+ data.tar.gz: c154ff88582bd2270e2f7bb7cbf91fea11c36e5383168580fd87d26d06d9127cfa1445b8889e25f6bf8744724dc735ef8380a069dc45360d20f3cfa8cdafc935
@@ -0,0 +1,2 @@
1
+ *.gem
2
+ tmp
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fx-tftpd.gemspec
4
+ gemspec
@@ -0,0 +1,22 @@
1
+
2
+ ## Description
3
+
4
+ prometheus-config-builder can be used to build main prometheus.yaml and related rules yaml files from a combination of different sources such as files in S3 buckets.
5
+
6
+ It also supports some extended ways to build scrape_configs such as fetching AWS ECS service endpoints.
7
+
8
+ ## Quickstart
9
+
10
+ 1) Install via Rubygems:
11
+
12
+ $ gem install prometheus-config-builder
13
+
14
+ 2) Create a prometheus.yaml to be used as a base template
15
+
16
+ 3) Create one or more source yaml files (See test/data/test*.yaml as examples) to be included into your prometheus configuration.
17
+
18
+ 4) Start the program: prometheus-config-builder --prometheus-src=test/data/prometheus.yaml --dst-dir=tmp --path=test/data/test*.yaml
19
+
20
+ The program will re-create the config files every n seconds. If you set --pgrep=[STRING] it will try to find your Prometheus instance pid and send a SIGHUP to Prometheus so that it reloads new configurations. Try the --pgrep search string with "pgrep -f [STRING]" to be use you will get just one correct result.
21
+
22
+
@@ -0,0 +1,35 @@
1
+ require 'rake/testtask'
2
+
3
+ begin
4
+ require 'rubygems/tasks'
5
+ Gem::Tasks.new
6
+ rescue LoadError => e
7
+ warn e.message
8
+ end
9
+
10
+ begin
11
+ require 'yard'
12
+ YARD::Rake::YardocTask.new
13
+ task :doc => :yard
14
+ rescue LoadError => e
15
+ warn e.message
16
+ end
17
+
18
+ task :default => :test
19
+
20
+ Rake::TestTask.new do |t|
21
+ t.libs = ['lib', 'test']
22
+ t.name = 'test'
23
+ t.warning = true
24
+ t.test_files = FileList['test/*.rb']
25
+ end
26
+
27
+ FileList['test/*.rb'].each do |p|
28
+ name = p.split('/').last.split('.').first
29
+ Rake::TestTask.new do |t|
30
+ t.libs = ['lib', 'test']
31
+ t.name = "test:#{name}"
32
+ t.warning = true
33
+ t.test_files = [p]
34
+ end
35
+ end
@@ -0,0 +1,128 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ $LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib')
4
+
5
+ require 'logger'
6
+ require 'optparse'
7
+ require 'prometheus-config-builder'
8
+ require 'pp'
9
+ require 'yaml'
10
+ require 'json'
11
+ require 'prometheus_exporter'
12
+ require 'prometheus_exporter/client'
13
+ require 'prometheus_exporter/server'
14
+
15
+ config = {}
16
+ config[:paths] ||= []
17
+ config[:root] ||= 'test/data'
18
+ config[:every] = 60
19
+ config[:pgrep] = nil
20
+
21
+ # Defaults
22
+ op = OptionParser.new do |o|
23
+ o.banner = "Usage: #{$PROGRAM_NAME} [OPTIONS]"
24
+ o.on('--prometheus-src=[FILE]', 'Location of prometheus.yaml source config') do |arg|
25
+ config[:prometheus_src] = arg
26
+ end
27
+ o.on('--dst-dir=[FILE]', 'Destination directory') do |arg|
28
+ config[:dst_dir] = arg
29
+ end
30
+ o.on('--path=[PATH]', 'Directory or S3 location for individual config yaml files. Can be set multiple times') do |arg|
31
+ config[:paths] << arg
32
+ end
33
+ o.on('--http PORT', Integer,
34
+ 'Port to listen on for http for Prometheus /metrics api') do |port|
35
+ abort 'Invalid port' if port < 1 || port > 65535
36
+ config[:http] = http
37
+ end
38
+ o.on('-v', '--verbose', 'Enable verbose output') do
39
+ config[:verbose] = true
40
+ end
41
+ o.on('--every=[seconds]', 'Rerun every n seconds') do |seconds|
42
+ config[:every] = seconds
43
+ end
44
+ o.on('--pgrep=[string]', 'pgrep -f [string] for finding Prometheus process for SIGHUP') do |str|
45
+ config[:pgrep] = str
46
+ end
47
+ end
48
+ op.parse!
49
+
50
+ if config[:verbose]
51
+ PP.pp(config, STDERR)
52
+ end
53
+
54
+ abort "You need to set --prometheus-src" if !config[:prometheus_src]
55
+ abort "You need to set --dst_dir" if !config[:dst_dir]
56
+
57
+ log = Logger.new(STDOUT)
58
+ log.level = config[:verbose] ? Logger::DEBUG : Logger::INFO
59
+ log.formatter = lambda do |s, d, p, m|
60
+ "#{d.strftime('%Y-%m-%d %H:%M:%S.%3N')} | #{s.ljust(5)} | #{m}\n"
61
+ end
62
+ config[:logger] = log
63
+ PrometheusConfigBuilderLogger.logger = log
64
+
65
+ if !File.exist?(config[:dst_dir])
66
+ log.fatal("The --dst-dir #{config[:dst_dir]} does not exists. Create it first.")
67
+ exit!(1)
68
+ end
69
+
70
+ server = PrometheusExporter::Server::WebServer.new port: 12345
71
+ PrometheusExporter::Client.default = PrometheusExporter::LocalClient.new(collector: server.collector)
72
+ server.start
73
+
74
+ something_changed_count = PrometheusExporter::Client.default.register(:counter, "prometheusconfigbuilder_changes_count", "Number of times configuration has changed.")
75
+ error_count = PrometheusExporter::Client.default.register(:counter, "prometheusconfigbuilder_errors_count", "Number of exceptions during evaluation.")
76
+ iteration_count = PrometheusExporter::Client.default.register(:counter, "prometheusconfigbuilder_iteration_count", "Number of times configuration has changed.")
77
+ iteration_duration = PrometheusExporter::Client.default.register(:gauge, "prometheusconfigbuilder_last_iteration_duration", "Duration (in seconds) of the last full config iteration")
78
+
79
+ error_count.observe(0)
80
+
81
+ builder = PrometheusConfigBuilder::Builder.new(config[:prometheus_src], config[:dst_dir])
82
+
83
+ config[:paths].each do |path|
84
+ log.info("Adding path #{path} to config discovery list")
85
+ builder.add_path(path)
86
+ end
87
+
88
+
89
+ loop do
90
+ starting = Time.now
91
+ begin
92
+ something_changed = builder.write_out()
93
+ rescue Exception => e
94
+ log.warn("Error while building config: #{e}")
95
+ puts e.backtrace
96
+ error_count.increment
97
+ sleep(config[:every])
98
+ next
99
+ end
100
+
101
+ something_changed_count.increment if something_changed
102
+ iteration_count.increment
103
+
104
+ # Send a SIGHUP signal to Prometheus so that it knows to reload config files
105
+ begin
106
+ if something_changed && config[:pgrep] != nil
107
+ pid = `pgrep -f "#{config[:pgrep]}"`.split("\n").first.to_i
108
+ if pid
109
+ log.info("Sending SIGHUP signal to Prometheus at pid #{pid}.")
110
+ Process.kill "HUP", pid
111
+ end
112
+ end
113
+ rescue Exception => e
114
+ log.warn("Unable to send SIGHUP signal to Prometheus: #{e}")
115
+ puts e.backtrace
116
+ end
117
+
118
+ log.info("Sleeping for #{config[:every]} second and starting then again.")
119
+ if log.level == Logger::INFO
120
+ log.info("Setting log level to WARN for the rest of the program execution. Use -v to overwrite this.")
121
+ log.level = Logger::WARN
122
+ end
123
+
124
+ ending = Time.now
125
+ iteration_duration.observe(ending-starting)
126
+
127
+ sleep(config[:every])
128
+ end
@@ -0,0 +1 @@
1
+ require 'prometheus-config-builder/prometheus-config-builder'
@@ -0,0 +1,15 @@
1
+
2
+ module PrometheusConfigBuilderLogger
3
+ def logger
4
+ PrometheusConfigBuilderLogger.logger
5
+ end
6
+
7
+ def self.logger=(logger)
8
+ @logger = logger
9
+ end
10
+
11
+ def self.logger
12
+ @logger ||= Logger.new(STDOUT)
13
+ end
14
+
15
+ end
@@ -0,0 +1,252 @@
1
+ # After https://www.ietf.org/rfc/rfc1350.txt
2
+ #
3
+
4
+ require 'erb'
5
+ require 'net/http'
6
+ require 'uri'
7
+ require 'json'
8
+ require 'pp'
9
+ require 'yaml'
10
+ require 'time'
11
+ require 'fileutils'
12
+ require 'digest'
13
+ require 'logger'
14
+ require_relative './logger.rb'
15
+ require_relative './scrape_passthrough.rb'
16
+ require_relative './scrape_ecs.rb'
17
+ require 'prometheus_exporter'
18
+ require 'prometheus_exporter/client'
19
+
20
+ module PrometheusConfigBuilder
21
+
22
+
23
+ class ConfigFile
24
+ include PrometheusConfigBuilderLogger
25
+ @data = {}
26
+
27
+ attr_reader :basename
28
+
29
+ def open(filename)
30
+ filename = File.expand_path(filename)
31
+ @source = "file://#{filename}"
32
+ @basename = File.basename(filename)
33
+ @data = YAML.load_file(filename)
34
+ end
35
+
36
+ def load(file)
37
+ @source = "file://#{file[:filename]}"
38
+ @basename = file[:filename]
39
+ @data = YAML.load(file[:contents])
40
+ end
41
+
42
+ def config_rules
43
+ @data['config_rules']
44
+ end
45
+
46
+ def scrape_configs
47
+ @data['scrape_configs']
48
+ end
49
+
50
+ def get_scrape_configs(dst_prefix)
51
+ configs = []
52
+
53
+ @data['scrape_configs'].each do |config|
54
+ case config["type"]
55
+ when "passthrough"
56
+ configs << ScrapeConfigPassthrough::handle(config)
57
+ when "ecs-tasks"
58
+ configs << ScrapeConfigECS::handle(@basename, config, dst_prefix)
59
+ else
60
+ raise "Unknown scrape_config type #{config["type"]}"
61
+ end
62
+ end
63
+
64
+ return configs
65
+ end
66
+
67
+ def write_rules(path)
68
+ filename = path + "/" + @basename
69
+ File.open(filename, "w") do |file|
70
+ file.write("# AUTOGENERATED FILE, DO NOT MODIFY\n")
71
+ file.write("#\n")
72
+ file.write("# This file was automatically generated by prometheus-config-from-s3 at #{Time.now.utc.iso8601}\n")
73
+ file.write("# based on #{@source}\n")
74
+ file.write("#\n\n")
75
+ file.write(config_rules.to_yaml)
76
+ end
77
+
78
+ return Digest::SHA256.digest config_rules.to_yaml
79
+ end
80
+ end
81
+
82
+ class ConfigDiscover
83
+ include PrometheusConfigBuilderLogger
84
+ def discover(glob)
85
+
86
+ files = []
87
+ if m = glob.match(/s3:\/\/([^\/]+)\/(.*)/)
88
+ bucket_name = m[1]
89
+ prefix = m[2]
90
+
91
+ x = $VERBOSE
92
+ $VERBOSE = nil
93
+ require 'aws-sdk'
94
+ s3 = Aws::S3::Client.new
95
+ $VERBOSE = x
96
+ key = []
97
+ resp = s3.list_objects_v2({
98
+ bucket: bucket_name,
99
+ prefix: prefix
100
+ })
101
+
102
+ resp.contents.each do |file|
103
+ contents = s3.get_object({
104
+ key: file.key,
105
+ bucket: bucket_name
106
+ })
107
+
108
+ logger.debug("Found file \"#{file.key}\" from S3 bucket #{bucket_name}")
109
+ files << {
110
+ filename: File.basename(file.key),
111
+ contents: contents.body.string,
112
+ }
113
+ end
114
+ else
115
+ Dir.glob(glob).each do |file|
116
+ files << {
117
+ filename: File.basename(file),
118
+ contents: File.read(file)
119
+ }
120
+ end
121
+
122
+ end
123
+
124
+ return files
125
+ end
126
+ end
127
+
128
+
129
+ class ConfigFiles
130
+
131
+ def initialize(prometheus_yaml_filename)
132
+ @files = []
133
+ @prometheus_yaml_filename = prometheus_yaml_filename
134
+ @rules_dir = nil
135
+ @scrape_files_dir = nil
136
+ end
137
+
138
+ def add(file)
139
+ @files << file
140
+ end
141
+
142
+ def write_rules(path)
143
+
144
+ hash = ""
145
+ @files.each do |file|
146
+ hash += file.write_rules(path)
147
+ end
148
+
149
+ return hash
150
+ end
151
+
152
+ def set_rules_dir(rules_dir)
153
+ @rules_dir = rules_dir
154
+ end
155
+
156
+ def set_scrape_files_dir(scrape_files_dir)
157
+ @scrape_files_dir = scrape_files_dir
158
+ end
159
+
160
+ def write_prometheus_yaml(destination, scrape_files_dir)
161
+ data = YAML.load_file(@prometheus_yaml_filename)
162
+
163
+ if !data["scrape_configs"]
164
+ data["scrape_configs"] = []
165
+ end
166
+
167
+ FileUtils.mkdir_p(scrape_files_dir) if !File.exist?(scrape_files_dir)
168
+
169
+ @files.each do |file|
170
+ configs = file.get_scrape_configs(scrape_files_dir + "/" + File.basename(file.basename, ".*"))
171
+ data["scrape_configs"].push(*configs)
172
+ end
173
+
174
+ if !data["rule_files"] or data["rule_files"].class != Array
175
+ data["rule_files"] = []
176
+ end
177
+
178
+ if @rules_dir != nil && !data["rule_files"].include?(@rules_dir)
179
+ data["rule_files"] << File.expand_path(@rules_dir) + "/*.yaml"
180
+ end
181
+
182
+ File.open(destination, "w") do |file|
183
+ file.write("# AUTOGENERATED FILE, DO NOT MODIFY\n")
184
+ file.write("#\n")
185
+ file.write("# This file was automatically generated by prometheus-config-from-s3 at #{Time.now.utc.iso8601}\n")
186
+ file.write("# based on #{@prometheus_yaml_filename}\n")
187
+ file.write("#\n\n")
188
+ file.write(data.to_yaml)
189
+ end
190
+
191
+ return Digest::SHA256.digest data.to_yaml
192
+ end
193
+ end
194
+
195
+ class Builder
196
+ include PrometheusConfigBuilderLogger
197
+
198
+ def initialize(prometheus_src, dst_dir)
199
+ @prometheus_src = prometheus_src
200
+ @dst_dir = dst_dir
201
+
202
+ @discoverer = ConfigDiscover.new
203
+ @paths = []
204
+ @last_hash = ""
205
+
206
+ @@config_files = PrometheusExporter::Client.default.register(:gauge, "prometheusconfigbuilder_config_files", "Number of found config files")
207
+
208
+ end
209
+
210
+ def add_path(path)
211
+ @paths << path
212
+ end
213
+
214
+ def write_out
215
+ hash = ""
216
+ cfs = ConfigFiles.new(@prometheus_src)
217
+ files = []
218
+ @paths.each do |path|
219
+ found_files = @discoverer.discover(path)
220
+ logger.info("Found #{files.length} files from path #{path}")
221
+ files.push(*found_files)
222
+ end
223
+
224
+ files.each do |file|
225
+ cf = ConfigFile.new
226
+ cf.load(file)
227
+ cfs.add(cf)
228
+ end
229
+
230
+ @@config_files.observe(files.length)
231
+
232
+ rules_dir = @dst_dir + "/rules"
233
+ scrape_files_dir = @dst_dir + "/scrape_files"
234
+ cfs.set_rules_dir(rules_dir)
235
+ cfs.set_scrape_files_dir(scrape_files_dir)
236
+ hash += cfs.write_prometheus_yaml(@dst_dir + "/prometheus.yaml", scrape_files_dir)
237
+
238
+ FileUtils.mkdir_p(rules_dir) unless File.exist?(rules_dir)
239
+ FileUtils.mkdir_p(scrape_files_dir) unless File.exist?(scrape_files_dir)
240
+ hash += cfs.write_rules(rules_dir)
241
+
242
+ # Determine if something had changed since the last write_out based on the generated hashes
243
+ if hash != @last_hash
244
+ @last_hash = hash
245
+ return true
246
+ else
247
+ return false
248
+ end
249
+ end
250
+ end
251
+
252
+ end
@@ -0,0 +1,123 @@
1
+ require 'pp'
2
+ require_relative './logger.rb'
3
+
4
+ module PrometheusConfigBuilder
5
+
6
+ class ScrapeConfigECS
7
+ extend PrometheusConfigBuilderLogger
8
+
9
+ def self.handle(basename, config, dst_prefix)
10
+
11
+ x = $VERBOSE
12
+ $VERBOSE = nil
13
+ require 'aws-sdk'
14
+
15
+ if !config["region"]
16
+ logger.warn("File #{basename}: the scrape_configs of type:ecs-tasks doesn't have \"region\" field set. Ignoring!")
17
+ return nil
18
+
19
+ end
20
+
21
+ @@ecs = Aws::ECS::Client.new({
22
+ region:config["region"]
23
+ })
24
+ $VERBOSE = x
25
+ tasks = get_tasks(config["cluster"], config["service"])
26
+ endpoints = get_task_endpoints(config["cluster"], tasks, config["labels"], config["metrics_port"])
27
+
28
+ if !config["job_name"]
29
+ logger.warn("File #{basename}: the scrape_configs of type:ecs-tasks doesn't have \"job_name\" field set. Ignoring!")
30
+ return nil
31
+ end
32
+
33
+ file = File.expand_path(dst_prefix + "_" + config["job_name"] + ".json")
34
+ File.open(file, "w") do |file|
35
+ file.write(endpoints.to_json)
36
+ end
37
+
38
+ # Make copy of the settings and remove our custom properties from it.
39
+ # The rest user can set just as he wants according to the Prometheus schema.
40
+ # See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config
41
+ settings = config.clone
42
+ settings.delete("type")
43
+ settings.delete("cluster")
44
+ settings.delete("service")
45
+ settings.delete("metrics_port")
46
+ settings.delete("labels")
47
+ settings.delete("region")
48
+ settings["file_sd_configs"] = [
49
+ "files" => [
50
+ file
51
+ ]
52
+ ]
53
+
54
+ return settings
55
+ end
56
+
57
+ def self.get_tasks(cluster, service_name)
58
+ tasks = []
59
+ last_result = Aws::ECS::Types::ListTasksResponse.new
60
+ last_result.next_token = nil
61
+ loop do
62
+ begin
63
+ options = {
64
+ cluster: cluster,
65
+ service_name: service_name,
66
+ next_token: last_result.next_token
67
+ }
68
+ last_result = @@ecs.list_tasks(options)
69
+ if last_result && last_result.task_arns
70
+ tasks.push(*last_result.task_arns)
71
+ end
72
+ rescue Aws::ECS::Errors::ServiceError => e
73
+ print "Error listing ecs tasks: #{e}"
74
+ sleep(1)
75
+ next
76
+ end
77
+ break if !last_result.next_token
78
+ end
79
+
80
+ return tasks
81
+ end
82
+
83
+ def self.get_task_endpoints(cluster, tasks, common_labels, metrics_port)
84
+ endpoints = []
85
+ task_chunk = tasks.pop(100)
86
+ loop do
87
+ begin
88
+ options = {
89
+ cluster: cluster,
90
+ tasks: task_chunk,
91
+ }
92
+ result = @@ecs.describe_tasks(options)
93
+ if result && result.tasks
94
+ result.tasks.each do |task|
95
+ # FIXME: This assumes somewhat on the ip structure.
96
+ ip = task.containers[0].network_interfaces[0].private_ipv_4_address
97
+
98
+ if metrics_port
99
+ ip = ip + ":" + metrics_port.to_s
100
+ end
101
+ labels = common_labels.clone
102
+ labels["ecs_arn"] = task.task_arn
103
+ endpoints << {
104
+ "targets" => [ip],
105
+ "labels" => labels
106
+ }
107
+ end
108
+ end
109
+ rescue Aws::ECS::Errors::ServiceError => e
110
+ print "Error listing ecs tasks: #{e}\n"
111
+ sleep(1)
112
+ next
113
+ end
114
+ break if tasks.length == 0
115
+ task_chunk = tasks.pop(100)
116
+ end
117
+ return endpoints
118
+ end
119
+
120
+
121
+
122
+ end
123
+ end
@@ -0,0 +1,11 @@
1
+ module PrometheusConfigBuilder
2
+
3
+ class ScrapeConfigPassthrough
4
+ def self.handle(config)
5
+ settings = config.clone
6
+ settings.delete("type")
7
+ return settings
8
+ end
9
+ end
10
+
11
+ end
@@ -0,0 +1,4 @@
1
+ module TFTP
2
+ # Current version string.
3
+ VERSION = '0.1'
4
+ end
@@ -0,0 +1,25 @@
1
+ # coding: utf-8
2
+ #
3
+
4
+ Gem::Specification.new do |s|
5
+ s.name = 'prometheus-config-builder'
6
+ s.version = '0.0.15'
7
+ s.date = Time.now
8
+
9
+ s.summary = %q{Template based config generation}
10
+ s.files = `git ls-files`.split("\n")
11
+ s.executables = ['prometheus-config-builder']
12
+ s.test_files = s.files.grep(%r{^test/})
13
+ s.require_paths = ['lib']
14
+ s.authors = "Juho Mäkinen juho.makinen@gmail.com"
15
+
16
+ s.required_ruby_version = '>= 2.0.0'
17
+
18
+ s.add_dependency 'aws-sdk', '~> 3'
19
+ s.add_dependency 'prometheus_exporter', '>= 0.4.13'
20
+
21
+ s.add_development_dependency 'rubygems-tasks', '~> 0.2'
22
+ s.add_development_dependency 'minitest', '~> 5.4'
23
+ s.add_development_dependency 'minitest-reporters'
24
+ s.add_development_dependency 'rake', '~> 10.0'
25
+ end
@@ -0,0 +1,121 @@
1
+ require 'minitest/autorun'
2
+
3
+ require 'prometheus-config-builder'
4
+ require 'fileutils'
5
+
6
+ class ConfigDiscoveryTest < Minitest::Test
7
+
8
+ class MockDiscovery < PrometheusConfigBuilder::ConfigFile
9
+
10
+ attr_accessor :files
11
+ @files = []
12
+
13
+ def get_files()
14
+ @files
15
+ end
16
+ end
17
+
18
+ def test_can_get_rules_from_file
19
+ f = PrometheusConfigBuilder::ConfigFile.new()
20
+ f.open('test/data/test1.yaml')
21
+
22
+ assert_equal "from_terraform.generic.rules", f.config_rules[0]['name']
23
+ assert_equal "passthrough", f.scrape_configs[0]['type']
24
+ end
25
+
26
+ def test_scrape_config
27
+ f = PrometheusConfigBuilder::ConfigFile.new()
28
+ f.open('test/data/test1.yaml')
29
+
30
+ configs = f.get_scrape_configs('tmp/test')
31
+
32
+ assert_equal "discovery 1", configs[0]["job_name"]
33
+ assert_equal "localhost:9090", configs[0]["static_configs"][0]["targets"][0]
34
+
35
+ end
36
+
37
+ def test_can_write_rules_to_file
38
+ f = PrometheusConfigBuilder::ConfigFile.new()
39
+ f.open('test/data/test1.yaml')
40
+
41
+ filename = './tmp/test1.yaml'
42
+ FileUtils.mkdir('tmp') if not File.directory?('tmp')
43
+ File.delete(filename) if File.exist?(filename)
44
+ f.write_rules('./tmp')
45
+
46
+ contents = YAML.load_file(filename)
47
+
48
+ assert_equal "from_terraform.generic.rules", contents[0]["name"]
49
+
50
+ end
51
+
52
+ def test_can_get_all_files_from_mock
53
+
54
+ f1 = PrometheusConfigBuilder::ConfigFile.new
55
+ f2 = PrometheusConfigBuilder::ConfigFile.new
56
+ f1.open('test/data/test1.yaml')
57
+ f2.open('test/data/test2.yaml')
58
+
59
+ d = PrometheusConfigBuilder::ConfigFiles.new('test/data/prometheus.yaml')
60
+ d.add(f1)
61
+ d.add(f2)
62
+
63
+ FileUtils.rm_r('tmp/test_can_get_all_files_from_mock') if File.exist?('tmp/test_can_get_all_files_from_mock')
64
+ FileUtils.mkdir_p('tmp/test_can_get_all_files_from_mock')
65
+
66
+ d.write_rules('./tmp/test_can_get_all_files_from_mock')
67
+
68
+ # Test over all files
69
+ filename = './tmp/test_can_get_all_files_from_mock/test1.yaml'
70
+ contents = YAML.load_file(filename)
71
+ assert_equal "from_terraform.generic.rules", contents[0]["name"]
72
+
73
+ filename = './tmp/test_can_get_all_files_from_mock/test2.yaml'
74
+ contents = YAML.load_file(filename)
75
+ assert_equal "something_else", contents[0]["name"]
76
+
77
+ # Write main prometheus.yaml
78
+ filename = './tmp/test_can_get_all_files_from_mock/prometheus.yaml'
79
+ d.write_prometheus_yaml(filename, 'tmp/')
80
+
81
+ contents = YAML.load_file(filename)
82
+ assert_equal "15s", contents["global"]["scrape_interval"]
83
+ assert_equal "prometheus", contents["scrape_configs"][0]["job_name"]
84
+ assert_equal "alertmanager", contents["scrape_configs"][1]["job_name"]
85
+
86
+ assert_equal "discovery 1", contents["scrape_configs"][2]["job_name"]
87
+
88
+ end
89
+
90
+ def test_discover_config_files
91
+ x = PrometheusConfigBuilder::ConfigDiscover.new
92
+
93
+ files = x.discover("./test/data/test*.yaml")
94
+ assert_equal 2, files.length
95
+
96
+ end
97
+
98
+ def test_config_builder
99
+ x = PrometheusConfigBuilder::Builder.new("./test/data/prometheus.yaml", "tmp/")
100
+ x.add_path("./test/data/test*.yaml")
101
+
102
+ ret = x.write_out()
103
+ assert_equal true, ret # true as in config files were modified or at least rewritten
104
+
105
+ assert_equal true, File.exist?("tmp/prometheus.yaml")
106
+ assert_equal true, File.exist?("tmp/rules/test1.yaml")
107
+ assert_equal true, File.exist?("tmp/rules/test2.yaml")
108
+
109
+ end
110
+
111
+ def test_config_builder_knows_when_to_refresh
112
+ x = PrometheusConfigBuilder::Builder.new("./test/data/prometheus.yaml", "tmp/")
113
+ x.add_path("./test/data/test*.yaml")
114
+
115
+ x.write_out()
116
+ ret = x.write_out()
117
+ assert_equal false, ret # Nothing had changed, no need to rewrite
118
+
119
+ end
120
+
121
+ end
@@ -0,0 +1,22 @@
1
+ global:
2
+ scrape_interval: 15s
3
+ evaluation_interval: 15s
4
+
5
+ alerting:
6
+ alertmanagers:
7
+ - static_configs:
8
+ - targets:
9
+ - localhost:9093
10
+
11
+ rule_files:
12
+ - "/mnt/data/prometheus/rules/*.yaml"
13
+
14
+ scrape_configs:
15
+ - job_name: prometheus
16
+ static_configs:
17
+ - targets:
18
+ - localhost:9090
19
+ - job_name: alertmanager
20
+ static_configs:
21
+ - targets:
22
+ - localhost:9093
@@ -0,0 +1,17 @@
1
+ scrape_configs:
2
+ - type: passthrough
3
+ job_name: discovery 1
4
+ static_configs:
5
+ - targets: ['localhost:9090']
6
+
7
+
8
+ config_rules:
9
+ - name: from_terraform.generic.rules
10
+ rules:
11
+ - alert: DeadMansSwitch
12
+ annotations:
13
+ description: This is a DeadMansSwitch meant to ensure that the entire Alerting pipeline is functional.
14
+ summary: Alerting DeadMansSwitch
15
+ expr: vector(1)
16
+ labels:
17
+ severity: none
@@ -0,0 +1,17 @@
1
+ scrape_configs:
2
+ - type: passthrough
3
+ job_name: discovery 2
4
+ static_configs:
5
+ - targets: ['localhost:9090']
6
+
7
+
8
+ config_rules:
9
+ - name: something_else
10
+ rules:
11
+ - alert: DeadMansSwitch
12
+ annotations:
13
+ description: This is a DeadMansSwitch meant to ensure that the entire Alerting pipeline is functional.
14
+ summary: Alerting DeadMansSwitch
15
+ expr: vector(1)
16
+ labels:
17
+ severity: none
@@ -0,0 +1,4 @@
1
+ require "minitest/reporters"
2
+
3
+ Minitest::Reporters.use! Minitest::Reporters::SpecReporter.new
4
+
metadata ADDED
@@ -0,0 +1,149 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: prometheus-config-builder
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.15
5
+ platform: ruby
6
+ authors:
7
+ - Juho Mäkinen juho.makinen@gmail.com
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2019-09-11 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: aws-sdk
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '3'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '3'
27
+ - !ruby/object:Gem::Dependency
28
+ name: prometheus_exporter
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: 0.4.13
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: 0.4.13
41
+ - !ruby/object:Gem::Dependency
42
+ name: rubygems-tasks
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '0.2'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '0.2'
55
+ - !ruby/object:Gem::Dependency
56
+ name: minitest
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '5.4'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '5.4'
69
+ - !ruby/object:Gem::Dependency
70
+ name: minitest-reporters
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: rake
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '10.0'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - "~>"
95
+ - !ruby/object:Gem::Version
96
+ version: '10.0'
97
+ description:
98
+ email:
99
+ executables:
100
+ - prometheus-config-builder
101
+ extensions: []
102
+ extra_rdoc_files: []
103
+ files:
104
+ - ".gitignore"
105
+ - Gemfile
106
+ - README.md
107
+ - Rakefile
108
+ - bin/prometheus-config-builder
109
+ - lib/prometheus-config-builder.rb
110
+ - lib/prometheus-config-builder/logger.rb
111
+ - lib/prometheus-config-builder/prometheus-config-builder.rb
112
+ - lib/prometheus-config-builder/scrape_ecs.rb
113
+ - lib/prometheus-config-builder/scrape_passthrough.rb
114
+ - lib/prometheus-config-builder/version.rb
115
+ - prometheus-config-builder.gemspec
116
+ - test/configdiscovery.rb
117
+ - test/data/prometheus.yaml
118
+ - test/data/test1.yaml
119
+ - test/data/test2.yaml
120
+ - test/test_helper.rb
121
+ homepage:
122
+ licenses: []
123
+ metadata: {}
124
+ post_install_message:
125
+ rdoc_options: []
126
+ require_paths:
127
+ - lib
128
+ required_ruby_version: !ruby/object:Gem::Requirement
129
+ requirements:
130
+ - - ">="
131
+ - !ruby/object:Gem::Version
132
+ version: 2.0.0
133
+ required_rubygems_version: !ruby/object:Gem::Requirement
134
+ requirements:
135
+ - - ">="
136
+ - !ruby/object:Gem::Version
137
+ version: '0'
138
+ requirements: []
139
+ rubyforge_project:
140
+ rubygems_version: 2.5.2
141
+ signing_key:
142
+ specification_version: 4
143
+ summary: Template based config generation
144
+ test_files:
145
+ - test/configdiscovery.rb
146
+ - test/data/prometheus.yaml
147
+ - test/data/test1.yaml
148
+ - test/data/test2.yaml
149
+ - test/test_helper.rb