inventory-server 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +15 -0
  3. data/.rspec +3 -0
  4. data/.travis.yml +12 -0
  5. data/Dockerfile +22 -0
  6. data/Dockerfile-passenger +18 -0
  7. data/Dockerfile-unicorn +22 -0
  8. data/Gemfile +4 -0
  9. data/LICENSE.txt +22 -0
  10. data/README.md +32 -0
  11. data/Rakefile +2 -0
  12. data/bin/inventory-server +33 -0
  13. data/circle.yml +26 -0
  14. data/config.ru +8 -0
  15. data/docker/passenger/passenger.conf +15 -0
  16. data/docker/passenger/run-httpd.sh +8 -0
  17. data/docker/unicorn/nginx.conf +45 -0
  18. data/docker/unicorn/run-unicorn.sh +5 -0
  19. data/docker/unicorn/unicorn.conf +31 -0
  20. data/docker/unicorn/unicorn.rb +19 -0
  21. data/fig.yml +18 -0
  22. data/inventory-server.gemspec +45 -0
  23. data/lib/inventory/server.rb +30 -0
  24. data/lib/inventory/server/cli.rb +59 -0
  25. data/lib/inventory/server/config.rb +80 -0
  26. data/lib/inventory/server/email_parser.rb +28 -0
  27. data/lib/inventory/server/http_server.rb +49 -0
  28. data/lib/inventory/server/inventory_error.rb +9 -0
  29. data/lib/inventory/server/loader.rb +60 -0
  30. data/lib/inventory/server/logger.rb +16 -0
  31. data/lib/inventory/server/smtp_server.rb +36 -0
  32. data/lib/inventory/server/version.rb +5 -0
  33. data/plugins/facts_parser.rb +95 -0
  34. data/plugins/index.rb +56 -0
  35. data/plugins/json_schema_validator.rb +33 -0
  36. data/plugins/log_failures_on_disk.rb +35 -0
  37. data/public/.gitignore +0 -0
  38. data/spec/integration/fixtures/facter.xml +4543 -0
  39. data/spec/integration/http_spec.rb +24 -0
  40. data/spec/spec_helper.rb +96 -0
  41. data/spec/unit/cli_spec.rb +54 -0
  42. data/spec/unit/config_spec.rb +65 -0
  43. data/spec/unit/email_parser_spec.rb +53 -0
  44. data/spec/unit/facts_parser_spec.rb +176 -0
  45. data/spec/unit/fixtures/simple_plugin.rb +2 -0
  46. data/spec/unit/http_server_spec.rb +58 -0
  47. data/spec/unit/index_spec.rb +77 -0
  48. data/spec/unit/json_schema_validator_spec.rb +126 -0
  49. data/spec/unit/loader_spec.rb +34 -0
  50. data/spec/unit/log_failures_on_disk_spec.rb +50 -0
  51. data/spec/unit/server_spec.rb +11 -0
  52. data/spec/unit/smtp_server_spec.rb +68 -0
  53. data/tmp/.gitignore +0 -0
  54. metadata +434 -0
@@ -0,0 +1,30 @@
1
+ require 'middleware'
2
+ require "inventory/server/version"
3
+ require "inventory/server/loader"
4
+ require "inventory/server/config"
5
+ require "inventory/server/logger"
6
+
7
+ module Inventory
8
+ module Server
9
+ class Server
10
+ attr_reader :config, :middlewares
11
+
12
+ def initialize(cli_config)
13
+ config = Config.generate(cli_config)
14
+ @config = config
15
+
16
+ InventoryLogger.setup(config[:logger])
17
+ InventoryLogger.logger.level = config[:log_level]
18
+
19
+ # Dynamically load plugins from plugins_path
20
+ plugin_names = config[:plugins].split(',')
21
+ @middlewares = Middleware::Builder.new do
22
+ plugins = Loader.new(config).load_plugins(*plugin_names)
23
+ plugins.each {|klass|
24
+ use klass, config
25
+ }
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,59 @@
1
+ require 'optparse'
2
+ require "inventory/server/version"
3
+
4
+ module Inventory
5
+ module Server
6
+
7
+ class CLI
8
+
9
+ attr_reader :options
10
+
11
+ def initialize()
12
+ @options = {}
13
+ end
14
+
15
+ def parse!(args)
16
+ OptionParser.new do|opts|
17
+ opts.banner = "Usage: server [options]"
18
+ opts.separator ""
19
+ opts.separator "Specific options:"
20
+
21
+ opts.on("--host HOST", String, "IP or hostname to listen on") do |h|
22
+ @options[:host] = h
23
+ end
24
+
25
+ opts.on("--smtp_port PORT", Integer, "SMTP port to listen on") do |p|
26
+ @options[:smtp_port] = p
27
+ end
28
+
29
+ opts.on("-es", "--es_host URL", String, "ElasticSerach HTTP URL") do |url|
30
+ @options[:es_host] = url
31
+ end
32
+
33
+ opts.on("-o", "--output OUTPUT", String, "Log destination stdout/stderr/file") do |o|
34
+ @options[:logger] = o
35
+ end
36
+
37
+ opts.on("-l", "--level LEVEL", ['DEBUG', 'INFO', 'WARN', 'ERROR', 'FATAL'], "log level that will be printed DEBUG/INFO/WARN/ERROR/FATAL") do |l|
38
+ @options[:log_level] = l
39
+ end
40
+
41
+ opts.on("-d", "--[no-]debug", "how more details, need log_level on DEBUG") do |d|
42
+ @options[:debug] = d
43
+ end
44
+
45
+ opts.on_tail("-h", "--help", "Show this message") do
46
+ puts opts
47
+ exit
48
+ end
49
+
50
+ opts.on_tail("--version", "Show version") do
51
+ puts VERSION
52
+ exit
53
+ end
54
+ end.parse! args
55
+ @options
56
+ end
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,80 @@
1
+ require 'app_configuration'
2
+ require 'fileutils'
3
+ require "inventory/server/logger"
4
+
5
+ module Inventory
6
+ module Server
7
+
8
+ module Config
9
+ DEFAULTS = {
10
+ :host => '127.0.0.1',
11
+ :smtp_port => 2525,
12
+ :max_connections => 4,
13
+ :debug => false,
14
+ :es_host => 'http://localhost:9200',
15
+ :es_index_prefix => 'inventory_',
16
+ :failed_facts_dir => '/var/log/inventory/failures',
17
+ :logger => 'stdout',
18
+ :log_level => 'INFO',
19
+ :type_key => 'type',
20
+ :type_default => 'facts',
21
+ :version_key => 'version',
22
+ :version_default => '1-0-0',
23
+ :json_schema_dir => '/etc/inventory/json_schema',
24
+ :plugins_path => '',
25
+ :plugins => 'log_failures_on_disk,facts_parser,json_schema_validator,index',
26
+ }
27
+
28
+ def self.generate(cli_config)
29
+
30
+ global = AppConfiguration.new('inventory.yml') do
31
+ base_global_path '/etc'
32
+ use_env_variables true
33
+ prefix 'inventory' # ENV prefix: INVENTORY_XXXXX
34
+ end
35
+
36
+ config = {}
37
+
38
+ DEFAULTS.each{|sym, default_value|
39
+
40
+ val = cli_config[sym] || global[sym.to_s] || default_value
41
+ result = val
42
+
43
+ # cast config values (ENV values are strings only)
44
+ if default_value.is_a? Integer
45
+ result = val.to_i
46
+ elsif !!default_value == default_value
47
+ # Boolean
48
+ if val == true || val =~ /^(true|t|yes|y|1)$/i
49
+ result = true
50
+ elsif val == false || val.blank? || val =~ /^(false|f|no|n|0)$/i
51
+ result = false
52
+ end
53
+ # Logging
54
+ elsif val == 'stdout'
55
+ result = $stdout
56
+ elsif val == 'stderr'
57
+ result = $stderr
58
+ # Logging Level
59
+ elsif val == 'INFO'
60
+ result = Logger::INFO
61
+ elsif val == 'DEBUG'
62
+ result = Logger::DEBUG
63
+ elsif val == 'WARN'
64
+ result = Logger::WARN
65
+ elsif val == 'ERROR'
66
+ result = Logger::ERROR
67
+ elsif val == 'FATAL'
68
+ result = Logger::FATAL
69
+ end
70
+ config[sym] = result
71
+ }
72
+
73
+ FileUtils.mkdir_p config[:failed_facts_dir]
74
+
75
+ config
76
+ end
77
+ end
78
+ end
79
+ end
80
+
@@ -0,0 +1,28 @@
1
+ require "mail"
2
+ require 'inventory/server/inventory_error'
3
+ require "inventory/server/logger"
4
+
5
+ module Inventory
6
+ module Server
7
+ class EmailParser
8
+ def self.parse(message)
9
+ InventoryLogger.logger.info "Email parser"
10
+
11
+ # Parse the email
12
+ email = Mail.read_from_string(message)
13
+
14
+ # Use email subject as an ID
15
+ email_subject = email.subject
16
+ raise InventoryError.new "email subject is missing" if email_subject.nil? || email_subject.empty?
17
+ id = email_subject
18
+
19
+ # Decode the email body
20
+ email_body = email.body.decoded
21
+ raise InventoryError.new "email body is missing" if email_body.nil? || email_body.empty?
22
+ body = email_body
23
+
24
+ return id, body
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,49 @@
1
+ require 'sinatra'
2
+ require 'json'
3
+ require 'inventory/server/inventory_error'
4
+ require "inventory/server/logger"
5
+
6
+ module Inventory
7
+ module Server
8
+ class HTTPServer < Sinatra::Base
9
+
10
+ configure do
11
+ logger = Object.new.tap do |proxy|
12
+ def proxy.<<(message)
13
+ InventoryLogger.logger.info message
14
+ end
15
+ def proxy.write(message)
16
+ InventoryLogger.logger.info message
17
+ end
18
+ def proxy.flush; end
19
+ end
20
+
21
+ use Rack::CommonLogger, logger
22
+
23
+ before {
24
+ env["rack.errors"] = logger
25
+ }
26
+
27
+ Rack::Utils.key_space_limit = 262144
28
+
29
+ HTTPServer.set :raise_errors, false
30
+ HTTPServer.set :dump_errors, false
31
+ end
32
+
33
+ post "/api/v1/facts/:id" do
34
+ content_type :json
35
+ id= params[:id]
36
+ env[:id] = id
37
+ InventoryLogger.logger.context_id = id
38
+
39
+ request.body.rewind
40
+ settings.middlewares.call(:id => id, :body => request.body.read, :config => settings.config)
41
+ {:id => id, :ok => true, :status => 200}.to_json
42
+ end
43
+
44
+ error 400..500 do
45
+ {:id => env[:id], :ok => false, :status => status, :message => env['sinatra.error']}.to_json
46
+ end
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,9 @@
1
+ module Inventory
2
+ module Server
3
+
4
+ # The exception from which all other exceptions in this library derive.
5
+ class InventoryError < StandardError
6
+ end
7
+
8
+ end
9
+ end
@@ -0,0 +1,60 @@
1
+ require 'inventory/server/inventory_error'
2
+ require 'inventory/server/logger'
3
+
4
+ PLUGINS_DIR = Pathname.new(File.join File.dirname(__FILE__), '..', '..', '..', 'plugins').cleanpath
5
+
6
+ module Inventory
7
+ module Server
8
+ class Loader
9
+ def initialize(config)
10
+ @plugins_path = [PLUGINS_DIR] + config[:plugins_path].split(',')
11
+ @plugins_path.each { |path|
12
+ raise InventoryError.new "plugins_path #{path} not found" unless File.directory? path
13
+ }
14
+ @loaded = []
15
+ end
16
+
17
+ # search for plugins in the plugins_path and return a hash of plugin_filename:plugin_klass
18
+ def load_plugins(*plugins)
19
+ plugin_klasses = []
20
+ plugins.each {|plugin|
21
+ p = nil
22
+ @plugins_path.each {|plugin_path|
23
+ filepath = File.join plugin_path, "#{plugin}.rb"
24
+ next unless File.file? filepath
25
+ load_file filepath
26
+ klass_name = classify(plugin)
27
+ p = Object.const_get("Inventory").const_get("Server").const_get(klass_name)
28
+ }
29
+ raise InventoryError.new "Plugin #{plugin} not found" if !p
30
+ plugin_klasses << p
31
+ }
32
+ return plugin_klasses
33
+ end
34
+
35
+ # Load a ruby file if not already loaded
36
+ def load_file(file)
37
+ return if @loaded.include? file
38
+
39
+ begin
40
+ @loaded << file
41
+ kernel_load(file)
42
+ rescue => e
43
+ @loaded.delete(file)
44
+ InventoryLogger.logger.error("Fail to load #{file}: #{e}")
45
+ end
46
+ end
47
+
48
+ # Usefull for tests
49
+ def kernel_load(file)
50
+ InventoryLogger.logger.info "Load #{file}"
51
+ require file
52
+ end
53
+
54
+ # transform a snake case string into a upper camel case string
55
+ def classify(str)
56
+ str.split('_').collect!{ |w| w.capitalize }.join
57
+ end
58
+ end
59
+ end
60
+ end
@@ -0,0 +1,16 @@
1
+ require 'filum'
2
+
3
+ module Inventory
4
+ module Server
5
+ module InventoryLogger
6
+
7
+ def self.setup(*args)
8
+ Filum.setup(*args)
9
+ end
10
+
11
+ def self.logger
12
+ Filum.logger
13
+ end
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,36 @@
1
+ require "rubygems"
2
+ require "midi-smtp-server"
3
+ require "inventory/server/email_parser"
4
+ require 'inventory/server/inventory_error'
5
+ require "inventory/server/logger"
6
+
7
+ module Inventory
8
+ module Server
9
+
10
+ # Create an SMTP Server
11
+ class SMTPServer < MidiSmtpServer
12
+
13
+ def initialize(config, middlewares)
14
+ @config = config
15
+ @middlewares = middlewares
16
+ @audit = @config[:debug]
17
+ super(config[:smtp_port], config[:host], config[:max_connections])
18
+ end
19
+
20
+ def on_message_data_event(ctx)
21
+ begin
22
+ # execute middlewares
23
+ id, body = EmailParser.parse(ctx[:message])
24
+ InventoryLogger.logger.context_id = id
25
+ @middlewares.call(:id => id, :body => body)
26
+ rescue => e
27
+ # dot not raise the error to avoid the SMTP server relay to defer malformed emails
28
+ end
29
+ end
30
+
31
+ def log(msg)
32
+ InventoryLogger.logger.debug msg
33
+ end
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,5 @@
1
+ module Inventory
2
+ module Server
3
+ VERSION = "0.0.1"
4
+ end
5
+ end
@@ -0,0 +1,95 @@
1
+ # encoding: utf-8
2
+ require 'json'
3
+ require 'yaml'
4
+ require "base64"
5
+ require 'libxml_to_hash'
6
+
7
+ require 'ensure/encoding'
8
+ require 'inventory/server/inventory_error'
9
+ require "inventory/server/logger"
10
+
11
+ module Inventory
12
+ module Server
13
+ class FactsParser
14
+ def initialize(app, config)
15
+ @app = app
16
+ @config = config
17
+ end
18
+
19
+ def call(env)
20
+ InventoryLogger.logger.info "Facts parser"
21
+ body = env[:body]
22
+ raise InventoryError.new "body missing" if body.nil? || body.empty?
23
+
24
+ format = guess_format body
25
+ raise InventoryError.new "bad format" if !format
26
+
27
+ env[:facts] = parse(format, body)
28
+ @app.call(env)
29
+ end
30
+
31
+ # guess about the format of `str`
32
+ def guess_format(str)
33
+ case str
34
+ when /\A\s*[\[\{]/ then :json
35
+ when /\A\s*</ then :xml
36
+ when /\A---\s/ then :yaml
37
+ end
38
+ end
39
+
40
+
41
+ # Parse str into a hash, the format will ne guessed
42
+ def parse(format, str)
43
+ case format
44
+ when :xml
45
+ # XML validation
46
+
47
+ hash = nil
48
+ begin
49
+ hash = Hash.from_libxml! str
50
+ xml = decode_base64(hash)
51
+ keys = xml.keys
52
+ if keys.length == 1
53
+ xml = xml[keys[0]]
54
+ end
55
+ raise "Expect < found #{xml.text} " if xml.is_a?(LibXmlNode) and xml.text != ""
56
+ return xml
57
+ rescue => e
58
+ raise $!, "Invalid XML #{$!}", $!.backtrace
59
+ end
60
+ when :json
61
+ JSON.parse str
62
+ when :yaml
63
+ YAML.load str
64
+ end
65
+ end
66
+
67
+ # decode base64 if present in this deep structure
68
+ def decode_base64(something)
69
+ if something.is_a?(Hash)
70
+ return nil if something.empty?
71
+ something.each {|key, value|
72
+ something[key] = decode_base64(value)
73
+ }
74
+ elsif something.is_a?(Array)
75
+ return nil if something.empty?
76
+ something = something.map {|value|
77
+ decode_base64(value)
78
+ }
79
+ elsif something.is_a?(String) and something.include? "__base64__"
80
+ something.slice!("__base64__")
81
+ fix_bad_encoding Base64.decode64(something);
82
+ else
83
+ something
84
+ end
85
+ end
86
+
87
+ def fix_bad_encoding(str)
88
+ str.ensure_encoding('UTF-8',
89
+ :external_encoding => [Encoding::UTF_8, Encoding::ISO_8859_1],
90
+ :invalid_characters => :transcode
91
+ )
92
+ end
93
+ end
94
+ end
95
+ end