bjn_inventory 1.3.0

data/bin/inventory_model

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+
+# Creat list of inventory devices based on device model 
+
+require 'trollop'
+require 'json'
+require 'logger'
+require 'syslog/logger'
+require 'bjn_inventory'
+
+parser = Trollop::Parser.new do
+    version BjnInventory::VERSION
+    banner <<-USAGE.gsub(/^\s{8}/,'')
+        Usage:
+          inventory-devices [options]
+    USAGE
+
+    opt :key, 'Specify the key of inventory hash', required: true, type: :string
+    opt :manifest, 'Specify the manifest that defines this inventory', required: true, type: :string
+    opt :debug, 'Enable debug output', :short => '-D'
+    opt :syslog, 'Log to Syslog', :short => '-S'
+    stop_on_unknown
+end
+
+opt = Trollop::with_standard_exception_handling parser do
+    parser.parse(ARGV)
+end
+
+logger = opt[:syslog]? Syslog::Logger.new('inventory-devices') : Logger.new(STDERR)
+logger.level = opt[:debug]? Logger::DEBUG : Logger::WARN
+
+manifest = JSON.parse(File.read(opt[:manifest]))
+inventory = BjnInventory::Inventory.new(manifest.merge({logger: logger}))
+puts JSON.pretty_generate(inventory.by(opt[:key]))

data/bin/refresh_inventory_data

@@ -0,0 +1,51 @@
+#!/usr/bin/env ruby
+
+require 'trollop'
+require 'json'
+require 'bjn_inventory'
+require 'bjn_inventory/data_files'
+require 'bjn_inventory/ansible'
+
+parser = Trollop::Parser.new do
+    version BjnInventory::VERSION
+    banner <<-USAGE.gsub(/^\s{8}/,'')
+        Usage:
+          refresh_inventory_data [options]
+    USAGE
+
+    opt :ansible, 'Specify ansible groupings file', required: true, type: :string
+    opt :manifest, 'Specify the manifest that defines this inventory', required: true, type: :string
+    opt :debug, 'Enable debug output', :short => '-D'
+    opt :syslog, 'Log to Syslog', :short => '-S'
+    opt :datadir, 'Location of inventory data', default: '/var/cache/inventory'
+    stop_on_unknown
+end
+
+opt = Trollop::with_standard_exception_handling parser do
+    parser.parse(ARGV)
+end
+
+if opt[:syslog]
+    require 'syslog/logger'
+    logger = Syslog::Logger.new 'refresh_inventory_data'
+else
+    logger = Logger.new STDERR
+end
+
+if opt[:debug]
+    logger.level = Logger::DEBUG
+else
+    logger.level = Logger::WARN
+end
+
+manifest = JSON.parse(File.read(opt[:manifest]))
+ansible_spec = JSON.parse(File.read(opt[:ansible]))
+
+inventory = BjnInventory::Inventory.new(manifest.merge({logger: logger})).by('name')
+devices_data = inventory
+groups_data = BjnInventory::get_groups_data(inventory, ansible_spec)
+
+BjnInventory::refresh_inventory_data('devices', devices_data, opt)
+BjnInventory::refresh_inventory_data('groups', groups_data, opt)
+
+

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/bjn_inventory.gemspec

@@ -0,0 +1,33 @@
+# coding: utf-8
+# -*- mode: ruby; ruby-indent-level: 2; -*-
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'bjn_inventory/version'
+require 'bjn_inventory/metadata'
+
+Gem::Specification.new do |spec|
+  spec.name          = "bjn_inventory"
+  spec.version       = BjnInventory::VERSION
+  spec.authors       = [BjnInventory::AUTHOR]
+  spec.email         = [BjnInventory::EMAIL]
+
+  spec.summary       = BjnInventory::SUMMARY
+  spec.homepage      = BjnInventory::URL
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "bin"
+  spec.executables   = ["ansible-from", "aws-ec2-source", "aws-rds-source", "inventory_model", "refresh_inventory_data"]
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'jsonpath', "~> 0.7.2"
+  spec.add_dependency 'trollop'
+  spec.add_dependency 'aws-sdk'
+
+  spec.add_development_dependency "bundler", "~> 1.9"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "rspec_junit_formatter"
+end

data/lib/bjn_inventory.rb

@@ -0,0 +1,5 @@
+require "bjn_inventory/version"
+require "bjn_inventory/device"
+require "bjn_inventory/inventory"
+require "bjn_inventory/bykey"
+require "bjn_inventory/list"

data/lib/bjn_inventory/ansible.rb

@@ -0,0 +1,86 @@
+require 'bjn_inventory'
+require 'bjn_inventory/default_logger'
+
+module BjnInventory
+
+    class ByKey
+
+        def _ansible_name(value)
+            value.gsub(/[^a-zA-Z0-9_-]+/, '_')
+        end
+
+        def _field_groups(fields, device, sep='__')
+            fields = [fields] unless fields.respond_to? :inject
+            value_map = fields.map do |field|
+                values = device[field]
+                values = [values] unless values.respond_to? :map
+                values.map { |val| _ansible_name(val) }
+            end
+            #
+            # So now we have an array of arrays, eg.:
+            # fields='region' =>
+            # [['dc2']]
+            #
+            # fields=['roles', 'region'] =>
+            # [['web', ['dc2']]
+            #   'db'],
+            #
+            groups =
+                if fields.length == 1
+                    value_map.first
+                else
+                    driving_array, *rest = value_map
+                    driving_array.product(*rest).map { |compound_value| compound_value.join(sep) }
+                end
+            groups
+        end
+
+        # This basically builds an ansible inventory given a hash of hostvars
+        def to_ansible(*ansible_spec)
+            if ansible_spec[-1].respond_to? :to_hash
+                kwargs = ansible_spec.pop.stringify_keys
+            else
+                kwargs = { }
+            end
+            group_by = []
+            if kwargs['group_by']
+                group_by = kwargs['group_by']
+                group_by = [group_by] unless group_by.respond_to? :each
+            end
+            group_by.concat(ansible_spec)
+
+            if group_by.empty?
+                raise ArgumentError, "Expected group_by either as keyword or as argument list"
+            end
+
+            logger ||= BjnInventory::DefaultLogger.new
+            # We need at least one field to create groups
+            separator = kwargs['separator'] || '__'
+
+            ansible_inventory = { '_meta' => {'hostvars' => self.to_hash } }
+
+            self.each do |name, device_hash|
+                group_by.each do |group_field_spec|
+                    group_field_spec = [group_field_spec] unless group_field_spec.respond_to? :all?
+                    if group_field_spec.all? { |field| !device_hash[field].nil? && !device_hash[field].empty? }
+                        field_groups = _field_groups(group_field_spec, device_hash, separator)
+                        field_groups.each do |group_name|
+                            ansible_inventory[group_name] = [ ] unless ansible_inventory.has_key? group_name
+                            ansible_inventory[group_name] << name
+                        end
+                    end
+                end
+            end
+
+            if kwargs['groups']
+                ansible_inventory.merge! Hash[kwargs['groups'].map do |group, children|
+                                                  [group, { "hosts" => [ ], "children" => children }]
+                                              end]
+            end
+
+            ansible_inventory
+        end
+
+    end
+
+end

data/lib/bjn_inventory/array.rb

@@ -0,0 +1,22 @@
+# Inventory array helper
+class Array
+    # This array is treated as a list of hashes, with
+    # certain command key or name fields that you can
+    # search by.
+
+    def find_key(key, value)
+        keys = [key, key.upcase, key.downcase, key.capitalize]
+        keys.concat(keys.map &:intern)
+        self.find do |hash|
+            keys.any? { |key| hash[key] == value }
+        end
+    end
+
+    def bykey(key)
+        find_key 'key', key
+    end
+
+    def byname(name)
+        find_key 'name', name
+    end
+end

data/lib/bjn_inventory/bykey.rb

@@ -0,0 +1,7 @@
+module BjnInventory
+
+    class ByKey < Hash
+
+    end
+
+end

data/lib/bjn_inventory/context.rb

@@ -0,0 +1,60 @@
+require 'json'
+require 'bjn_inventory/hash'
+
+module BjnInventory
+
+    class Context < Hash
+
+        def initialize(initial_data={})
+            if initial_data.respond_to? :stringify_keys
+                self.merge! initial_data.stringify_keys
+            elsif File.directory? initial_data
+                self.load_directory('', initial_data)
+            elsif File.exists? initial_data
+                self.load_file('', initial_data)
+            else
+                raise Errno::ENOENT, "File not found to create context: #{initial_data.inspect}"
+            end
+        end
+
+        def load_directory(key, dir)
+            other = _load_directory(key, dir)
+            if key.empty?
+                self.merge! other['']
+            else
+                self.merge! _load_directory(key, dir)
+            end
+        end
+
+        def load_file(key, file)
+            if key.empty?
+                self.merge! JSON.parse(File.read(file))
+            else
+                self[key] = JSON.parse(File.read(file))
+            end
+        end
+
+        def _load_directory(key, dir)
+            { key =>
+                Dir.entries(dir).inject({}) do |h, entry|
+                    unless entry.start_with? '.'
+                        if File.directory? File.join(dir, entry)
+                            h.merge! _load_directory(entry, File.join(dir, entry))
+                        elsif entry =~ /\.json$/
+                            key = entry.sub(/\.json$/, '')
+                            value = JSON.parse(File.read(File.join(dir, entry)))
+                            if h.has_key? key and h[key].respond_to? :merge! and value.is_a? Hash
+                                h[key].merge! value
+                            else
+                                h[key] = value
+                            end
+                        end
+                    end
+                    h
+                end
+            }
+        end
+
+    end
+
+end

data/lib/bjn_inventory/data_files.rb

@@ -0,0 +1,41 @@
+
+module BjnInventory
+  def self.get_groups_data(inventory, ansible_spec)
+    ansible_inventory = inventory.to_ansible(ansible_spec)
+    groups_data = ansible_inventory.reject {|group, devices| group == '_meta'}
+    groups_data
+  end
+
+  def self.refresh_inventory_data(type, data, opt)
+    type_dir = "#{opt[:datadir]}/#{type}"
+    IO.write("#{opt[:datadir]}/#{type}.json", JSON.pretty_generate(data))
+    if !Dir.exist?(type_dir)
+      Dir.mkdir(type_dir)
+    end
+    refresh_inventory_entries(type, data, type_dir)
+  end
+
+  def self.refresh_inventory_entries(type, data, type_dir)
+    previous_entries = Dir.entries(type_dir).reject {|listing| (listing == '.' || listing == '..' || listing == "#{type}.json")}
+    previous_entries = previous_entries.map { |file| file.sub('.json', '') }
+    current_entries = data.keys
+
+    remove_entries = previous_entries - current_entries
+
+    remove_entries.each do |name|
+      entry = "#{type_dir}/#{name}.json"
+      File.delete(entry)
+    end
+
+    current_entries.each do |name|
+      entry = "#{type_dir}/#{name}.json"
+      tmp_file = "#{type_dir}/.#{name}.#{Process.pid}.tmp"
+      begin
+        IO.write(tmp_file, JSON.pretty_generate(data[name]))
+        IO.copy_stream(tmp_file, entry)
+      rescue
+      end
+      File.delete(tmp_file)
+    end
+  end
+end

data/lib/bjn_inventory/default_logger.rb

@@ -0,0 +1,15 @@
+require 'logger'
+
+module BjnInventory
+
+    class DefaultLogger < Logger
+
+        def initialize(progname='bjn_inventory')
+            logger = Logger.new $stderr
+            logger.level = Logger::WARN
+            logger
+        end
+
+    end
+
+end

data/lib/bjn_inventory/device.rb

@@ -0,0 +1,272 @@
+require 'ipaddr'
+require 'json'
+require 'jsonpath'
+require 'bjn_inventory/hash'
+require 'bjn_inventory/array'
+
+module BjnInventory
+
+    def self.map(origin, &block)
+        return block
+    end
+
+    DEFAULT_MODEL = {
+        name: nil,
+        service_level: nil,
+        environment: nil,
+        chef_runlist: [],
+        roles: [],
+        interfaces: [],
+        ipaddress: nil,
+        management_ipaddress: nil,
+        tags: [],
+        region: nil,
+        type: nil,
+        os: nil,
+        os_release: nil
+    }
+
+    class Device
+
+        @@model = DEFAULT_MODEL
+        @@context = { }
+
+        def self.use_context(data)
+            if data
+                @@context = @@context.merge(data.stringify_keys)
+            end
+        end
+
+        def self.use_model(model)
+            if model
+                @@model = model
+            else
+                @@model = DEFAULT_MODEL
+            end
+            self
+        end
+
+        def self.use_default_model()
+            @@model = DEFAULT_MODEL
+            self
+        end
+
+        def self.model()
+            @@model
+        end
+
+        def self.model=(new_model)
+            @@model = new_model
+        end
+
+        def self.maybe_block(args, block=nil, default=nil)
+            if args.length == 1
+                if args[0].respond_to? :call
+                    return args[0]
+                else
+                    # A constant
+                    return Proc.new { |_dummy| args[0] }
+                end
+            elsif block
+                return block
+            end
+            return default
+        end
+
+        def generate_getter(field)
+            self.define_singleton_method(field.intern) do |*args, &block|
+                value = nil
+                new_block = Device.maybe_block(args, block)
+                if new_block
+                    @rule[field.to_s] = new_block
+                else
+                    if @rule[field.to_s]
+                        begin
+                            value = @rule[field.to_s].call(@data, self)
+                        rescue Exception => err
+                            identifier_phrase = " for #{self.guessed_identity}"
+                            raise RuntimeError, "Error evaluating #{field.to_s} rule (#{err.class}: #{err.to_s})#{identifier_phrase}"
+                        end
+                    else
+                        value = @data[field.to_s]
+                    end
+                end
+                value
+            end
+        end
+
+        def guessed_identity
+            # try to find a way to summarize the name or id
+            # this needs to work regardless of rules (within rules) so it can't exercise actual name/id methods
+            identifier_key = %w(name id fqdn device_name device_id hostname instance_id).find { |key| @data[key] }
+            if identifier_key
+                identifier_phrase = "#{identifier_key}=#{@data[identifier_key]}"
+            else
+                identifier_phrase = "(no-id)"
+            end
+            identifier_phrase
+        end
+
+        def generate_setter(field)
+            self.define_singleton_method((field.to_s + '=').intern) do |new_value|
+                @data[field.to_s] = new_value
+            end
+        end
+
+        # Creates a prototype using the specified origin rules
+        # file (or rules text), which can be used to create new
+        # objects based on data, kind of like a class (but it's
+        # not a class, it's a prototype).
+        def self.using(rules=nil)
+            prototype = self.new()
+            if rules
+                prototype.load_rules(rules)
+            end
+            # Define an instance method new() that copies
+            # the prototype
+            prototype.define_singleton_method(:new) do |hash_data={}|
+                self.clone().set(hash_data)
+            end
+            prototype
+        end
+
+        def initialize(hash_data={})
+            @@model.each do |field, _dummy|
+                generate_getter(field)
+                generate_setter(field)
+            end
+            @rule = { }
+            @data = @@model.stringify_keys
+            self.set(hash_data)
+        end
+
+        def set(hash_data={})
+            @data = @data.merge(hash_data.stringify_keys)
+            self
+        end
+
+        def context()
+            @@context
+        end
+
+        def load_rules(origin)
+            if File.exist? origin
+                rules_text = File.read(origin)
+            else
+                rules_text = origin
+            end
+            self.instance_eval(rules_text)
+        end
+
+        # DSL - Set origin name
+        def origin(origin_name=nil)
+            if origin_name
+                @origin = origin_name
+            else
+                @origin
+            end
+        end
+
+        # DSL
+        # Allows: map field: ruby { |data| data['alias'] }
+        # Allows: map field: jsonpath 'expr'
+        # but it is actually unnecessary
+        def map(fieldmap)
+            fieldmap.each do |field, value|
+                self.send(field.intern, value)
+            end
+        end
+
+        # DSL
+        # Syntactic sugar for Proc.new basically. Allows:
+        # Allows: name ruby { |data| data['aliases'][0] || device['name'] }
+        # Allows: name ruby { |_dummy| name + '.domain.org' } ???
+        def ruby(&block)
+            block
+        end
+
+        # DSL
+        # Allows: name jsonpath '$.aliases[0]'
+        def jsonpath(expr)
+            Proc.new do |data|
+                value = JsonPath.on(data.to_json, expr).first
+                value
+            end
+        end
+
+        # DSL
+        # Allows: name synonym :model_field
+        def synonym(symbol)
+            Proc.new do |data, device|
+                device.send(symbol)
+            end
+        end
+
+        # DSL
+        # Allows: name always 'virtual'
+        def always(value)
+            Proc.new do |_data, _device|
+                value
+            end
+        end
+
+        def entry
+            @data
+        end
+
+        def validate()
+            # Raises exceptions if any of the getters barf, basically
+            @@model.each { |field, _default| self.send field.intern }
+            self
+        end
+
+        def merge(other)
+            # Use myself as prototype or create generic Device?
+            origins = self.origin.nil? ? [] : [@origin]
+
+            unless other.origin.nil?
+                origins.push other.origin
+            end
+
+            merged_data = Hash[@@model.map do |field, default|
+                                   mine = self.send field.intern
+                                   theirs = other.send field.intern
+
+                                   value = if mine.nil?
+                                               theirs
+                                           elsif theirs.nil?
+                                               mine
+                                           elsif default.respond_to? :merge
+                                               # The default value is a hash
+                                               begin
+                                                   mine.merge theirs
+                                               rescue Exception => err
+                                                   raise RuntimeError, "Error hash-merging field value #{theirs.inspect} " +
+                                                       "into #{mine.inspect} for #{self.guessed_identity}: #{err.to_s}"
+                                               end
+                                           elsif default.respond_to? :push
+                                               # The default value is an array
+                                               begin
+                                                   mine.concat theirs.reject { |el| mine.include? el }
+                                               rescue Exception => err
+                                                   raise RuntimeError, "Error concatenating field value #{theirs.inspect} " +
+                                                       "onto #{mine.inspect} for #{self.guessed_identity}: #{err.to_s}"
+                                               end
+                                           else
+                                               theirs
+                                           end
+                                   [field, value]
+                               end]
+
+            device = Device.new(merged_data)
+            device.origin origins
+            device
+        end
+
+
+        def to_hash()
+            Hash[@@model.map { |field, _default| [field.to_s, self.send(field.intern)] }]
+        end
+    end
+
+end