bjn_inventory 1.3.0

data/lib/bjn_inventory/device/map.rb

@@ -0,0 +1,18 @@
+module BjnInventory
+
+    def self.map(name, &block)
+        context = BjnInventory::Mapping.new(name)
+        context.instance_eval(&block)
+    end
+
+    class Mapping
+
+        def initialize()
+            @model = BjnInventory::Device::MODEL
+        end
+
+        def get(field, &block)
+            @field[field] = block
+        end
+
+        def from(

data/lib/bjn_inventory/hash.rb

@@ -0,0 +1,6 @@
+class Hash
+    def stringify_keys
+        Hash[self.map { |k, v| [k.to_s, v] }]
+    end
+end
+

data/lib/bjn_inventory/inventory.rb

@@ -0,0 +1,105 @@
+require 'bjn_inventory/inventory/source'
+require 'bjn_inventory/context'
+require 'bjn_inventory/default_logger'
+
+module BjnInventory
+
+# { "model": "device.json",
+#   "context": "data/context",
+#   "sources": [
+#     { "file": "data/device42.json",
+#       "map": "maps/device42.rb" },
+#     { "file": "data/aws.json",
+#       "map": "maps/aws.rb" }
+#   ]
+# }
+
+    class Inventory
+
+        # Create a new Inventory according to the specified manifest (optional).
+        #
+        # @param manifest :model   Optionally, a device model to use when listing
+        #                          entries (passed to BjnInventory::Device.use_model)
+        # @param manifest :context A directory or file containing context data
+        #                          to use when evaluating mapping rules.
+        # @param manifest :sources An array of source specifiers, each source
+        #                          specifier is used to create a new
+        #                          BjnInventory::Inventory::Source
+        def initialize(manifest={})
+            @manifest = manifest.stringify_keys
+            @logger = @manifest.delete('logger') || BjnInventory::DefaultLogger.new
+            # Check for data vs file
+            @model   = @manifest['model']   || nil
+            # Check for data vs file
+            @context = @manifest['context'] || { }
+            @sources = [ ]
+            if @manifest.has_key? 'sources'
+                @manifest['sources'].each do |source|
+                    @sources << BjnInventory::Inventory::Source.new(source.merge({logger: @logger,
+                                                                                 model: make_model(@model),
+                                                                                 context: BjnInventory::Context.new(@context) }))
+                end
+            end
+        end
+
+        # Return all of the devices in the inventory in a flat array, model-standardized
+        # by the inventory source (with rules applied, etc.)
+        def devices()
+            BjnInventory::List.new(@sources.map do |source|
+                source.devices
+            end.flatten)
+        end
+
+        def make_model(model_data)
+            if model_data.nil?
+                nil
+            elsif model_data.respond_to? :to_hash
+                model_data
+            else
+                # Must be a filename
+                JSON.parse(File.read(model_data))
+            end
+        end
+
+        # Return devices in the inventory as a hash, where the keys of the
+        # hash are the specified field values. For example, if the
+        # devices look like this:
+        # ```
+        #   [
+        #     { 'name' => 'testnode0', 'cost' => 15 }
+        #     { 'name' => 'testnode1', 'cost' => 10 }
+        #   ]
+        # ```
+        # then the result of inventory.by('name') would be:
+        # ```
+        #   {
+        #     'testnode0' => { 'name' => 'testnode0', 'cost' => 15 },
+        #     'testnode1' => { 'name' => 'testnode1', 'cost' => 10 }
+        #   }
+        # ```
+        # @param field  The field value to use as the keys of the hash
+        def by(field)
+            BjnInventory::ByKey[devices_by(field).map { |key, device| [key, device.to_hash] }]
+        end
+
+        def devices_by(field)
+            field = field.intern
+            devices.inject(BjnInventory::ByKey.new) do |hash, device|
+                key = device.send field
+                if key and !key.empty?
+                    if hash.has_key? key
+                        hash[key] =
+                            hash[key].merge(device)
+                    else
+                        hash[key] = device
+                    end
+                else
+                    @logger.warn "Skipping device with empty or null #{field.to_s}: ``#{device.inspect}''"
+                end
+                hash
+            end
+        end
+
+    end
+
+end

data/lib/bjn_inventory/inventory/source.rb

@@ -0,0 +1,66 @@
+require 'bjn_inventory/default_logger'
+
+module BjnInventory
+
+    class Inventory
+
+        # The BjnInventory::Inventory::Source describes a source of inventory data: at its heart, a list
+        # (array) of entries, each of which is a hash, in the source's "native" schema.
+        class Source
+
+            # Create new instance with the entries supplied (or fetch the entries
+            # from the specified source.
+            # The constructor accepts the following keyword arguments:
+            # @param spec :file    A JSON-formatted file containing a JSON array to
+            #                      read entries from
+            # @param spec :entries Directly-specified inventory source entries
+            # @param spec :rules   A rules specification (either a file or a text)
+            def initialize(spec={ entries: { } })
+                spec = spec.stringify_keys
+                @context = spec.delete('context')
+                @model = spec.delete('model')
+                @rules = spec.delete('rules')
+                @devices = nil
+                @logger = spec.delete('logger') || BjnInventory::DefaultLogger.new
+                @spec = spec
+                source_type = spec.keys.first
+                case source_type
+                when 'entries'
+                    @entries = spec[source_type]
+                when 'file'
+                    @entries = JSON.parse(File.read(spec[source_type]))
+                else
+                    raise ArgumentError.new("Unimplemented inventory source '#{source_type}'")
+                end
+            end
+
+            # Return a list of model-standardized device hashes
+            # @param kwargs :context  Context data to be used when applying mapping rules
+            # @param kwargs :model    The device model to be used. If not specified, the
+            #                         BjnInventory::Device default model is used.
+            def devices(kwargs={})
+                if @devices
+                    @devices
+                else
+                    BjnInventory::Device.use_context(@context)
+                    BjnInventory::Device.use_model(@model)
+
+                    @logger.debug("Creating device list from #{@spec}")
+                    device_prototype = BjnInventory::Device.using(@rules)
+                    @devices = BjnInventory::List.new(@entries.map do |entry|
+                        begin
+                            device_prototype.new(entry).validate
+                        rescue Exception => err
+                            @logger.error "#{err.class}: #{err.message}"
+                            nil
+                        end
+                    end.reject { |entry| entry.nil? })
+                    @devices
+                end
+            end
+
+        end
+
+    end
+
+end

data/lib/bjn_inventory/list.rb

@@ -0,0 +1,11 @@
+module BjnInventory
+
+    class List < Array
+
+        def to_hashes
+            self.map { |thing| thing.to_hash }
+        end
+
+    end
+
+end

data/lib/bjn_inventory/metadata.rb

@@ -0,0 +1,7 @@
+module BjnInventory
+    AUTHOR    = 'Jeremy Brinkley'
+    EMAIL     = 'jbrinkley@bluejeans.com'
+    LICENSE   = 'All rights reserved'
+    SUMMARY   = 'Generate inventory lists based on flexible sources, rules and a standard device model'
+    URL       = 'https://git.corp.bluejeans.com:8443/projects/AS/repos/bjn_inventory/browse'
+end

data/lib/bjn_inventory/source_command.rb

@@ -0,0 +1,41 @@
+require 'bjn_inventory/default_logger'
+
+module BjnInventory
+
+    class SourceCommand
+
+        # Intended to be the result of command-line parsing
+        def initialize(kwargs={})
+            @output = kwargs[:output] || nil
+            @logger = kwargs[:logger] || BjnInventory::DefaultLogger.new
+            set_options kwargs
+        end
+
+        def set_options(kwargs={})
+            @options = kwargs
+        end
+
+        def logger()
+            @logger
+        end
+
+        def run()
+            entries = retrieve_entries
+            if @output
+                tmpfile = File.join(File.dirname(@output), '.' + File.basename(@output) + '.tmp.' + Process.pid.to_s)
+                File.open(tmpfile, 'w') do |fh|
+                    fh.write JSON.pretty_generate entries
+                end
+                File.rename(tmpfile, @output)
+            else
+                $stdout.write JSON.pretty_generate entries
+            end
+        end
+
+        def retrieve_entries
+            raise RuntimeError, "#{self.class} has no retrieve_entries() implementation - source type unsupported"
+        end
+
+    end
+
+end

data/lib/bjn_inventory/source_command/aws_ec2.rb

@@ -0,0 +1,58 @@
+require 'aws-sdk'
+require 'bjn_inventory/source_command'
+
+module BjnInventory
+
+    class SourceCommand
+
+        class AwsEc2 < BjnInventory::SourceCommand
+
+            def set_options(opt)
+                @regions = nil
+                if opt[:region] and !opt[:region].empty?
+                    @regions = opt[:region].respond_to?(:to_ary) ? opt[:region] : [ opt[:region] ]
+                    region = @regions.first
+                else
+                    region = 'us-west-2'
+                end
+
+                @filters = JSON.parse(opt[:filters] || '[ ]')
+
+                @client = opt[:client]
+                @profile = opt[:profile]
+                client = @client || new_client(region)
+                if @regions.nil? or @regions.empty?
+                    @regions = client.describe_regions().regions.map { |region| region.region_name }
+                end
+            end
+
+            def new_client(region=nil)
+                opts = {
+                    region: (region || @regions.first)
+                }
+                opts.update({profile: @profile}) if @profile
+                Aws::EC2::Client.new(opts)
+            end
+
+            def retrieve_entries(override_client=nil)
+                @regions.map do |region|
+                    client = override_client || @client || new_client(region)
+                    reservations = case @filters
+                    when []
+                      client.describe_instances()
+                    else
+                      client.describe_instances(filters: @filters)
+                    end.reservations
+                    reservations.map do |reservation|
+                        reservation.instances.map do |instance|
+                            instance.to_hash
+                        end
+                    end
+                end.flatten
+            end
+
+        end
+
+    end
+
+end

data/lib/bjn_inventory/source_command/aws_rds.rb

@@ -0,0 +1,92 @@
+require 'aws-sdk'
+require 'bjn_inventory/source_command'
+
+# Only retrieves clusters
+
+module BjnInventory
+
+    class SourceCommand
+
+        class AwsRds < BjnInventory::SourceCommand
+
+            def set_options(opt)
+                @regions = nil
+                if opt[:region] and !opt[:region].empty?
+                    @regions = opt[:region].respond_to?(:to_ary) ? opt[:region] : [ opt[:region] ]
+                    region = @regions.first
+                else
+                    region = 'us-west-2'
+                end
+
+                @filters = JSON.parse(opt[:filters] || '[ ]')
+
+                @tag_filters = JSON.parse(opt[:tag_filters] || '[ ]')
+
+                @client = opt[:client]
+                @profile = opt[:profile]
+                client = @client || new_client(region)
+                region_list = %w{
+                    us-east-2
+                    us-east-1
+                    us-west-1
+                    us-west-2
+                    ca-central-1
+                    ap-south-1
+                    ap-northeast-2
+                    ap-southeast-1
+                    ap-southeast-2
+                    ap-northeast-1
+                    eu-central-1
+                    eu-west-1
+                    eu-west-2
+                    sa-east-1
+                }
+                if @regions.nil? or @regions.empty?
+                    @regions = region_list
+                end
+            end
+
+            def new_client(region=nil)
+                opts = {
+                    region: (region || @regions.first)
+                }
+                opts.update({profile: @profile}) if @profile
+                Aws::RDS::Client.new(opts)
+            end
+
+            def retrieve_entries(override_client=nil)
+                filter_map = {}
+                @tag_filters.each do |tag_filter|
+                    filter_map[tag_filter['key']] = tag_filter['values']
+                end
+                    
+                @regions.map do |region|
+                    client = override_client || @client || new_client(region)
+                    client.describe_db_clusters(filters: @filters).db_clusters.collect do |cluster|
+                        tags = client.list_tags_for_resource(resource_name:cluster.db_cluster_arn)
+                        cluster_hash = cluster.to_hash
+                        tag_map = {}
+                        filter_map.keys.each do |key|
+                            tag_map[key] = false
+                        end
+                        cluster_hash['tags'] = tags.tag_list.map do |tag|
+                            if filter_map.keys.include?(tag.key) && \
+                               filter_map[tag.key].include?(tag.value)
+                                tag_map[tag.key] = true
+                            end
+                            tag.to_hash
+                        end
+                        if tag_map.values.include? false
+                            []
+                        else
+                            [ cluster_hash ]
+                        end
+                    end
+                end.flatten
+            end
+
+        end
+
+    end
+
+end

data/lib/bjn_inventory/version.rb

@@ -0,0 +1,3 @@
+module BjnInventory
+  VERSION = "1.3.0"
+end

data/lib/inventory.rb

@@ -0,0 +1,12 @@
+module BjnInventory
+
+    class Inventory
+
+        # Create a new Inventory
+        def initialize(manifest={})
+            @manifest = manifest
+        end
+
+    end
+
+end

data/tasks/package/_package.sh

@@ -0,0 +1,131 @@
+#!/usr/bin/env bash
+set -x
+set -e
+
+NAME="$1"
+ARTIFACTS="${2:-artifacts}"
+distro="${3:-distro}"
+prefix=/opt/${NAME#bjn-}
+
+apt-get update
+
+# Create packaging environment - using system ruby
+if [ "$distro" = precise ]
+then
+    apt-get install -y ruby1.9.1 ruby1.9.1-dev \
+            rubygems1.9.1 irb1.9.1 ri1.9.1 rdoc1.9.1 \
+            libopenssl-ruby1.9.1 libreadline-dev libffi-dev libssl-dev
+    update-alternatives --install /usr/bin/ruby ruby /usr/bin/ruby1.9.1 400 \
+                        --slave   /usr/share/man/man1/ruby.1.gz ruby.1.gz \
+                        /usr/share/man/man1/ruby1.9.1.1.gz \
+                        --slave   /usr/bin/ri ri /usr/bin/ri1.9.1 \
+                        --slave   /usr/bin/irb irb /usr/bin/irb1.9.1 \
+                        --slave   /usr/bin/rdoc rdoc /usr/bin/rdoc1.9.1
+
+    update-alternatives --config ruby
+    update-alternatives --config gem
+else
+    packager_deps=(ruby ruby-dev rubygems-integration build-essential libssl-dev libreadline-dev libffi-dev git)
+    apt-get install -y "${packager_deps[@]}"
+fi
+/usr/bin/gem install bundler
+/usr/local/bin/bundle install
+/usr/bin/gem install fpm
+
+
+## Build gem
+rm -rf pkg/*.gem
+/usr/bin/rake build
+
+# Build Ruby from source
+RUBY_SOURCE=https://cache.ruby-lang.org/pub/ruby/2.3/ruby-2.3.4.tar.gz
+RUBY_SHA256SUM=98e18f17c933318d0e32fed3aea67e304f174d03170a38fd920c4fbe49fec0c3
+RUBY_SOURCE_SIZE=17820518
+
+build_deps=(build-essential git llvm zlib1g-dev libssl-dev libncurses5-dev libreadline6-dev libbz2-dev xz-utils)
+
+package_deps=(zlib1g libssl1.0.0 libncurses5 libreadline6 libbz2-1.0 xz-utils)
+
+# Prepare ruby build environment
+apt-get install -y curl "${build_deps[@]}" "${package_deps[@]}"
+
+# Build ground-up Ruby for self-contained package
+rm -rf $prefix
+mkdir -m 0775 -p $prefix
+curl -O $RUBY_SOURCE
+source_tarball=$(basename $RUBY_SOURCE)
+actual_size=$(stat -c %s $source_tarball)
+actual_sha256=$(sha256sum $source_tarball | awk '{print $1}')
+if [ $actual_sha256 != $RUBY_SHA256SUM ]
+then
+    echo "Ruby tarball invalid, SHA256 sum mismatch (expected $RUBY_SHA256SUM, got $actual_sha256)" >&2
+    exit 10
+fi
+if [ $actual_size != $RUBY_SOURCE_SIZE ]
+then
+    echo "Ruby tarball invalid, size mismatch (expected $RUBY_SOURCE_SIZE, got $actual_size)" >&2
+    exit 11
+fi
+
+gzip -dc $source_tarball | tar xf -
+(cd ${source_tarball%.t*} && \
+ ./configure --prefix=$prefix && \
+ make && \
+ make install)
+
+rm -rf ${source_tarball%.t*}
+
+export PATH=$prefix/bin:$PATH
+$prefix/bin/gem install pkg/*.gem
+
+## Set packaging info
+get_metadata() {
+    metadata_constant=$(echo $1 | tr '[[:lower:]]' '[[:upper:]]')
+    $prefix/bin/ruby -Ilib -rbjn_inventory/metadata -rbjn_inventory/version -e "puts BjnInventory::${metadata_constant}"
+}
+
+provides="${NAME}"
+description=$(get_metadata summary)
+license=$(get_metadata license)
+url=$(get_metadata url)
+maintainer="$(get_metadata author) <$(get_metadata email)>"
+version=$(get_metadata version)
+revision="$version-${BUILD_NUMBER:-1}+${distro}-$(git rev-parse --short HEAD)"
+
+
+##  Hard system deps (will be coded as debian package deps)
+declare -a fpm_dependencies
+for dependency in "${package_deps[@]}"
+do
+  fpm_dependencies[${#fpm_dependencies[@]}]=-d
+  fpm_dependencies[${#fpm_dependencies[@]}]="$dependency"
+done
+
+# Create the package
+rm -f *.deb
+fpm \
+  -s dir \
+  -t deb \
+  -n "${NAME}" \
+  -v "${revision}" \
+  "${fpm_dependencies[@]}" \
+  --provides "${provides}" \
+  --description "${description}" \
+  --maintainer "${maintainer}" \
+  --url "${url}" \
+  --exclude "tasks" \
+  --exclude "tests" \
+  --exclude ".git" \
+  --exclude "tmp" \
+  --license "${license}" \
+  "$prefix=/opt"
+
+rm -rf $prefix
+
+dpkg -i *.deb
+ls -la $prefix
+$prefix/bin/ansible-from --version
+$prefix/bin/aws-ec2-source --version
+
+mkdir -p $ARTIFACTS
+mv *.deb $ARTIFACTS