bjn_inventory 1.3.1 → 1.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9f4dd05a9291f7ca5aa3054f5247e7bdb48b2d8b
-  data.tar.gz: b26400af160b26f373de7bcc736c4b98e1995e12
+  metadata.gz: dcd99bdf01564c9b9328bd816c7d682dca6ac94d
+  data.tar.gz: 003a564bc69f38caa487927671c7f98e4eaa201e
 SHA512:
-  metadata.gz: 680f516945b414a66bfdf4a2ba4368d18e5738bfc5b9c179e15c3d7576cd492548e85f71bcdfbabd348dff1023b9dc003bf4af950c1b35d4cd37cebcdb68dcef
-  data.tar.gz: 42e9b62882481e6d18605e4bd9cb0e74f3fe2fb38232d0a19fd889e94ea5a543f9a180e844bc87949c9b5350cc89d48b549bb9f218e0930c9e222af5c8f4155a
+  metadata.gz: 891137548bf9420e4a7bd654d23776257d9d1253d4f349439183dee2efc45cfb00717ac3ab2e9321d96b587ab902a36161d1bcc103ea09487cde7931e657147d
+  data.tar.gz: e839db1948ea6d4bc9861edf16547bd0304f05e308606918e1b2a89e38511bdf11340bd712c7debe44ebf5eb7d7c24e23a3969c7d7d92b45a2571de2be295c39

data/.gitignore

@@ -9,7 +9,9 @@
 /spec/reports/
 /tmp/
 /d42/
+/cache/
 /aws/
 /inv/
 tasks/package/artifacts
 /tasks/test/reports/
+.ruby-version

data/README.md

@@ -196,6 +196,171 @@ to `ec2_instance` unconditionally:
 system_type always 'ec2_instance'
 ```
 
+## Commands
+
+### Formatters
+
+The inventory can be output in different formats. Two formatters
+are provided which can display the inventory as model-conformant
+devices: either as a JSON array or an object keyed by an identifying
+field (`inventory-model`); or in the
+[Ansible Dynamic Inventory](http://docs.ansible.com/ansible/latest/intro_dynamic_inventory.html)
+format.
+
+The `refresh_inventory_data` formatter formats the inventory into
+groups and devices in a file tree. A `devices.json` index is produced,
+with all devices, and each device also gets a file in `devices/<key>.json`.
+In addition the groups are listed in a `groups.json` index, and each
+has a list of devices in `groups/<group>.json`. This facilitates sharing
+the inventory over the web, as well (though **bjn_inventory** is not
+itself a network service).
+
+### Downloaders
+
+The overall design of the software encourages you to download entries
+from inventory sources in a "close to raw" manner, and let
+the source merging and mapping rules transform the entries into
+devices. Because the core inventory generation offers no mapping,
+the downloader is also the point at which to filter out entries
+that should not be devices.
+
+For convenience, AWS downloaders are provided for instances
+(`aws-ec2-source`), classic ELB (`aws-elb-source`) and RDS
+(`aws-rds-source`) resources. Run each with valid AWS credentials in
+your `~/.aws/credentials` file to see what the output looks like.
+They each accept a `--filter` argument: in the EC2 downloader's case,
+this is passed to the AWS API to filter instances according to the
+attributes and tags it offers; in the case of the ELB downloader,
+the syntax is the same, but it is enforced by an internal rules-
+matching library. The RDS downloader also does filtering based only
+on tags, using the `--tag-filters` option.
+
+### Service Maps
+
+A common use case for device inventories is to be able to present
+service endpoints calculated dynamically from inventory. These could
+be used as-is, imported into a service discovery system, etc. To
+facilitate this, a special formatter is provided which maps devices
+and groups into "service endpoints", which are defined by a service
+map.
+
+A service map is a JSON object stored in a file (and provided to the
+`service-map` command via the `--map` option), where the keys are
+service prefixes and the values are objects. It can be arbitrarily
+deeply nested, with the deepest object in the tree being a service
+specifier.
+
+A service specifier consists of the special field `hosts`, the value
+of which is a list of groups. Each device in all the groups is added
+to the service under the preceding prefix. The other fields in the
+service specifier are arbitrary; each is passed through unchanged, so
+that the "leaf" of the service map consists of an object where the
+service specifier's key is joined with each of the specifier fields by
+a dot (`.`); and the `hosts` field has each device's endpoint listed
+with the `join_with` character (by default, a comma; or as a JSON
+array).
+
+The groups are determined by a group specification: similar to the
+`ansible-from` command, a JSON object (in the file given by the
+`--groups` option) with a `"group_by"` key, the value of which is a
+list of fields to create groups by. Note that if you use
+the `ansible-from` command, you can use the same groups file; but
+Ansible's `"groups"` key, which specifies groups of groups, is ignored.
+
+In the service prefix (the nested keys and objects which precede the
+"leaf" of the service map), device fields can be given with a dollar sign
+(`$`) prepended; these will be interpolated with the actual devices'
+field values. This also means that trees can be copied into the map multiple
+times, once for every unique value of the field in the relevant groups.
+
+The above description sounds complicated, but the result is a fairly
+simple way to map service endpoints into an arbitrarily complex tree, and
+a couple of examples are probably best to illustrate the point. Let's say
+you have three devices in your inventory: two in the `us-west-2` region
+and one in the `eu-west-1` region. In your `us-west-2` region, one instance
+has the `web` role and one has the `db` role. In the `us-west-1` region,
+you just have a webserver. Your whole inventory looks like this (for
+example, if you run `inventory-model --manifest manifest.json`:
+
+```javascript
+[
+  { "name": "web-01",
+    "roles": ["web"],
+    "region": "us-west-2" },
+  { "name": "db-01",
+    "roles": ["db"],
+    "region": "us-west-2" },
+  { "name": "web-02",
+    "roles": ["web"],
+    "region": "eu-west-1" }
+]
+```
+ 
+You create the following service map:
+
+
+```javascript
+{
+  "services": {
+    "$region": {
+      "www": {
+        "hosts": ["web"],
+        "port": 80
+      }
+    }
+  },
+  "monitor": {
+    "$region": {
+      "nagios": {
+        "hosts": ["web", "db"],
+        "key": "monitoring-key.rsa"
+      }
+    }
+  }
+}
+```
+
+When you run `service-map --map map.json --manifest manifest.json --hosts-field name`, you
+get the following output:
+
+```javascript
+{
+  "services": {
+    "us-west-2": {
+      "www.hosts": "10.0.1.1",
+      "www.port": 80
+    },
+    "eu-west-1": {
+      "www.hosts": "10.0.10.1",
+      "www.port": 80
+    }
+  },
+  "monitor": {
+    "us-west-2": {
+      "nagios.hosts": "10.0.1.1,10.0.1.2",
+      "nagios.key": "monitoring-key.rsa"
+    },
+    "eu-west-1": {
+      "nagios.hosts": "10.0.10.1",
+      "nagios.key": "monitoring-key.rsa"
+    }
+  }
+}
+```
+
+If you need to have a key in your service map that starts with a
+dollar sign, use two dollar signs instead.
+
+## TODO
+
+* The `refresh_inventory_data` formatter needs to changed to be
+  more parallel with the other formatters (`--ansible` should
+  be `--groups`, it should probably be named `inventory-files`
+  or something.
+* The `aws-rds-source` should be refactored to take the `--filters`
+  argument and use **BjnInventory::Util::Filter::JsonAws** like
+  `aws-elb-source`.
+
 ## Design Decisions
 
 * No calculated fields in the model.

data/bin/ansible-from

@@ -17,7 +17,8 @@ parser = Trollop::Parser.new do
           ansible-from [options]
     USAGE
 
-    opt :ansible, 'Specify ansible groupings file', required: true, type: :string
+    opt :groups, 'Specify ansible groupings file', required: false, type: :string
+    opt :ansible, 'Synonym for --groups', required: false, type: :string
     opt :manifest, 'Specify the manifest that defines this inventory', required: true, type: :string
     opt :debug, 'Enable debug output', :short => '-D'
     opt :syslog, 'Log to Syslog', :short => '-S'
@@ -28,6 +29,8 @@ opt = Trollop::with_standard_exception_handling parser do
     parser.parse(ARGV)
 end
 
+opt[:groups] = opt[:ansible] if opt[:groups].nil?
+
 if opt[:syslog]
     require 'syslog/logger'
     logger = Syslog::Logger.new 'ansible-from'
@@ -35,14 +38,20 @@ else
     logger = Logger.new STDERR
 end
 
+
 if opt[:debug]
     logger.level = Logger::DEBUG
 else
     logger.level = Logger::WARN
 end
 
+if opt[:groups].nil?
+    logger.fatal "Need to specify --groups file"
+    exit 1
+end
+
 manifest = JSON.parse(File.read(opt[:manifest]))
-ansible_spec = JSON.parse(File.read(opt[:ansible]))
+ansible_spec = JSON.parse(File.read(opt[:groups]))
 
 inventory = BjnInventory::Inventory.new(manifest.merge({logger: logger}))
 puts JSON.pretty_generate(inventory.by('name').to_ansible(ansible_spec))

data/bin/aws-elb-source

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+
+require 'trollop'
+require 'logger'
+
+require 'bjn_inventory/version'
+require 'bjn_inventory/source_command/aws_elb'
+
+parser = Trollop::Parser.new do
+    version BjnInventory::VERSION
+    banner <<-USAGE.gsub(/^\s{8}/,'')
+        Usage:
+          aws-elb-source [options]
+          Download entries corresponding to classic elastic loadbalancers (not application load balancers)
+    USAGE
+
+    opt :region, "Specify region (default is all regions)", type: :string, multi: true
+    opt :profile, "Specify AWS client profile", type: :string
+    opt :filters, 'Specify AWS filters in JSON (e.g. `[{"name":"tag:ENVIRONMENT","values":["Production"]}]`)', type: :string
+    opt :output, "Send output to file instead of stdout", type: :string
+    opt :verbose, "Produce verbose output", short: '-v'
+    opt :debug, 'Enable debug output', short: '-D'
+    opt :syslog, 'Log to Syslog', short: '-S'
+    stop_on_unknown
+end
+
+opt = Trollop::with_standard_exception_handling parser do
+    parser.parse(ARGV)
+end
+
+if opt[:syslog]
+    require 'syslog/logger'
+    logger = Syslog::Logger.new 'aws-elb-source'
+else
+    logger = Logger.new STDERR
+end
+
+if opt[:debug]
+    logger.level = Logger::DEBUG
+elsif opt[:verbose]
+    logger.level = Logger::INFO
+else
+    logger.level = Logger::WARN
+end
+
+command = BjnInventory::SourceCommand::AwsElb.new(opt.merge({ logger: logger }))
+command.run

data/bin/{inventory_model → inventory-model}

@@ -15,7 +15,7 @@ parser = Trollop::Parser.new do
           inventory-devices [options]
     USAGE
 
-    opt :key, 'Specify the key of inventory hash', required: true, type: :string
+    opt :key, 'Specify the key of inventory hash', type: :string
     opt :manifest, 'Specify the manifest that defines this inventory', required: true, type: :string
     opt :debug, 'Enable debug output', :short => '-D'
     opt :syslog, 'Log to Syslog', :short => '-S'
@@ -31,4 +31,11 @@ logger.level = opt[:debug]? Logger::DEBUG : Logger::WARN
 
 manifest = JSON.parse(File.read(opt[:manifest]))
 inventory = BjnInventory::Inventory.new(manifest.merge({logger: logger}))
-puts JSON.pretty_generate(inventory.by(opt[:key]))
+
+if opt[:key]
+    results = inventory.by(opt[:key])
+else
+    results = inventory.devices.map { |device| device.to_hash }
+end
+
+puts JSON.pretty_generate(results)

data/bin/service-map

@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+
+# Create list of inventory devices based on device model
+
+require 'trollop'
+require 'json'
+require 'logger'
+require 'syslog/logger'
+require 'bjn_inventory'
+require 'bjn_inventory/service_map'
+
+parser = Trollop::Parser.new do
+    version BjnInventory::VERSION
+    banner <<-USAGE.gsub(/^\s{8}/,'')
+        Usage:
+          inventory-devices [options]
+    USAGE
+
+    opt :map, 'Specify file containing service map', required: true, type: :string
+    opt :groups, 'Specify file containing group definitions', required: true, type: :string
+    opt :manifest, 'Specify the manifest that defines this inventory', required: true, type: :string
+    opt :key, 'Specify device key field', type: :string, default: 'name'
+    opt :endpoint_fields, 'Specify the names of fields that can be used', default: ['endpoint', 'ip_address'], type: :strings
+    opt(:hosts_field,
+       'Specify the name of the service map field that contains the group specification and on which the endpoints are emitted',
+        default: 'hosts', type: :string, short: '-H')
+    opt :join_with, 'Use this character to join endpoints (or specify \'json\')', default: ',', type: :string
+    opt :debug, 'Enable debug output', :short => '-D'
+    opt :syslog, 'Log to Syslog', :short => '-S'
+    stop_on_unknown
+end
+
+opt = Trollop::with_standard_exception_handling parser do
+    parser.parse(ARGV)
+end
+
+logger = opt[:syslog]? Syslog::Logger.new('service-map') : Logger.new(STDERR)
+logger.level = opt[:debug]? Logger::DEBUG : Logger::WARN
+
+manifest = JSON.parse(File.read(opt[:manifest]))
+service_map = JSON.parse(File.read(opt[:map]))
+groups = JSON.parse(File.read(opt[:groups]))
+inventory = BjnInventory::Inventory.new(manifest.merge({logger: logger}))
+
+services = inventory.by(opt[:key]).to_services(map: service_map,
+                                               group_by: groups['group_by'],
+                                               hosts_field: opt[:hosts_field],
+                                               endpoint_fields: opt[:endpoint_fields],
+                                               join: (opt[:join_with] == 'json' ? :json : opt[:join_with]),
+                                               logger: logger)
+
+puts JSON.pretty_generate(services)

data/bjn_inventory.gemspec

@@ -19,7 +19,8 @@ Gem::Specification.new do |spec|
     f.match(%r{^(test|spec|features)/})
   end
   spec.bindir        = "bin"
-  spec.executables   = ["ansible-from", "aws-ec2-source", "aws-rds-source", "inventory_model", "refresh_inventory_data"]
+  spec.executables   = ["ansible-from", "aws-ec2-source", "aws-rds-source", "aws-elb-source",
+        "inventory-model", "refresh_inventory_data", "service-map"]
   spec.require_paths = ["lib"]
 
   spec.add_dependency 'jsonpath', "~> 0.7.2"

data/lib/bjn_inventory/ansible.rb

@@ -5,40 +5,9 @@ module BjnInventory
 
     class ByKey
 
-        def _ansible_name(value)
-            value.gsub(/[^a-zA-Z0-9_-]+/, '_')
-        end
-
-        def _field_groups(fields, device, sep='__')
-            fields = [fields] unless fields.respond_to? :inject
-            value_map = fields.map do |field|
-                values = device[field]
-                values = [values] unless values.respond_to? :map
-                values.map { |val| _ansible_name(val) }
-            end
-            #
-            # So now we have an array of arrays, eg.:
-            # fields='region' =>
-            # [['dc2']]
-            #
-            # fields=['roles', 'region'] =>
-            # [['web', ['dc2']]
-            #   'db'],
-            #
-            groups =
-                if fields.length == 1
-                    value_map.first
-                else
-                    driving_array, *rest = value_map
-                    driving_array.product(*rest).map { |compound_value| compound_value.join(sep) }
-                end
-            groups
-        end
-
-        # This basically builds an ansible inventory given a hash of hostvars
-        def to_ansible(*ansible_spec)
-            if ansible_spec[-1].respond_to? :to_hash
-                kwargs = ansible_spec.pop.stringify_keys
+        def to_ansible(*args)
+            if args[-1].respond_to? :to_hash
+                kwargs = args.pop.stringify_keys
             else
                 kwargs = { }
             end
@@ -47,31 +16,19 @@ module BjnInventory
                 group_by = kwargs['group_by']
                 group_by = [group_by] unless group_by.respond_to? :each
             end
-            group_by.concat(ansible_spec)
+            group_by.concat(args)
 
-            if group_by.empty?
-                raise ArgumentError, "Expected group_by either as keyword or as argument list"
-            end
-
-            logger ||= BjnInventory::DefaultLogger.new
-            # We need at least one field to create groups
+            logger = kwargs['logger'] || BjnInventory::DefaultLogger.new
             separator = kwargs['separator'] || '__'
 
-            ansible_inventory = { '_meta' => {'hostvars' => self.to_hash } }
-
-            self.each do |name, device_hash|
-                group_by.each do |group_field_spec|
-                    group_field_spec = [group_field_spec] unless group_field_spec.respond_to? :all?
-                    if group_field_spec.all? { |field| !device_hash[field].nil? && !device_hash[field].empty? }
-                        field_groups = _field_groups(group_field_spec, device_hash, separator)
-                        field_groups.each do |group_name|
-                            ansible_inventory[group_name] = [ ] unless ansible_inventory.has_key? group_name
-                            ansible_inventory[group_name] << name
-                        end
-                    end
-                end
-            end
+            ansible_inventory = self.to_groups(group_by: group_by, logger: logger, separator: separator).
+                merge({
+                          '_meta' => {
+                              'hostvars' => self.to_hash
+                          }
+                      })
 
+            # Do my own groups
             if kwargs['groups']
                 ansible_inventory.merge! Hash[kwargs['groups'].map do |group, children|
                                                   [group, { "hosts" => [ ], "children" => children }]

data/lib/bjn_inventory/bykey.rb

@@ -2,6 +2,70 @@ module BjnInventory
 
     class ByKey < Hash
 
+        def escape_name(value)
+            value.gsub(/[^a-zA-Z0-9_-]+/, '_')
+        end
+
+        def _field_groups(fields, device, sep='__')
+            fields = [fields] unless fields.respond_to? :inject
+            value_map = fields.map do |field|
+                values = device[field]
+                values = [values] unless values.respond_to? :map
+                values.map { |val| escape_name(val) }
+            end
+            #
+            # So now we have an array of arrays, eg.:
+            # fields='region' =>
+            # [['dc2']]
+            #
+            # fields=['roles', 'region'] =>
+            # [['web', ['dc2']]
+            #   'db'],
+            #
+            groups =
+                if fields.length == 1
+                    value_map.first
+                else
+                    driving_array, *rest = value_map
+                    driving_array.product(*rest).map { |compound_value| compound_value.join(sep) }
+                end
+            groups
+        end
+
+        # This basically builds an ansible inventory given a hash of hostvars
+        def to_groups(kwargs)
+            group_by = kwargs[:group_by]
+
+            if group_by.empty?
+                raise ArgumentError, "Expected group_by either as keyword or as argument list"
+            end
+
+            logger = kwargs[:logger] || BjnInventory::DefaultLogger.new
+            # We need at least one field to create groups
+            separator = kwargs[:separator] || '__'
+
+            groups = { }
+
+            self.each do |name, device_hash|
+                group_by.each do |group_field_spec|
+                    group_field_spec = [group_field_spec] unless group_field_spec.respond_to? :all?
+                    if group_field_spec.all? { |field| !device_hash[field].nil? && !device_hash[field].empty? }
+                        field_groups = _field_groups(group_field_spec, device_hash, separator)
+                        field_groups.each do |group_name|
+                            groups[group_name] = [ ] unless groups.has_key? group_name
+                            groups[group_name] << name
+                        end
+                    end
+                end
+            end
+
+            # I can't really do children, they need to be generators and I'm not
+            # about that right now. Save it for ansible.
+
+            groups
+        end
+
+
     end
 
 end

data/lib/bjn_inventory/default_logger.rb

@@ -6,7 +6,11 @@ module BjnInventory
 
         def initialize(progname='bjn_inventory')
             logger = Logger.new $stderr
-            logger.level = Logger::WARN
+            if ENV['BJN_INVENTORY_DEBUG'] and ! ENV['BJN_INVENTORY_DEBUG'].empty?
+                logger.level = Logger::DEBUG
+            else
+                logger.level = Logger::WARN
+            end
             logger
         end
 

data/lib/bjn_inventory/service_map.rb

@@ -0,0 +1,116 @@
+# Adds the .to_services method to inventory
+# which takes a grouping directive
+
+module BjnInventory
+
+    class ByKey
+
+        def to_services(kwargs)
+            @map = kwargs[:map]
+            @group_by = kwargs[:group_by]
+            @logger = kwargs[:logger] || BjnInventory::DefaultLogger.new
+
+            @hosts_field = kwargs[:hosts]               || 'hosts'
+            @endpoint_fields = kwargs[:endpoint_fields] || ['endpoint', 'ip_address']
+            @join = kwargs[:join]                       || ','
+            separator = kwargs[:separator]              || '__'
+
+            raise ArgumentError, "BjnInventory::ByKey#to_services requires a service map" unless @map
+            raise ArgumentError, "BjnInventory::ByKey#to_services requires a group_by argument" unless @group_by
+
+            # Build un-interpolated path => service correspondence
+            spec_keys = _pathlist(@map)
+            @logger.debug "paths in map: #{spec_keys.inspect}"
+
+            groups = self.to_groups(group_by: @group_by, separator: separator)
+
+            service_keylist = _interpolate_pathlist(spec_keys, groups)
+
+            services = { }
+
+            service_keylist.each do |path, service|
+                _deep_set_service(services, path, service.merge({ @hosts_field => _join_endpoints(service[@hosts_field]) }))
+            end
+
+            services
+        end
+
+        # Take a path--a list of key components, and deeply set them into a hash of hashes
+        def _deep_set_service(hsh, path, object)
+            car, *cdr = path
+            if cdr.empty?
+                # This is kind of weird, use-case-specific logic
+                object.each do |key, value|
+                    hsh[car + '.' + key] = value
+                end
+            else
+                hsh[car] = { } unless hsh.has_key? car
+                _deep_set_service(hsh[car], cdr, object)
+            end
+        end
+
+        # Take uninterpolated paths and expand them with the groups
+        # of devices we have
+        def _interpolate_pathlist(spec_keys, groups)
+            service_keylist = { }
+
+            # Interpolate based on hosts_field values
+            spec_keys.each do |path, service|
+                if service[@hosts_field]
+                    device_names = service[@hosts_field].map { |group| groups[group] || [] }.flatten
+                    device_names.each do |device_name|
+                        if self[device_name]
+                            interpolated_path = _interpolate_path(path, self[device_name])
+                            next if interpolated_path.any? { |el| el.nil? }
+                            unless service_keylist.has_key? interpolated_path
+                                service_keylist[interpolated_path] = service.merge({ @hosts_field => [] })
+                            end
+                            service_keylist[interpolated_path][@hosts_field] << _endpoint(self[device_name])
+                        end
+                    end
+                end
+            end
+            service_keylist
+        end
+
+        def _interpolate_path(path, device)
+            path.map do |component|
+                if component.start_with? '$$'
+                    component[1 .. -1]
+                elsif component.start_with? '$'
+                    device[component[1 .. -1]]
+                else
+                    component
+                end
+            end
+        end
+
+        def _endpoint(device)
+            @endpoint_fields.map { |field| device[field] }.reject { |e| e.nil? }.first
+        end
+
+        def _join_endpoints(endpoints)
+            case @join
+            when :json
+                endpoints.uniq.sort.to_json
+            else
+                endpoints.uniq.sort.join(@join)
+            end
+        end
+
+        # Create a hash where the keys are paths (arrays of path components)
+        # and the values are service specs
+        def _pathlist(this_map, prefix=[], cur={})
+            if this_map.any? { |k, v| ! v.respond_to? :keys }
+                cur[prefix] = this_map
+            else
+                this_map.map do |key, value|
+                    _pathlist(value, prefix + [key], cur)
+                end
+            end
+            cur
+        end
+
+    end
+
+end

data/lib/bjn_inventory/source_command/aws_elb.rb

@@ -0,0 +1,86 @@
+require 'aws-sdk'
+require 'bjn_inventory/source_command'
+require 'bjn_inventory/util/filter/json_aws'
+
+module BjnInventory
+
+    class SourceCommand
+
+        class AwsElb < BjnInventory::SourceCommand
+
+            def set_options(opt)
+                # Amazon only allows 20 names in describe_tags, so this is one way of accomplishing that
+                @page_size = opt[:page_size] || 20
+                @regions = nil
+                if opt[:region] and !opt[:region].empty?
+                    @regions = opt[:region].respond_to?(:to_ary) ? opt[:region] : [ opt[:region] ]
+                    region = @regions.first
+                else
+                    region = 'us-west-2'
+                end
+
+                @filters = BjnInventory::Util::Filter::JsonAws.new(filters: opt[:filters] || '[ ]', logger: logger)
+
+                @client = opt[:client]
+                @ec2_client = opt[:ec2_client]
+                @profile = opt[:profile]
+                ec2_client = @ec2_client || new_ec2_client(region)
+                if @regions.nil? or @regions.empty?
+                    logger.debug "Listing regions"
+                    @regions = ec2_client.describe_regions().regions.map { |region| region.region_name }
+                end
+            end
+
+            def new_ec2_client(region=nil)
+                opts = {
+                    region: (region || @regions.first)
+                }
+                opts.update({profile: @profile}) if @profile
+                Aws::EC2::Client.new(opts)
+            end
+
+            def new_client(region=nil)
+                opts = {
+                    region: (region || @regions.first)
+                }
+                opts.update({profile: @profile}) if @profile
+                Aws::ElasticLoadBalancing::Client.new(opts)
+            end
+
+            def retrieve_entries(override_client=nil)
+                entries = @regions.map do |region|
+                    client = override_client || @client || new_client(region)
+                    _retrieve_entries([], client)
+                end.flatten
+            end
+
+            def _retrieve_entries(current, client, marker=nil)
+                logger.debug "Describing load balancers (page #{marker})"
+                chunk_result = client.describe_load_balancers(marker: marker, page_size: @page_size)
+                logger.debug "... described"
+                marker = chunk_result.next_marker
+                if chunk_result.load_balancer_descriptions.length > 0
+                    chunk = Hash[chunk_result.load_balancer_descriptions.map { |lb| [lb.load_balancer_name, lb.to_h] }]
+                    logger.debug "Describing load balancer tags"
+                    # Amazon only allows 20 names to be submitted
+                    chunk_tags = Hash[client.describe_tags(load_balancer_names: chunk.keys).tag_descriptions.map do |lbtags|
+                                          [lbtags.load_balancer_name, { tags: lbtags.to_h[:tags] }]
+                                      end]
+                    logger.debug "... described"
+                    chunk_list = chunk.map { |lbname, lb| lb.merge(chunk_tags[lbname]) }
+                else
+                    chunk_list = [ ]
+                end
+                chunk_list = @filters.select(chunk_list)
+                if marker
+                    return _retrieve_entries(current + chunk_list, client, marker)
+                else
+                    return current + chunk_list
+                end
+            end
+
+        end
+
+    end
+
+end

data/lib/bjn_inventory/util/filter/json_aws.rb

@@ -0,0 +1,108 @@
+require 'json'
+require 'bjn_inventory/util/filter'
+
+module BjnInventory
+
+    module Util
+
+        class Filter
+
+            # Implements JSON filtering AWS-style, i.e. similar to
+            # how the EC2 API filters instances by tags and attributes
+            # when querying.
+            class JsonAws < BjnInventory::Util::Filter
+
+                def set_options(opt)
+                    @filters = JSON.parse(opt[:filters]).map do |filter|
+                        {
+                            name: filter['name'],
+                            matcher: _matcher(filter['values'])
+                        }
+                    end
+                end
+
+                def match(entity)
+                    _match_filters(entity, @filters)
+                end
+
+                def _matcher(values)
+                    match_one_list = values.map { |expression| _match_one expression }
+                    proc do |value|
+                        match_one_list.any? { |match_one| match_one.call value }
+                    end
+                end
+
+                def _glob2re(str)
+                    str.scan(/\\\*|\\\?|\?|\*|\\|[^\\\?\*]*/).map do |group|
+                        case group
+                        when '\?'
+                            '\?'
+                        when '\*'
+                            '\*'
+                        when '\\'
+                            '\\'
+                        when '*'
+                            '.*'
+                        when '?'
+                            '.'
+                        else
+                            Regexp::escape(group)
+                        end
+                    end.join('')
+                end
+
+                def _match_one(expression)
+                    if /\?|\*/.match expression
+                        rx  = Regexp.new('\A' + _glob2re(expression) + '\Z')
+                        proc do |value|
+                            rx.match value
+                        end
+                    else
+                        proc do |value|
+                            expression == value
+                        end
+                    end
+                end
+
+                # Since this is an "AWS" style filter, accepts entities
+                # with symbols for keys, the way you get when you do an
+                # AWS API call and do `.to_h`
+                def _match_filters(entity, filters)
+                    return true if filters.nil? or filters.empty?
+                    filter_expr = filters[0]
+                    name = filter_expr[:name]
+                    matcher = filter_expr[:matcher]
+                    this_expr =
+                        begin
+                            case name
+                            when 'tag-key'
+                                (entity[:tags] || [ ]).any? { |tag| matcher.call tag[:key] }
+                            when 'tag-value'
+                                (entity[:tags] || [ ]).any? { |tag| matcher.call tag[:value] }
+                            when /^tag:/
+                                tag_key = name[4 .. -1]
+                                entity_value = (entity[:tags] || [ ]).select { |tag| tag[:key] == tag_key }
+                                if ! entity_value.empty?
+                                    matcher.call entity_value[0][:value]
+                                else
+                                    false
+                                end
+                            else
+                                matcher.call entity[name.intern]
+                            end
+                        rescue StandardError => err
+                            logger.debug("rejected invalid entity #{entity.inspect}")
+                            false
+                        end
+                    return false unless this_expr
+
+                    _match_filters(entity, filters[1 .. -1])
+                end
+
+            end
+
+        end
+
+    end
+
+end

data/lib/bjn_inventory/util/filter.rb

@@ -0,0 +1,42 @@
+require 'bjn_inventory/default_logger'
+
+module BjnInventory
+
+    module Util
+
+        class Filter
+
+            def initialize(kwargs={})
+                @logger = kwargs.delete(:logger) || BjnInventory::DefaultLogger.new
+                unless kwargs.has_key? :filters
+                    raise RuntimeError, "#{self.class} requires :filters to be set"
+                end
+                @filters = kwargs[:filters]
+                set_options(kwargs)
+            end
+
+            def set_options(kwargs={})
+                @options = kwargs
+            end
+
+            def logger()
+                @logger
+            end
+
+            def match(entity)
+                if @filters.respond_to? :call
+                    @filters.call entity
+                else
+                    raise RuntimeError, "#{self.class} requires :filters to be callable"
+                end
+            end
+
+            def select(arr)
+                arr.select { |entity| match entity }
+            end
+
+        end
+
+    end
+
+end

data/lib/bjn_inventory/version.rb

@@ -1,3 +1,3 @@
 module BjnInventory
-  VERSION = "1.3.1"
+  VERSION = "1.5.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bjn_inventory
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.5.1
 platform: ruby
 authors:
 - Jeremy Brinkley
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-17 00:00:00.000000000 Z
+date: 2017-10-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jsonpath
@@ -115,8 +115,10 @@ executables:
 - ansible-from
 - aws-ec2-source
 - aws-rds-source
-- inventory_model
+- aws-elb-source
+- inventory-model
 - refresh_inventory_data
+- service-map
 extensions: []
 extra_rdoc_files: []
 files:
@@ -128,10 +130,12 @@ files:
 - Rakefile
 - bin/ansible-from
 - bin/aws-ec2-source
+- bin/aws-elb-source
 - bin/aws-rds-source
 - bin/console
-- bin/inventory_model
+- bin/inventory-model
 - bin/refresh_inventory_data
+- bin/service-map
 - bin/setup
 - bjn_inventory.gemspec
 - lib/bjn_inventory.rb
@@ -148,9 +152,13 @@ files:
 - lib/bjn_inventory/inventory/source.rb
 - lib/bjn_inventory/list.rb
 - lib/bjn_inventory/metadata.rb
+- lib/bjn_inventory/service_map.rb
 - lib/bjn_inventory/source_command.rb
 - lib/bjn_inventory/source_command/aws_ec2.rb
+- lib/bjn_inventory/source_command/aws_elb.rb
 - lib/bjn_inventory/source_command/aws_rds.rb
+- lib/bjn_inventory/util/filter.rb
+- lib/bjn_inventory/util/filter/json_aws.rb
 - lib/bjn_inventory/version.rb
 - lib/inventory.rb
 - tasks/package/_package.sh
@@ -186,3 +194,4 @@ specification_version: 4
 summary: Generate inventory lists based on flexible sources, rules and a standard
   device model
 test_files: []
+has_rdoc: