bjn_inventory 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +15 -0
  3. data/.rspec +2 -0
  4. data/.travis.yml +5 -0
  5. data/Gemfile +4 -0
  6. data/README.md +227 -0
  7. data/Rakefile +17 -0
  8. data/bin/ansible-from +48 -0
  9. data/bin/aws-ec2-source +46 -0
  10. data/bin/aws-rds-source +47 -0
  11. data/bin/console +14 -0
  12. data/bin/inventory_model +34 -0
  13. data/bin/refresh_inventory_data +51 -0
  14. data/bin/setup +8 -0
  15. data/bjn_inventory.gemspec +33 -0
  16. data/lib/bjn_inventory.rb +5 -0
  17. data/lib/bjn_inventory/ansible.rb +86 -0
  18. data/lib/bjn_inventory/array.rb +22 -0
  19. data/lib/bjn_inventory/bykey.rb +7 -0
  20. data/lib/bjn_inventory/context.rb +60 -0
  21. data/lib/bjn_inventory/data_files.rb +41 -0
  22. data/lib/bjn_inventory/default_logger.rb +15 -0
  23. data/lib/bjn_inventory/device.rb +272 -0
  24. data/lib/bjn_inventory/device/map.rb +18 -0
  25. data/lib/bjn_inventory/hash.rb +6 -0
  26. data/lib/bjn_inventory/inventory.rb +105 -0
  27. data/lib/bjn_inventory/inventory/source.rb +66 -0
  28. data/lib/bjn_inventory/list.rb +11 -0
  29. data/lib/bjn_inventory/metadata.rb +7 -0
  30. data/lib/bjn_inventory/source_command.rb +41 -0
  31. data/lib/bjn_inventory/source_command/aws_ec2.rb +58 -0
  32. data/lib/bjn_inventory/source_command/aws_rds.rb +92 -0
  33. data/lib/bjn_inventory/version.rb +3 -0
  34. data/lib/inventory.rb +12 -0
  35. data/tasks/package/_package.sh +131 -0
  36. data/tasks/package/_validate.sh +36 -0
  37. data/tasks/package/run.sh +41 -0
  38. data/tasks/package/validate.sh +41 -0
  39. data/tasks/package/validate/01version.sh +11 -0
  40. data/tasks/test/Dockerfile +14 -0
  41. data/tasks/test/run.sh +23 -0
  42. data/tools/packaging_tasks.rb +123 -0
  43. metadata +188 -0
@@ -0,0 +1,18 @@
1
+ module BjnInventory
2
+
3
+ def self.map(name, &block)
4
+ context = BjnInventory::Mapping.new(name)
5
+ context.instance_eval(&block)
6
+ end
7
+
8
+ class Mapping
9
+
10
+ def initialize()
11
+ @model = BjnInventory::Device::MODEL
12
+ end
13
+
14
+ def get(field, &block)
15
+ @field[field] = block
16
+ end
17
+
18
+ def from(
@@ -0,0 +1,6 @@
1
+ class Hash
2
+ def stringify_keys
3
+ Hash[self.map { |k, v| [k.to_s, v] }]
4
+ end
5
+ end
6
+
@@ -0,0 +1,105 @@
1
+ require 'bjn_inventory/inventory/source'
2
+ require 'bjn_inventory/context'
3
+ require 'bjn_inventory/default_logger'
4
+
5
+ module BjnInventory
6
+
7
+ # { "model": "device.json",
8
+ # "context": "data/context",
9
+ # "sources": [
10
+ # { "file": "data/device42.json",
11
+ # "map": "maps/device42.rb" },
12
+ # { "file": "data/aws.json",
13
+ # "map": "maps/aws.rb" }
14
+ # ]
15
+ # }
16
+
17
+ class Inventory
18
+
19
+ # Create a new Inventory according to the specified manifest (optional).
20
+ #
21
+ # @param manifest :model Optionally, a device model to use when listing
22
+ # entries (passed to BjnInventory::Device.use_model)
23
+ # @param manifest :context A directory or file containing context data
24
+ # to use when evaluating mapping rules.
25
+ # @param manifest :sources An array of source specifiers, each source
26
+ # specifier is used to create a new
27
+ # BjnInventory::Inventory::Source
28
+ def initialize(manifest={})
29
+ @manifest = manifest.stringify_keys
30
+ @logger = @manifest.delete('logger') || BjnInventory::DefaultLogger.new
31
+ # Check for data vs file
32
+ @model = @manifest['model'] || nil
33
+ # Check for data vs file
34
+ @context = @manifest['context'] || { }
35
+ @sources = [ ]
36
+ if @manifest.has_key? 'sources'
37
+ @manifest['sources'].each do |source|
38
+ @sources << BjnInventory::Inventory::Source.new(source.merge({logger: @logger,
39
+ model: make_model(@model),
40
+ context: BjnInventory::Context.new(@context) }))
41
+ end
42
+ end
43
+ end
44
+
45
+ # Return all of the devices in the inventory in a flat array, model-standardized
46
+ # by the inventory source (with rules applied, etc.)
47
+ def devices()
48
+ BjnInventory::List.new(@sources.map do |source|
49
+ source.devices
50
+ end.flatten)
51
+ end
52
+
53
+ def make_model(model_data)
54
+ if model_data.nil?
55
+ nil
56
+ elsif model_data.respond_to? :to_hash
57
+ model_data
58
+ else
59
+ # Must be a filename
60
+ JSON.parse(File.read(model_data))
61
+ end
62
+ end
63
+
64
+ # Return devices in the inventory as a hash, where the keys of the
65
+ # hash are the specified field values. For example, if the
66
+ # devices look like this:
67
+ # ```
68
+ # [
69
+ # { 'name' => 'testnode0', 'cost' => 15 }
70
+ # { 'name' => 'testnode1', 'cost' => 10 }
71
+ # ]
72
+ # ```
73
+ # then the result of inventory.by('name') would be:
74
+ # ```
75
+ # {
76
+ # 'testnode0' => { 'name' => 'testnode0', 'cost' => 15 },
77
+ # 'testnode1' => { 'name' => 'testnode1', 'cost' => 10 }
78
+ # }
79
+ # ```
80
+ # @param field The field value to use as the keys of the hash
81
+ def by(field)
82
+ BjnInventory::ByKey[devices_by(field).map { |key, device| [key, device.to_hash] }]
83
+ end
84
+
85
+ def devices_by(field)
86
+ field = field.intern
87
+ devices.inject(BjnInventory::ByKey.new) do |hash, device|
88
+ key = device.send field
89
+ if key and !key.empty?
90
+ if hash.has_key? key
91
+ hash[key] =
92
+ hash[key].merge(device)
93
+ else
94
+ hash[key] = device
95
+ end
96
+ else
97
+ @logger.warn "Skipping device with empty or null #{field.to_s}: ``#{device.inspect}''"
98
+ end
99
+ hash
100
+ end
101
+ end
102
+
103
+ end
104
+
105
+ end
@@ -0,0 +1,66 @@
1
+ require 'bjn_inventory/default_logger'
2
+
3
+ module BjnInventory
4
+
5
+ class Inventory
6
+
7
+ # The BjnInventory::Inventory::Source describes a source of inventory data: at its heart, a list
8
+ # (array) of entries, each of which is a hash, in the source's "native" schema.
9
+ class Source
10
+
11
+ # Create new instance with the entries supplied (or fetch the entries
12
+ # from the specified source.
13
+ # The constructor accepts the following keyword arguments:
14
+ # @param spec :file A JSON-formatted file containing a JSON array to
15
+ # read entries from
16
+ # @param spec :entries Directly-specified inventory source entries
17
+ # @param spec :rules A rules specification (either a file or a text)
18
+ def initialize(spec={ entries: { } })
19
+ spec = spec.stringify_keys
20
+ @context = spec.delete('context')
21
+ @model = spec.delete('model')
22
+ @rules = spec.delete('rules')
23
+ @devices = nil
24
+ @logger = spec.delete('logger') || BjnInventory::DefaultLogger.new
25
+ @spec = spec
26
+ source_type = spec.keys.first
27
+ case source_type
28
+ when 'entries'
29
+ @entries = spec[source_type]
30
+ when 'file'
31
+ @entries = JSON.parse(File.read(spec[source_type]))
32
+ else
33
+ raise ArgumentError.new("Unimplemented inventory source '#{source_type}'")
34
+ end
35
+ end
36
+
37
+ # Return a list of model-standardized device hashes
38
+ # @param kwargs :context Context data to be used when applying mapping rules
39
+ # @param kwargs :model The device model to be used. If not specified, the
40
+ # BjnInventory::Device default model is used.
41
+ def devices(kwargs={})
42
+ if @devices
43
+ @devices
44
+ else
45
+ BjnInventory::Device.use_context(@context)
46
+ BjnInventory::Device.use_model(@model)
47
+
48
+ @logger.debug("Creating device list from #{@spec}")
49
+ device_prototype = BjnInventory::Device.using(@rules)
50
+ @devices = BjnInventory::List.new(@entries.map do |entry|
51
+ begin
52
+ device_prototype.new(entry).validate
53
+ rescue Exception => err
54
+ @logger.error "#{err.class}: #{err.message}"
55
+ nil
56
+ end
57
+ end.reject { |entry| entry.nil? })
58
+ @devices
59
+ end
60
+ end
61
+
62
+ end
63
+
64
+ end
65
+
66
+ end
@@ -0,0 +1,11 @@
1
+ module BjnInventory
2
+
3
+ class List < Array
4
+
5
+ def to_hashes
6
+ self.map { |thing| thing.to_hash }
7
+ end
8
+
9
+ end
10
+
11
+ end
@@ -0,0 +1,7 @@
1
+ module BjnInventory
2
+ AUTHOR = 'Jeremy Brinkley'
3
+ EMAIL = 'jbrinkley@bluejeans.com'
4
+ LICENSE = 'All rights reserved'
5
+ SUMMARY = 'Generate inventory lists based on flexible sources, rules and a standard device model'
6
+ URL = 'https://git.corp.bluejeans.com:8443/projects/AS/repos/bjn_inventory/browse'
7
+ end
@@ -0,0 +1,41 @@
1
+ require 'bjn_inventory/default_logger'
2
+
3
+ module BjnInventory
4
+
5
+ class SourceCommand
6
+
7
+ # Intended to be the result of command-line parsing
8
+ def initialize(kwargs={})
9
+ @output = kwargs[:output] || nil
10
+ @logger = kwargs[:logger] || BjnInventory::DefaultLogger.new
11
+ set_options kwargs
12
+ end
13
+
14
+ def set_options(kwargs={})
15
+ @options = kwargs
16
+ end
17
+
18
+ def logger()
19
+ @logger
20
+ end
21
+
22
+ def run()
23
+ entries = retrieve_entries
24
+ if @output
25
+ tmpfile = File.join(File.dirname(@output), '.' + File.basename(@output) + '.tmp.' + Process.pid.to_s)
26
+ File.open(tmpfile, 'w') do |fh|
27
+ fh.write JSON.pretty_generate entries
28
+ end
29
+ File.rename(tmpfile, @output)
30
+ else
31
+ $stdout.write JSON.pretty_generate entries
32
+ end
33
+ end
34
+
35
+ def retrieve_entries
36
+ raise RuntimeError, "#{self.class} has no retrieve_entries() implementation - source type unsupported"
37
+ end
38
+
39
+ end
40
+
41
+ end
@@ -0,0 +1,58 @@
1
+ require 'aws-sdk'
2
+ require 'bjn_inventory/source_command'
3
+
4
+ module BjnInventory
5
+
6
+ class SourceCommand
7
+
8
+ class AwsEc2 < BjnInventory::SourceCommand
9
+
10
+ def set_options(opt)
11
+ @regions = nil
12
+ if opt[:region] and !opt[:region].empty?
13
+ @regions = opt[:region].respond_to?(:to_ary) ? opt[:region] : [ opt[:region] ]
14
+ region = @regions.first
15
+ else
16
+ region = 'us-west-2'
17
+ end
18
+
19
+ @filters = JSON.parse(opt[:filters] || '[ ]')
20
+
21
+ @client = opt[:client]
22
+ @profile = opt[:profile]
23
+ client = @client || new_client(region)
24
+ if @regions.nil? or @regions.empty?
25
+ @regions = client.describe_regions().regions.map { |region| region.region_name }
26
+ end
27
+ end
28
+
29
+ def new_client(region=nil)
30
+ opts = {
31
+ region: (region || @regions.first)
32
+ }
33
+ opts.update({profile: @profile}) if @profile
34
+ Aws::EC2::Client.new(opts)
35
+ end
36
+
37
+ def retrieve_entries(override_client=nil)
38
+ @regions.map do |region|
39
+ client = override_client || @client || new_client(region)
40
+ reservations = case @filters
41
+ when []
42
+ client.describe_instances()
43
+ else
44
+ client.describe_instances(filters: @filters)
45
+ end.reservations
46
+ reservations.map do |reservation|
47
+ reservation.instances.map do |instance|
48
+ instance.to_hash
49
+ end
50
+ end
51
+ end.flatten
52
+ end
53
+
54
+ end
55
+
56
+ end
57
+
58
+ end
@@ -0,0 +1,92 @@
1
+ require 'aws-sdk'
2
+ require 'bjn_inventory/source_command'
3
+
4
+ # Only retrieves clusters
5
+
6
+ module BjnInventory
7
+
8
+ class SourceCommand
9
+
10
+ class AwsRds < BjnInventory::SourceCommand
11
+
12
+ def set_options(opt)
13
+ @regions = nil
14
+ if opt[:region] and !opt[:region].empty?
15
+ @regions = opt[:region].respond_to?(:to_ary) ? opt[:region] : [ opt[:region] ]
16
+ region = @regions.first
17
+ else
18
+ region = 'us-west-2'
19
+ end
20
+
21
+ @filters = JSON.parse(opt[:filters] || '[ ]')
22
+
23
+ @tag_filters = JSON.parse(opt[:tag_filters] || '[ ]')
24
+
25
+ @client = opt[:client]
26
+ @profile = opt[:profile]
27
+ client = @client || new_client(region)
28
+ region_list = %w{
29
+ us-east-2
30
+ us-east-1
31
+ us-west-1
32
+ us-west-2
33
+ ca-central-1
34
+ ap-south-1
35
+ ap-northeast-2
36
+ ap-southeast-1
37
+ ap-southeast-2
38
+ ap-northeast-1
39
+ eu-central-1
40
+ eu-west-1
41
+ eu-west-2
42
+ sa-east-1
43
+ }
44
+ if @regions.nil? or @regions.empty?
45
+ @regions = region_list
46
+ end
47
+ end
48
+
49
+ def new_client(region=nil)
50
+ opts = {
51
+ region: (region || @regions.first)
52
+ }
53
+ opts.update({profile: @profile}) if @profile
54
+ Aws::RDS::Client.new(opts)
55
+ end
56
+
57
+ def retrieve_entries(override_client=nil)
58
+ filter_map = {}
59
+ @tag_filters.each do |tag_filter|
60
+ filter_map[tag_filter['key']] = tag_filter['values']
61
+ end
62
+
63
+ @regions.map do |region|
64
+ client = override_client || @client || new_client(region)
65
+ client.describe_db_clusters(filters: @filters).db_clusters.collect do |cluster|
66
+ tags = client.list_tags_for_resource(resource_name:cluster.db_cluster_arn)
67
+ cluster_hash = cluster.to_hash
68
+ tag_map = {}
69
+ filter_map.keys.each do |key|
70
+ tag_map[key] = false
71
+ end
72
+ cluster_hash['tags'] = tags.tag_list.map do |tag|
73
+ if filter_map.keys.include?(tag.key) && \
74
+ filter_map[tag.key].include?(tag.value)
75
+ tag_map[tag.key] = true
76
+ end
77
+ tag.to_hash
78
+ end
79
+ if tag_map.values.include? false
80
+ []
81
+ else
82
+ [ cluster_hash ]
83
+ end
84
+ end
85
+ end.flatten
86
+ end
87
+
88
+ end
89
+
90
+ end
91
+
92
+ end
@@ -0,0 +1,3 @@
1
+ module BjnInventory
2
+ VERSION = "1.3.0"
3
+ end
data/lib/inventory.rb ADDED
@@ -0,0 +1,12 @@
1
+ module BjnInventory
2
+
3
+ class Inventory
4
+
5
+ # Create a new Inventory
6
+ def initialize(manifest={})
7
+ @manifest = manifest
8
+ end
9
+
10
+ end
11
+
12
+ end
@@ -0,0 +1,131 @@
1
+ #!/usr/bin/env bash
2
+ set -x
3
+ set -e
4
+
5
+ NAME="$1"
6
+ ARTIFACTS="${2:-artifacts}"
7
+ distro="${3:-distro}"
8
+ prefix=/opt/${NAME#bjn-}
9
+
10
+ apt-get update
11
+
12
+ # Create packaging environment - using system ruby
13
+ if [ "$distro" = precise ]
14
+ then
15
+ apt-get install -y ruby1.9.1 ruby1.9.1-dev \
16
+ rubygems1.9.1 irb1.9.1 ri1.9.1 rdoc1.9.1 \
17
+ libopenssl-ruby1.9.1 libreadline-dev libffi-dev libssl-dev
18
+ update-alternatives --install /usr/bin/ruby ruby /usr/bin/ruby1.9.1 400 \
19
+ --slave /usr/share/man/man1/ruby.1.gz ruby.1.gz \
20
+ /usr/share/man/man1/ruby1.9.1.1.gz \
21
+ --slave /usr/bin/ri ri /usr/bin/ri1.9.1 \
22
+ --slave /usr/bin/irb irb /usr/bin/irb1.9.1 \
23
+ --slave /usr/bin/rdoc rdoc /usr/bin/rdoc1.9.1
24
+
25
+ update-alternatives --config ruby
26
+ update-alternatives --config gem
27
+ else
28
+ packager_deps=(ruby ruby-dev rubygems-integration build-essential libssl-dev libreadline-dev libffi-dev git)
29
+ apt-get install -y "${packager_deps[@]}"
30
+ fi
31
+ /usr/bin/gem install bundler
32
+ /usr/local/bin/bundle install
33
+ /usr/bin/gem install fpm
34
+
35
+
36
+ ## Build gem
37
+ rm -rf pkg/*.gem
38
+ /usr/bin/rake build
39
+
40
+ # Build Ruby from source
41
+ RUBY_SOURCE=https://cache.ruby-lang.org/pub/ruby/2.3/ruby-2.3.4.tar.gz
42
+ RUBY_SHA256SUM=98e18f17c933318d0e32fed3aea67e304f174d03170a38fd920c4fbe49fec0c3
43
+ RUBY_SOURCE_SIZE=17820518
44
+
45
+ build_deps=(build-essential git llvm zlib1g-dev libssl-dev libncurses5-dev libreadline6-dev libbz2-dev xz-utils)
46
+
47
+ package_deps=(zlib1g libssl1.0.0 libncurses5 libreadline6 libbz2-1.0 xz-utils)
48
+
49
+ # Prepare ruby build environment
50
+ apt-get install -y curl "${build_deps[@]}" "${package_deps[@]}"
51
+
52
+ # Build ground-up Ruby for self-contained package
53
+ rm -rf $prefix
54
+ mkdir -m 0775 -p $prefix
55
+ curl -O $RUBY_SOURCE
56
+ source_tarball=$(basename $RUBY_SOURCE)
57
+ actual_size=$(stat -c %s $source_tarball)
58
+ actual_sha256=$(sha256sum $source_tarball | awk '{print $1}')
59
+ if [ $actual_sha256 != $RUBY_SHA256SUM ]
60
+ then
61
+ echo "Ruby tarball invalid, SHA256 sum mismatch (expected $RUBY_SHA256SUM, got $actual_sha256)" >&2
62
+ exit 10
63
+ fi
64
+ if [ $actual_size != $RUBY_SOURCE_SIZE ]
65
+ then
66
+ echo "Ruby tarball invalid, size mismatch (expected $RUBY_SOURCE_SIZE, got $actual_size)" >&2
67
+ exit 11
68
+ fi
69
+
70
+ gzip -dc $source_tarball | tar xf -
71
+ (cd ${source_tarball%.t*} && \
72
+ ./configure --prefix=$prefix && \
73
+ make && \
74
+ make install)
75
+
76
+ rm -rf ${source_tarball%.t*}
77
+
78
+ export PATH=$prefix/bin:$PATH
79
+ $prefix/bin/gem install pkg/*.gem
80
+
81
+ ## Set packaging info
82
+ get_metadata() {
83
+ metadata_constant=$(echo $1 | tr '[[:lower:]]' '[[:upper:]]')
84
+ $prefix/bin/ruby -Ilib -rbjn_inventory/metadata -rbjn_inventory/version -e "puts BjnInventory::${metadata_constant}"
85
+ }
86
+
87
+ provides="${NAME}"
88
+ description=$(get_metadata summary)
89
+ license=$(get_metadata license)
90
+ url=$(get_metadata url)
91
+ maintainer="$(get_metadata author) <$(get_metadata email)>"
92
+ version=$(get_metadata version)
93
+ revision="$version-${BUILD_NUMBER:-1}+${distro}-$(git rev-parse --short HEAD)"
94
+
95
+
96
+ ## Hard system deps (will be coded as debian package deps)
97
+ declare -a fpm_dependencies
98
+ for dependency in "${package_deps[@]}"
99
+ do
100
+ fpm_dependencies[${#fpm_dependencies[@]}]=-d
101
+ fpm_dependencies[${#fpm_dependencies[@]}]="$dependency"
102
+ done
103
+
104
+ # Create the package
105
+ rm -f *.deb
106
+ fpm \
107
+ -s dir \
108
+ -t deb \
109
+ -n "${NAME}" \
110
+ -v "${revision}" \
111
+ "${fpm_dependencies[@]}" \
112
+ --provides "${provides}" \
113
+ --description "${description}" \
114
+ --maintainer "${maintainer}" \
115
+ --url "${url}" \
116
+ --exclude "tasks" \
117
+ --exclude "tests" \
118
+ --exclude ".git" \
119
+ --exclude "tmp" \
120
+ --license "${license}" \
121
+ "$prefix=/opt"
122
+
123
+ rm -rf $prefix
124
+
125
+ dpkg -i *.deb
126
+ ls -la $prefix
127
+ $prefix/bin/ansible-from --version
128
+ $prefix/bin/aws-ec2-source --version
129
+
130
+ mkdir -p $ARTIFACTS
131
+ mv *.deb $ARTIFACTS