barnyard_aws 0.0.1

data/.gitignore

@@ -0,0 +1,6 @@
+*.gem
+*.idea
+.bundle
+Gemfile.lock
+pkg/*
+1-BarnyardHarvester::*

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/.rvmrc

@@ -0,0 +1 @@
+rvm use ruby-1.9.3-p125@barnyard

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in aws_auditor.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,23 @@
+Copyright (c) 2012 Jon Gillies
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

data/README.md

@@ -0,0 +1,82 @@
+# BarnyardAws
+
+This gem provides access to the following Amazon AWS objects:
+
+* BarnyardAws::AwsElbs
+* BarnyardAws::AwsSecurityGroups
+* BarnyardAws::AwsInstances
+* BarnyardAws::AwsSnapshots
+* BarnyardAws::AwsVolumes
+* BarnyardAws::AwsSubnets
+* BarnyardAws::AwsIamUsers
+* BarnyardAws::AwsIamGroupPolicies
+
+This gem requires the "barnyard_havester" gem.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'barnyard_aws'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install barnyard_aws
+
+## Usage
+
+Using this gem is trival.  It will iterate the AWS objects and send them to the BarnyardHarvester.
+
+WARNING!  Do not use crop number 1-9 in production, they are reserved for system testing.
+
+      require "barnyard_aws"
+
+      my_logger = Logger.new(STDOUT)
+      my_logger.level = Logger::INFO
+
+      redis_settings = {
+          :host => "localhost",
+          :port => 6379,
+      }
+
+      BarnyardAws::AwsElbs.new(aws_access_key_id: AWS_ACCESS_KEY_ID,
+                               aws_secret_access_key: AWS_SECRET_ACCESS_KEY,
+                               region: "us-west-1",
+                               account_id: "dev",
+                               crop_number: 1,
+                               logger: my_logger,
+                               redis_settings: redis_settings,
+                               debug: true)
+
+A note about primary keys using BarnyardAws.  The "account_id" is added befor the AWS ObjectID separated by a dash.  It is imparative that you use unique "account_ids" across different aws accounts.  The account_id can be any string and is not tied to the AWS API.  It is merly for reference.
+
+## Testing
+
+RSPEC is used to test this gem.  Note that you should run each spec test individually.  For some reason they confilict if run together.  You will need Redis installed on the localhost:6379 or adjust the test accordingly.
+
+This tests all of the AWS supported objects:
+
+    rspec spec/aws_spec.rb
+
+Note that this test requires the AWS credentails be set in the environment:
+
+    AWS_ACCESS_KEY_ID="****"
+    AWS_SECRET_ACCESS_KEY="****"
+
+This test will use the crop_number 1 and will iterate all supported AWS objects.
+
+## Questions or Problems?
+
+supercoder@gmail.com
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/barnyard_aws.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "barnyard_aws/version"
+
+Gem::Specification.new do |gem|
+  gem.name          = "barnyard_aws"
+  gem.version       = BarnyardAws::VERSION
+  gem.authors       = ["Jon Gillies"]
+  gem.email         = %w(supercoder@gmail.com)
+  gem.description   = %q{This gem provides access to the Amazon AWS objects for use with BarnyardHarvester.}
+  gem.summary       = %q{Please check the README.md for more information.}
+  gem.homepage      = "https://github.com/jongillies/barnyard/tree/master/barnyard_aws"
+
+  gem.rubyforge_project = "barnyard_aws"
+
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = %w(lib)
+
+  # specify any dependencies here; for example:
+  gem.add_development_dependency "rspec"
+  gem.add_runtime_dependency "barnyard_harvester"
+  gem.add_runtime_dependency "fog"
+
+end

data/lib/barnyard_aws/version.rb

@@ -0,0 +1,3 @@
+module BarnyardAws
+  VERSION = "0.0.1"
+end

data/lib/barnyard_aws.rb

@@ -0,0 +1,252 @@
+require "fog"
+require "barnyard_harvester"
+require "json"
+
+require "barnyard_aws/version"
+
+module BarnyardAws
+
+  class AwsObject
+
+    attr_reader :synchronizer, :began_at, :ended_at
+
+    def make_primary_key(account, key)
+      "#{account}-#{key}"
+    end
+
+    def initialize(args={})
+
+      @began_at = Time.now
+
+      @debug = args.fetch(:debug) { false }
+      @log = args.fetch(:logger) { Logger.new(STDOUT) }
+
+      @aws_access_key_id = args.fetch(:aws_access_key_id) { raise "You must provide :aws_access_key_id" }
+      @aws_secret_access_key = args.fetch(:aws_secret_access_key) { raise "You must provide :aws_secret_access_key" }
+      @region = args.fetch(:region) { raise "You must provide :region" }
+      @account_id = args.fetch(:account_id) { raise "You must provide :account_id" }
+
+      @log.debug JSON.pretty_generate(args)
+      @log.debug "region=#{@region}"
+
+      @synchronizer = BarnyardHarvester::Sync.new(args)
+
+      @synchronizer.run do
+
+        case self.class.to_s
+          when "BarnyardAws::AwsElbs"
+            compute = Fog::AWS::ELB.new({
+                                            :aws_access_key_id => @aws_access_key_id,
+                                            :aws_secret_access_key => @aws_secret_access_key,
+                                            :region => @region
+                                        })
+          when "BarnyardAws::AwsIamUsers",
+              "BarnyardAws::AwsIamGroupPolicies"
+            compute = Fog::AWS::IAM.new({
+                                            :aws_access_key_id => @aws_access_key_id,
+                                            :aws_secret_access_key => @aws_secret_access_key,
+                                            #:region => @region
+                                        })
+          else
+            compute = Fog::Compute.new({
+                                           :provider => 'AWS',
+                                           :aws_access_key_id => @aws_access_key_id,
+                                           :aws_secret_access_key => @aws_secret_access_key,
+                                           :region => @region
+                                       })
+        end
+
+        case self.class.to_s
+          when "BarnyardAws::AwsSecurityGroups"
+            compute.security_groups.each do |security_group|
+              unless security_group.group_id.nil?
+                obj = Crack::JSON.parse security_group.to_json
+                obj["account_id"] = @account_id
+
+                primary_key = make_primary_key(@account_id, security_group.group_id)
+                @log.info "#{primary_key}"
+
+                @synchronizer.process primary_key, obj
+
+              end
+            end
+
+          when "BarnyardAws::AwsInstances"
+            compute.servers.each do |s|
+              unless s.id.nil?
+                obj = Crack::JSON.parse s.to_json
+                obj["account_id"] = @account_id
+
+                primary_key = make_primary_key(@account_id, s.id)
+                @log.info "#{primary_key}"
+
+                @synchronizer.process primary_key, obj
+
+              end
+            end
+
+          when "BarnyardAws::AwsSnapshots"
+            compute.snapshots.each do |snapshot|
+              unless snapshot.id.nil?
+                obj = Crack::JSON.parse snapshot.to_json
+                obj["account_id"] = @account_id
+
+                primary_key = make_primary_key(@account_id, snapshot.id)
+                @log.info "#{primary_key}"
+
+                @synchronizer.process primary_key, obj
+
+              end
+            end
+
+          when "BarnyardAws::AwsVolumes"
+            compute.volumes.each do |s|
+              unless s.id.nil?
+                obj = Crack::JSON.parse s.to_json
+                obj["account_id"] = @account_id
+
+                primary_key = make_primary_key(@account_id, s.id)
+                @log.info "#{primary_key}"
+
+                @synchronizer.process primary_key, obj
+
+              end
+            end
+
+          when "BarnyardAws::AwsSubnets"
+            compute.subnets.each do |s|
+              unless s.subnet_id.nil?
+                obj = Crack::JSON.parse s.to_json
+                obj["account_id"] = @account_id
+
+                primary_key = make_primary_key(@account_id, s.subnet_id)
+                @log.info "#{primary_key}"
+
+                @synchronizer.process primary_key, obj
+
+              end
+            end
+
+          when "BarnyardAws::AwsElbs"
+            compute.load_balancers.each do |elb|
+              unless elb.dns_name.nil?
+                obj = Crack::JSON.parse elb.to_json
+                obj["account_id"] = @account_id
+
+                primary_key = make_primary_key(@account_id, elb.dns_name)
+                @log.info "#{primary_key}"
+
+                @synchronizer.process primary_key, obj
+
+              end
+            end
+
+          when "BarnyardAws::AwsIamUsers"
+            compute.list_users.body["Users"].each do |user|
+              unless user['UserId'].nil?
+                access_keys = compute.list_access_keys({'UserName' => user['UserName']}).body["AccessKeys"]
+                obj = Crack::JSON.parse user.to_json
+                obj["account_id"] = @account_id
+                obj["access_keys"] = access_keys
+
+                primary_key = make_primary_key(@account_id, user['UserId'])
+                @log.info "#{primary_key}"
+
+                @synchronizer.process primary_key, obj
+
+              end
+            end
+
+          when "BarnyardAws::AwsIamGroupPolicies"
+
+            obj = Hash.new
+            obj["account_id"] = @account_id
+
+            compute.list_groups.body["Groups"].each do |group|
+              obj = group
+              obj["account_id"] = @account_id
+              obj["policies"] = Array.new
+
+              compute.list_group_policies(group["GroupName"]).body["PolicyNames"].each do |policy|
+
+                begin
+                  policy_obj = Crack::JSON.parse CGI::unescape(compute.get_group_policy(policy, group["GroupName"]).body["PolicyDocument"])
+
+                  obj["policies"] << policy_obj
+
+                rescue Exception => e
+                  @log.warn "Unable to Crack policy >#{compute.get_group_policy(policy, group["GroupName"]).body["PolicyDocument"]}<"
+                end
+
+              end
+
+              primary_key = make_primary_key(@account_id, group["GroupId"])
+              @log.info "#{primary_key}"
+
+              @synchronizer.process primary_key, obj
+
+            end
+
+        end
+
+
+      end
+
+      @log.info @synchronizer.stats
+
+    end
+
+    @ended_at = Time.now
+
+  end
+
+  class AwsIamGroupPolicies < AwsObject
+    def initialize args
+      super args
+    end
+  end
+
+  class AwsIamUsers < AwsObject
+    def initialize args
+      super args
+    end
+  end
+
+  class AwsElbs < AwsObject
+    def initialize args
+      super args
+    end
+  end
+  class AwsSnapshots < AwsObject
+    def initialize args
+      super args
+    end
+  end
+
+  class AwsVolumes < AwsObject
+    def initialize args
+      super args
+    end
+  end
+
+  class AwsSubnets < AwsObject
+    def initialize args
+      super args
+    end
+  end
+
+  class AwsSecurityGroups < AwsObject
+    def initialize args
+      super args
+    end
+  end
+
+  class AwsInstances < AwsObject
+    def initialize args
+      super args
+    end
+  end
+
+
+end
+

data/spec/aws_spec.rb

@@ -0,0 +1,36 @@
+require "barnyard_aws"
+require "logger"
+
+describe BarnyardAws do
+
+  AWS_SECRET_ACCESS_KEY=ENV["AWS_SECRET_ACCESS_KEY"]
+  AWS_ACCESS_KEY_ID=ENV["AWS_ACCESS_KEY_ID"]
+
+  it "all should work" do
+
+    my_logger = Logger.new(STDOUT)
+    my_logger.level = Logger::INFO
+
+    redis_settings = {
+        :host => "localhost",
+        :port => 6379,
+    }
+
+    [BarnyardAws::AwsElbs, BarnyardAws::AwsSecurityGroups,
+     BarnyardAws::AwsInstances, BarnyardAws::AwsSnapshots,
+     BarnyardAws::AwsVolumes, BarnyardAws::AwsSubnets,
+     BarnyardAws::AwsIamUsers, BarnyardAws::AwsIamGroupPolicies].each do |o|
+
+      o.new(aws_access_key_id: AWS_ACCESS_KEY_ID,
+            aws_secret_access_key: AWS_SECRET_ACCESS_KEY,
+            region: "us-west-1",
+            account_id: "dev",
+            crop_number: 1,
+            logger: my_logger,
+            redis_settings: redis_settings,
+            debug: true)
+
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,17 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+end

metadata

@@ -0,0 +1,93 @@
+--- !ruby/object:Gem::Specification
+name: barnyard_aws
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Jon Gillies
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-11-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70304793803660 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70304793803660
+- !ruby/object:Gem::Dependency
+  name: barnyard_harvester
+  requirement: &70304793802920 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70304793802920
+- !ruby/object:Gem::Dependency
+  name: fog
+  requirement: &70304793802120 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70304793802120
+description: This gem provides access to the Amazon AWS objects for use with BarnyardHarvester.
+email:
+- supercoder@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .DS_Store
+- .gitignore
+- .rspec
+- .rvmrc
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- barnyard_aws.gemspec
+- lib/barnyard_aws.rb
+- lib/barnyard_aws/version.rb
+- spec/aws_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/jongillies/barnyard/tree/master/barnyard_aws
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: barnyard_aws
+rubygems_version: 1.8.15
+signing_key: 
+specification_version: 3
+summary: Please check the README.md for more information.
+test_files:
+- spec/aws_spec.rb
+- spec/spec_helper.rb