crop-duster 0.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/Gemfile +3 -0
- data/Gemfile.lock +73 -0
- data/LICENSE.txt +22 -0
- data/README.md +24 -0
- data/Rakefile +60 -0
- data/bin/crop-duster +6 -0
- data/crop-duster.gemspec +30 -0
- data/db/migrate/20140113115505_create_volumes.rb +17 -0
- data/db/migrate/20140113115506_create_servers.rb +15 -0
- data/db/migrate/20140113115507_create_tags.rb +10 -0
- data/db/migrate/20140113120016_create_availability_zones.rb +8 -0
- data/db/migrate/20140113120017_create_flavors.rb +41 -0
- data/db/migrate/20140113192351_add_hourly_price_to_servers_and_volumes.rb +6 -0
- data/db/migrate/20140113204901_create_aws_import_table.rb +26 -0
- data/db/migrate/20140113214652_create_indexes.rb +19 -0
- data/db/migrate/20140114141226_add_type_and_account_id_to_servers.rb +6 -0
- data/db/migrate/20140114161225_add_index_to_usage_type.rb +5 -0
- data/lib/crop_duster/cli.rb +47 -0
- data/lib/crop_duster/importer.rb +81 -0
- data/lib/crop_duster/version.rb +5 -0
- data/lib/crop_duster.rb +61 -0
- data/lib/models/availability_zone.rb +12 -0
- data/lib/models/aws_billing_import.rb +4 -0
- data/lib/models/flavor.rb +5 -0
- data/lib/models/server.rb +33 -0
- data/lib/models/tag.rb +3 -0
- data/lib/models/volume.rb +43 -0
- metadata +184 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 482a4f648759ef823f98109266ed74dbcb539f69
|
4
|
+
data.tar.gz: 964dd97a1c21a9f98f37d08977be33aab1f298ae
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: bdf1940a9637197a7d504cc159e3e0d4b5ce4e655a936800cf5767c197c341f865282f8afd435cd41d0fbf18b6752ac174ba69c0f8fe4dfe96dfd3c7fdc48c7f
|
7
|
+
data.tar.gz: 318ef45c68818fc635fe37e305cf63f75b927e3182472668db1f466765586e05466b5424dd2f06abb0bbbb9fc87d1332c986a63930d1514c2342c8baa5e833fe
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,73 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
crop-duster (0.0.1)
|
5
|
+
activerecord (~> 4.0.2)
|
6
|
+
activesupport (~> 4.0.2)
|
7
|
+
bundler (~> 1.3)
|
8
|
+
fog (~> 1.19.0)
|
9
|
+
pg (~> 0.17.1)
|
10
|
+
thor (~> 0.18.1)
|
11
|
+
|
12
|
+
GEM
|
13
|
+
remote: https://rubygems.org/
|
14
|
+
specs:
|
15
|
+
activemodel (4.0.2)
|
16
|
+
activesupport (= 4.0.2)
|
17
|
+
builder (~> 3.1.0)
|
18
|
+
activerecord (4.0.2)
|
19
|
+
activemodel (= 4.0.2)
|
20
|
+
activerecord-deprecated_finders (~> 1.0.2)
|
21
|
+
activesupport (= 4.0.2)
|
22
|
+
arel (~> 4.0.0)
|
23
|
+
activerecord-deprecated_finders (1.0.3)
|
24
|
+
activesupport (4.0.2)
|
25
|
+
i18n (~> 0.6, >= 0.6.4)
|
26
|
+
minitest (~> 4.2)
|
27
|
+
multi_json (~> 1.3)
|
28
|
+
thread_safe (~> 0.1)
|
29
|
+
tzinfo (~> 0.3.37)
|
30
|
+
arel (4.0.1)
|
31
|
+
atomic (1.1.14)
|
32
|
+
builder (3.1.4)
|
33
|
+
dotenv (0.9.0)
|
34
|
+
excon (0.31.0)
|
35
|
+
fog (1.19.0)
|
36
|
+
builder
|
37
|
+
excon (~> 0.31.0)
|
38
|
+
formatador (~> 0.2.0)
|
39
|
+
mime-types
|
40
|
+
multi_json (~> 1.0)
|
41
|
+
net-scp (~> 1.1)
|
42
|
+
net-ssh (>= 2.1.3)
|
43
|
+
nokogiri (~> 1.5)
|
44
|
+
ruby-hmac
|
45
|
+
foreman (0.63.0)
|
46
|
+
dotenv (>= 0.7)
|
47
|
+
thor (>= 0.13.6)
|
48
|
+
formatador (0.2.4)
|
49
|
+
i18n (0.6.9)
|
50
|
+
mime-types (2.0)
|
51
|
+
mini_portile (0.5.2)
|
52
|
+
minitest (4.7.5)
|
53
|
+
multi_json (1.8.4)
|
54
|
+
net-scp (1.1.2)
|
55
|
+
net-ssh (>= 2.6.5)
|
56
|
+
net-ssh (2.7.0)
|
57
|
+
nokogiri (1.6.1)
|
58
|
+
mini_portile (~> 0.5.0)
|
59
|
+
pg (0.17.1)
|
60
|
+
rake (10.1.1)
|
61
|
+
ruby-hmac (0.4.0)
|
62
|
+
thor (0.18.1)
|
63
|
+
thread_safe (0.1.3)
|
64
|
+
atomic
|
65
|
+
tzinfo (0.3.38)
|
66
|
+
|
67
|
+
PLATFORMS
|
68
|
+
ruby
|
69
|
+
|
70
|
+
DEPENDENCIES
|
71
|
+
crop-duster!
|
72
|
+
foreman (~> 0.63)
|
73
|
+
rake (~> 10.1.1)
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2014 Chris Winslett
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
# AWS Crop Duster
|
2
|
+
|
3
|
+
Fly over your AWS servers and get information that you can query on.
|
4
|
+
Sometimes, AWS just doesn't give you the information you need. You want
|
5
|
+
to build a system that will allow you to query changes in your AWS
|
6
|
+
stack. With Crop Duster, you can both query state and transition.
|
7
|
+
|
8
|
+
## Running & Installation
|
9
|
+
|
10
|
+
Crop Duster is a stand alone application for mfile:
|
11
|
+
|
12
|
+
gem install crop-duster
|
13
|
+
|
14
|
+
And then execute:
|
15
|
+
|
16
|
+
$ crop-duster AWS_SECRET=<IAM secret> AWS_KEY=<IAM key> AWS_BILLING_BUCKET=<S3 Bucket>
|
17
|
+
|
18
|
+
## Contributing
|
19
|
+
|
20
|
+
1. Fork it
|
21
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
22
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
23
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
24
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1,60 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
require 'logger'
|
3
|
+
require 'active_record'
|
4
|
+
|
5
|
+
require "bundler/gem_tasks"
|
6
|
+
|
7
|
+
namespace :db do
|
8
|
+
def create_database config
|
9
|
+
options = {:charset => 'utf8', :collation => 'utf8_unicode_ci'}
|
10
|
+
|
11
|
+
create_db = lambda do |config|
|
12
|
+
ActiveRecord::Base.establish_connection config.merge('database' => nil)
|
13
|
+
ActiveRecord::Base.connection.create_database config['database'], options
|
14
|
+
ActiveRecord::Base.establish_connection config
|
15
|
+
end
|
16
|
+
|
17
|
+
create_db.call config
|
18
|
+
end
|
19
|
+
|
20
|
+
task :environment do
|
21
|
+
DATABASE_ENV = ENV['DATABASE_ENV'] || 'development'
|
22
|
+
MIGRATIONS_DIR = ENV['MIGRATIONS_DIR'] || 'db/migrate'
|
23
|
+
end
|
24
|
+
|
25
|
+
task :configuration => :environment do
|
26
|
+
@config = YAML.load_file('config/databases.yml')[DATABASE_ENV]
|
27
|
+
end
|
28
|
+
|
29
|
+
task :configure_connection => :configuration do
|
30
|
+
ActiveRecord::Base.establish_connection @config
|
31
|
+
ActiveRecord::Base.logger = Logger.new STDOUT if @config['logger']
|
32
|
+
end
|
33
|
+
|
34
|
+
desc 'Create the database from config/database.yml for the current DATABASE_ENV'
|
35
|
+
task :create => :configure_connection do
|
36
|
+
create_database @config
|
37
|
+
end
|
38
|
+
|
39
|
+
desc 'Drops the database for the current DATABASE_ENV'
|
40
|
+
task :drop => :configure_connection do
|
41
|
+
ActiveRecord::Base.connection.drop_database @config['database']
|
42
|
+
end
|
43
|
+
|
44
|
+
desc 'Migrate the database (options: VERSION=x, VERBOSE=false).'
|
45
|
+
task :migrate => :configure_connection do
|
46
|
+
ActiveRecord::Migration.verbose = true
|
47
|
+
ActiveRecord::Migrator.migrate MIGRATIONS_DIR, ENV['VERSION'] ? ENV['VERSION'].to_i : nil
|
48
|
+
end
|
49
|
+
|
50
|
+
desc 'Rolls the schema back to the previous version (specify steps w/ STEP=n).'
|
51
|
+
task :rollback => :configure_connection do
|
52
|
+
step = ENV['STEP'] ? ENV['STEP'].to_i : 1
|
53
|
+
ActiveRecord::Migrator.rollback MIGRATIONS_DIR, step
|
54
|
+
end
|
55
|
+
|
56
|
+
desc "Retrieves the current schema version number"
|
57
|
+
task :version => :configure_connection do
|
58
|
+
puts "Current version: #{ActiveRecord::Migrator.current_version}"
|
59
|
+
end
|
60
|
+
end
|
data/bin/crop-duster
ADDED
data/crop-duster.gemspec
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'crop_duster/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "crop-duster"
|
8
|
+
spec.version = Crop::Duster::VERSION
|
9
|
+
spec.authors = ["Chris Winslett"]
|
10
|
+
spec.email = ["chris@mongohq.com"]
|
11
|
+
spec.description = %q{Fly low and slow over your AWS servers looking for weeds}
|
12
|
+
spec.summary = %q{A system for AWS server change management.}
|
13
|
+
spec.homepage = "http://www.mongohq.com"
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files`.split($/)
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_development_dependency "rake", "~> 10.1.1"
|
22
|
+
spec.add_development_dependency "foreman", "~> 0.63"
|
23
|
+
|
24
|
+
spec.add_dependency "bundler", "~> 1.3"
|
25
|
+
spec.add_dependency 'thor', "~> 0.18.1"
|
26
|
+
spec.add_dependency 'fog', "~> 1.19.0"
|
27
|
+
spec.add_dependency 'activesupport', "~> 4.0.2"
|
28
|
+
spec.add_dependency 'activerecord', "~> 4.0.2"
|
29
|
+
spec.add_dependency 'pg', "~> 0.17.1"
|
30
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
class CreateVolumes < ActiveRecord::Migration
|
2
|
+
def change
|
3
|
+
create_table :volumes do |t|
|
4
|
+
t.string :name
|
5
|
+
t.string :state
|
6
|
+
t.string :remote_id
|
7
|
+
t.integer :size
|
8
|
+
t.integer :server_id
|
9
|
+
t.integer :availability_zone_id
|
10
|
+
t.string :volume_type
|
11
|
+
t.integer :iops
|
12
|
+
t.string :last_import_id
|
13
|
+
t.datetime :created_at
|
14
|
+
t.datetime :deleted_at
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
class CreateServers < ActiveRecord::Migration
|
2
|
+
def change
|
3
|
+
create_table :servers do |t|
|
4
|
+
t.string :name
|
5
|
+
t.string :state
|
6
|
+
t.string :remote_id
|
7
|
+
t.integer :availability_zone_id
|
8
|
+
t.integer :flavor_id
|
9
|
+
t.boolean :ebs_optimized
|
10
|
+
t.string :last_import_id
|
11
|
+
t.datetime :created_at
|
12
|
+
t.datetime :deleted_at
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
class CreateFlavors < ActiveRecord::Migration
|
2
|
+
def change
|
3
|
+
create_table :flavors do |t|
|
4
|
+
t.string :name
|
5
|
+
t.float :hourly_rate
|
6
|
+
t.float :ebs_optimized
|
7
|
+
end
|
8
|
+
|
9
|
+
[
|
10
|
+
["c3.xlarge", 0.300, 0.02],
|
11
|
+
["c3.2xlarge", 0.600, 0.05],
|
12
|
+
["c3.4xlarge", 1.200, 0.10],
|
13
|
+
["c3.8xlarge", 2.400, 0.05],
|
14
|
+
["c1.medium", 0.145],
|
15
|
+
["c1.xlarge", 0.580],
|
16
|
+
["cc2.8xlarge", 2.400],
|
17
|
+
["g2.2xlarge", 0.650, 0.05],
|
18
|
+
["cg1.4xlarge", 2.100],
|
19
|
+
["m2.xlarge", 0.410],
|
20
|
+
["m2.2xlarge", 0.820, 0.025],
|
21
|
+
["m2.4xlarge", 1.640, 0.05],
|
22
|
+
["cr1.8xlarge", 3.500],
|
23
|
+
["i2.xlarge", 0.853, 0.02],
|
24
|
+
["i2.2xlarge", 1.705, 0.05],
|
25
|
+
["i2.4xlarge", 3.410, 0.10],
|
26
|
+
["i2.8xlarge", 6.820],
|
27
|
+
["hs1.8xlarge", 4.600],
|
28
|
+
["hi1.4xlarge", 3.100],
|
29
|
+
["t1.micro", 0.020],
|
30
|
+
["m3.xlarge", 0.450, 0.025],
|
31
|
+
["m3.2xlarge", 0.900, 0.05],
|
32
|
+
["m1.small", 0.060],
|
33
|
+
["m1.medium", 0.120],
|
34
|
+
["m1.large", 0.240, 0.025],
|
35
|
+
["m1.xlarge", 0.480, 0.05],
|
36
|
+
["c3.large", 0.150]
|
37
|
+
].each do |name, hourly_rate, ebs_optimized|
|
38
|
+
ActiveRecord::Base.connection.execute("INSERT INTO flavors (name, hourly_rate, ebs_optimized) VALUES ('#{name}', #{hourly_rate}, #{ebs_optimized || 0})")
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
class CreateAwsImportTable < ActiveRecord::Migration
|
2
|
+
def change
|
3
|
+
create_table :aws_billing_imports, id: false do |t|
|
4
|
+
t.string :invoice_id
|
5
|
+
t.string :payer_account_id
|
6
|
+
t.string :linked_account_id
|
7
|
+
t.string :record_type
|
8
|
+
t.string :record_id
|
9
|
+
t.string :product_name
|
10
|
+
t.string :rate_id
|
11
|
+
t.string :subscription_id
|
12
|
+
t.string :pricing_plan_id
|
13
|
+
t.string :usage_type
|
14
|
+
t.string :operation
|
15
|
+
t.string :availability_zone
|
16
|
+
t.string :reserved_instance
|
17
|
+
t.string :item_description
|
18
|
+
t.datetime :usage_start_date
|
19
|
+
t.datetime :usage_end_date
|
20
|
+
t.float :usage_quantity
|
21
|
+
t.float :rate
|
22
|
+
t.float :cost
|
23
|
+
t.string :resource_id
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
class CreateIndexes < ActiveRecord::Migration
|
2
|
+
def change
|
3
|
+
add_index :volumes, :remote_id
|
4
|
+
add_index :volumes, :server_id
|
5
|
+
add_index :volumes, :availability_zone_id
|
6
|
+
add_index :volumes, :last_import_id
|
7
|
+
add_index :servers, :remote_id
|
8
|
+
add_index :servers, :availability_zone_id
|
9
|
+
add_index :servers, :flavor_id
|
10
|
+
add_index :servers, :last_import_id
|
11
|
+
add_index :tags, [:taggable_id, :taggable_type]
|
12
|
+
|
13
|
+
add_index :aws_billing_imports, :operation
|
14
|
+
add_index :aws_billing_imports, :item_description
|
15
|
+
add_index :aws_billing_imports, :resource_id
|
16
|
+
add_index :aws_billing_imports, :usage_start_date
|
17
|
+
add_index :aws_billing_imports, :usage_end_date
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
require 'thor'
|
2
|
+
|
3
|
+
module CropDuster
|
4
|
+
class CLI < Thor
|
5
|
+
|
6
|
+
desc "migrate", "Migrate the database to the latest version"
|
7
|
+
option :db, required: true
|
8
|
+
def migrate
|
9
|
+
CropDuster.configure do |config|
|
10
|
+
config.db = options[:db]
|
11
|
+
end
|
12
|
+
|
13
|
+
ActiveRecord::Migration.verbose = true
|
14
|
+
ActiveRecord::Migrator.migrate File.join(File.dirname(__FILE__), "..", "..", "db", "migrate")
|
15
|
+
end
|
16
|
+
|
17
|
+
desc "import", "Migrate the database to the latest version"
|
18
|
+
option :db, required: true
|
19
|
+
option :aws_secret, required: true
|
20
|
+
option :aws_key, required: true
|
21
|
+
option :aws_billing_bucket
|
22
|
+
def import
|
23
|
+
set_configs(options[:db], options[:aws_secret], options[:aws_key], options[:aws_billing_bucket])
|
24
|
+
CropDuster::Importer.import
|
25
|
+
end
|
26
|
+
|
27
|
+
desc "import_billing", "Imports billing information from AWS"
|
28
|
+
option :db, required: true
|
29
|
+
option :aws_secret, required: true
|
30
|
+
option :aws_key, required: true
|
31
|
+
option :aws_billing_bucket, required: true
|
32
|
+
def import_billing
|
33
|
+
set_configs(options[:db], options[:aws_secret], options[:aws_key], options[:aws_billing_bucket])
|
34
|
+
CropDuster::Importer.import_billing
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
def set_configs(db_url, aws_secret, aws_key, aws_billing_bucket = nil)
|
39
|
+
CropDuster.configure do |config|
|
40
|
+
config.db = db_url
|
41
|
+
config.aws_secret = aws_secret
|
42
|
+
config.aws_key = aws_key
|
43
|
+
config.aws_billing_bucket = aws_billing_bucket
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
module CropDuster
|
2
|
+
class Importer
|
3
|
+
|
4
|
+
REGIONS = {
|
5
|
+
aws: ['ap-northeast-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-west-1', 'sa-east-1', 'us-east-1', 'us-west-1', 'us-west-2']
|
6
|
+
}
|
7
|
+
|
8
|
+
class << self
|
9
|
+
def import
|
10
|
+
import_id = SecureRandom.hex
|
11
|
+
|
12
|
+
self.import_servers(import_id)
|
13
|
+
|
14
|
+
if CropDuster.aws_billing_bucket.present?
|
15
|
+
self.import_billing(import_id)
|
16
|
+
else
|
17
|
+
puts "No billing bucket provided, so precise AWS billing information is unknown."
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def import_servers(import_id = SecureRandom.hex)
|
22
|
+
REGIONS[:aws].each do |region|
|
23
|
+
compute = Fog::Compute.new CropDuster.aws_keys.merge(region: region)
|
24
|
+
|
25
|
+
compute.servers.each do |s|
|
26
|
+
server = Server.import(s)
|
27
|
+
server.update_attributes(last_import_id: import_id)
|
28
|
+
end
|
29
|
+
|
30
|
+
compute.volumes.each do |v|
|
31
|
+
volume = Volume.import(v)
|
32
|
+
volume.update_attributes(last_import_id: import_id)
|
33
|
+
end
|
34
|
+
|
35
|
+
#compute.snapshots.each do |hash|
|
36
|
+
#snapshot = Snapshot.import(hash)
|
37
|
+
#snapshot.update_attributes(last_import_id: import_id)
|
38
|
+
#end
|
39
|
+
|
40
|
+
#Snapshot.where("last_import_id != ? AND deleted_at IS NULL", import_id).update_all(deleted_at: Time.now)
|
41
|
+
end
|
42
|
+
|
43
|
+
Server.where("last_import_id != ? AND deleted_at IS NULL", import_id).update_all(deleted_at: Time.now)
|
44
|
+
Volume.where("last_import_id != ? AND deleted_at IS NULL", import_id).update_all(deleted_at: Time.now)
|
45
|
+
end
|
46
|
+
|
47
|
+
def import_billing(import_id = SecureRandom.hex)
|
48
|
+
ActiveRecord::Base.connection.execute("TRUNCATE aws_billing_imports")
|
49
|
+
s = Fog::Storage.new CropDuster.aws_keys
|
50
|
+
path = "/tmp/#{import_id}"
|
51
|
+
Dir.mkdir path
|
52
|
+
|
53
|
+
s.directories.get(CropDuster.aws_billing_bucket).files.find_all { |f| f.key =~ /-aws-billing-detailed-line-items-with-resources-and-tags-/ }.each do |file|
|
54
|
+
`curl -o #{path}/file.zip '#{file.url(15.minutes.since)}'`
|
55
|
+
`unzip -p #{path}/file.zip > #{path}/#{Time.now.strftime('%Y%m%d%H%M%S.csv')}`
|
56
|
+
`rm #{path}/file.zip`
|
57
|
+
end
|
58
|
+
|
59
|
+
Dir.new(path).each do |file|
|
60
|
+
next if file =~ /^\./
|
61
|
+
headers = nil
|
62
|
+
|
63
|
+
ar_conn = ActiveRecord::Base.connection_pool.checkout
|
64
|
+
conn = ar_conn.raw_connection
|
65
|
+
conn.copy_data "COPY aws_billing_imports FROM STDIN CSV" do
|
66
|
+
File.new(File.join(path, file)).each_line do |line|
|
67
|
+
if headers.nil?
|
68
|
+
headers = line.strip.split(/,/).map(&:underscore)
|
69
|
+
else
|
70
|
+
conn.put_copy_data line
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
ActiveRecord::Base.connection_pool.checkin(ar_conn)
|
75
|
+
end
|
76
|
+
|
77
|
+
`rm #{path}/*.csv`
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
data/lib/crop_duster.rb
ADDED
@@ -0,0 +1,61 @@
|
|
1
|
+
require 'active_record'
|
2
|
+
require 'active_support'
|
3
|
+
require 'fog'
|
4
|
+
|
5
|
+
module CropDuster
|
6
|
+
|
7
|
+
class << self
|
8
|
+
def db=(url)
|
9
|
+
db_url = URI.parse(url)
|
10
|
+
options = (db_url.query || "").split(/\&/).inject({}) { |hash, s| hash[s.split(/=/)[0].to_sym] = s.split(/=/)[1]; hash }
|
11
|
+
|
12
|
+
config_hash = {
|
13
|
+
adapter: db_url.scheme,
|
14
|
+
host: db_url.host,
|
15
|
+
port: db_url.port,
|
16
|
+
database: db_url.path.gsub(/^\//, ''),
|
17
|
+
username: db_url.user,
|
18
|
+
password: db_url.password
|
19
|
+
}
|
20
|
+
|
21
|
+
config_hash.delete_if { |k, v| v.nil? }
|
22
|
+
|
23
|
+
ActiveRecord::Base.establish_connection config_hash.merge(options)
|
24
|
+
end
|
25
|
+
|
26
|
+
def aws_secret=(string)
|
27
|
+
@aws_secret = string
|
28
|
+
end
|
29
|
+
|
30
|
+
def aws_key=(string)
|
31
|
+
@aws_key = string
|
32
|
+
end
|
33
|
+
|
34
|
+
def aws_keys
|
35
|
+
{:provider => 'AWS', :aws_access_key_id => @aws_key, :aws_secret_access_key => @aws_secret}
|
36
|
+
end
|
37
|
+
|
38
|
+
def aws_billing_bucket=(string)
|
39
|
+
@aws_billing_bucket = string
|
40
|
+
end
|
41
|
+
|
42
|
+
def aws_billing_bucket
|
43
|
+
@aws_billing_bucket
|
44
|
+
end
|
45
|
+
|
46
|
+
def configure(&block)
|
47
|
+
block.call(self)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
Dir.new(File.join(File.dirname(__FILE__), "models")).each do |file|
|
53
|
+
next if file =~ /^\./
|
54
|
+
require File.join(File.dirname(__FILE__), "models", file)
|
55
|
+
end
|
56
|
+
|
57
|
+
Dir.new(File.join(File.dirname(__FILE__), "crop_duster")).each do |file|
|
58
|
+
next if file =~ /^\./
|
59
|
+
require File.join(File.dirname(__FILE__), "crop_duster", file)
|
60
|
+
end
|
61
|
+
|
@@ -0,0 +1,12 @@
|
|
1
|
+
class AvailabilityZone < ActiveRecord::Base
|
2
|
+
|
3
|
+
has_many :servers, inverse_of: :availability_zone
|
4
|
+
has_many :volumes, inverse_of: :availability_zone
|
5
|
+
|
6
|
+
before_validation :set_region
|
7
|
+
|
8
|
+
def set_region
|
9
|
+
self.region = self.name.gsub(/[a-z]$/, '')
|
10
|
+
end
|
11
|
+
|
12
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
class Server < ActiveRecord::Base
|
2
|
+
|
3
|
+
belongs_to :availability_zone, inverse_of: :servers
|
4
|
+
belongs_to :flavor, inverse_of: :servers
|
5
|
+
|
6
|
+
has_many :volumes, inverse_of: :server
|
7
|
+
has_many :tags, as: :taggable
|
8
|
+
|
9
|
+
def self.import(aws_host)
|
10
|
+
server = Server.where(remote_id: aws_host.id).first || Server.new(remote_id: aws_host.id)
|
11
|
+
server.attributes = {
|
12
|
+
name: aws_host.tags["Name"],
|
13
|
+
server_type: aws_host.tags["type"],
|
14
|
+
account_id: aws_host.tags["account_id"],
|
15
|
+
state: aws_host.state,
|
16
|
+
remote_id: aws_host.id,
|
17
|
+
ebs_optimized: aws_host.ebs_optimized,
|
18
|
+
availability_zone: AvailabilityZone.find_or_create_by(name: aws_host.availability_zone),
|
19
|
+
flavor: Flavor.find_or_create_by(name: aws_host.flavor_id),
|
20
|
+
created_at: aws_host.created_at
|
21
|
+
}
|
22
|
+
|
23
|
+
print "."
|
24
|
+
server.save!
|
25
|
+
|
26
|
+
aws_host.tags.each do |key, value|
|
27
|
+
next if value.nil?
|
28
|
+
server.tags.find_or_create_by(name: key.downcase).update_attributes(value: value.downcase)
|
29
|
+
end
|
30
|
+
|
31
|
+
server
|
32
|
+
end
|
33
|
+
end
|
data/lib/models/tag.rb
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
class Volume < ActiveRecord::Base
|
2
|
+
|
3
|
+
belongs_to :server, inverse_of: :volumes
|
4
|
+
belongs_to :availability_zone, inverse_of: :volumes
|
5
|
+
|
6
|
+
has_many :tags, as: :taggable
|
7
|
+
|
8
|
+
before_validation :set_hourly_rate
|
9
|
+
|
10
|
+
def self.import(aws_volume)
|
11
|
+
volume = Volume.where(remote_id: aws_volume.id).first || Volume.new(remote_id: aws_volume.id)
|
12
|
+
volume.attributes = {
|
13
|
+
name: aws_volume.tags["Name"],
|
14
|
+
remote_id: aws_volume.id,
|
15
|
+
state: aws_volume.state,
|
16
|
+
volume_type: aws_volume.type,
|
17
|
+
iops: aws_volume.iops,
|
18
|
+
size: aws_volume.size,
|
19
|
+
server: Server.where(remote_id: aws_volume.server_id).first,
|
20
|
+
availability_zone: AvailabilityZone.find_or_create_by(name: aws_volume.availability_zone),
|
21
|
+
created_at: aws_volume.created_at
|
22
|
+
}
|
23
|
+
|
24
|
+
volume.save!
|
25
|
+
|
26
|
+
aws_volume.tags.each do |key, value|
|
27
|
+
volume.tags.find_or_create_by(name: key.downcase).update_attributes(value: value.downcase)
|
28
|
+
end
|
29
|
+
|
30
|
+
volume
|
31
|
+
end
|
32
|
+
|
33
|
+
def set_hourly_rate
|
34
|
+
self.hourly_rate = if volume_type == "standard"
|
35
|
+
(0.10 * self.size) / 720
|
36
|
+
elsif volume_type == "io1"
|
37
|
+
((0.125 * self.size) + (0.10 * self.iops)) / 720
|
38
|
+
else
|
39
|
+
raise "Unknown volume_type : #{self.volume_type}"
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
end
|
metadata
ADDED
@@ -0,0 +1,184 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: crop-duster
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.7
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Chris Winslett
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-02-04 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: rake
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 10.1.1
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 10.1.1
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: foreman
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0.63'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0.63'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: bundler
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '1.3'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '1.3'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: thor
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 0.18.1
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 0.18.1
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: fog
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: 1.19.0
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: 1.19.0
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: activesupport
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: 4.0.2
|
90
|
+
type: :runtime
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: 4.0.2
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: activerecord
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: 4.0.2
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: 4.0.2
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: pg
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - "~>"
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: 0.17.1
|
118
|
+
type: :runtime
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - "~>"
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: 0.17.1
|
125
|
+
description: Fly low and slow over your AWS servers looking for weeds
|
126
|
+
email:
|
127
|
+
- chris@mongohq.com
|
128
|
+
executables:
|
129
|
+
- crop-duster
|
130
|
+
extensions: []
|
131
|
+
extra_rdoc_files: []
|
132
|
+
files:
|
133
|
+
- Gemfile
|
134
|
+
- Gemfile.lock
|
135
|
+
- LICENSE.txt
|
136
|
+
- README.md
|
137
|
+
- Rakefile
|
138
|
+
- bin/crop-duster
|
139
|
+
- crop-duster.gemspec
|
140
|
+
- db/migrate/20140113115505_create_volumes.rb
|
141
|
+
- db/migrate/20140113115506_create_servers.rb
|
142
|
+
- db/migrate/20140113115507_create_tags.rb
|
143
|
+
- db/migrate/20140113120016_create_availability_zones.rb
|
144
|
+
- db/migrate/20140113120017_create_flavors.rb
|
145
|
+
- db/migrate/20140113192351_add_hourly_price_to_servers_and_volumes.rb
|
146
|
+
- db/migrate/20140113204901_create_aws_import_table.rb
|
147
|
+
- db/migrate/20140113214652_create_indexes.rb
|
148
|
+
- db/migrate/20140114141226_add_type_and_account_id_to_servers.rb
|
149
|
+
- db/migrate/20140114161225_add_index_to_usage_type.rb
|
150
|
+
- lib/crop_duster.rb
|
151
|
+
- lib/crop_duster/cli.rb
|
152
|
+
- lib/crop_duster/importer.rb
|
153
|
+
- lib/crop_duster/version.rb
|
154
|
+
- lib/models/availability_zone.rb
|
155
|
+
- lib/models/aws_billing_import.rb
|
156
|
+
- lib/models/flavor.rb
|
157
|
+
- lib/models/server.rb
|
158
|
+
- lib/models/tag.rb
|
159
|
+
- lib/models/volume.rb
|
160
|
+
homepage: http://www.mongohq.com
|
161
|
+
licenses:
|
162
|
+
- MIT
|
163
|
+
metadata: {}
|
164
|
+
post_install_message:
|
165
|
+
rdoc_options: []
|
166
|
+
require_paths:
|
167
|
+
- lib
|
168
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
169
|
+
requirements:
|
170
|
+
- - ">="
|
171
|
+
- !ruby/object:Gem::Version
|
172
|
+
version: '0'
|
173
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
174
|
+
requirements:
|
175
|
+
- - ">="
|
176
|
+
- !ruby/object:Gem::Version
|
177
|
+
version: '0'
|
178
|
+
requirements: []
|
179
|
+
rubyforge_project:
|
180
|
+
rubygems_version: 2.2.0
|
181
|
+
signing_key:
|
182
|
+
specification_version: 4
|
183
|
+
summary: A system for AWS server change management.
|
184
|
+
test_files: []
|