aws-rds-instance_types 0.2.0 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/aws-rds-instance_types.gemspec +9 -4
- data/bin/aws-rds-instance_types +9 -15
- data/lib/dldinternet/aws/rds/instance_types.rb +7 -107
- data/lib/dldinternet/aws/rds/instance_types/cli.rb +87 -0
- data/lib/dldinternet/aws/rds/instance_types/error.rb +9 -0
- data/lib/dldinternet/aws/rds/instance_types/mixins/no_commands.rb +119 -0
- data/lib/dldinternet/aws/rds/instance_types/mixins/rds_instance_types.rb +91 -0
- data/lib/dldinternet/aws/rds/instance_types/scraper.rb +127 -0
- data/lib/dldinternet/aws/rds/instance_types/version.rb +10 -0
- metadata +83 -8
- data/lib/aws/rds/instance_types/version.rb +0 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 10825fbd401fd97265d9c6cc789190576052127b
|
4
|
+
data.tar.gz: 1b962f5264e9c1dba4ad9277a8d03eccbf0cc182
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ee70bc544cb03aadc0861f044081ce2169ff0007a9d05d8ee39ee521df899490c92a85dd45f81ccd12220a59991f932766014c181f2a20f8a2d465fcc8f502f0
|
7
|
+
data.tar.gz: afb92c16617868792c599b4100ae3580a0d9e80d539565a5a98d13c158a42a3719839c0852356533f664523ac56e10da2b5b69161bb60f32cde11fe2122155cc
|
@@ -1,10 +1,10 @@
|
|
1
1
|
# -*- encoding: utf-8 -*-
|
2
2
|
|
3
|
-
require File.expand_path('../lib/aws/rds/instance_types/version', __FILE__)
|
3
|
+
require File.expand_path('../lib/dldinternet/aws/rds/instance_types/version', __FILE__)
|
4
4
|
|
5
5
|
Gem::Specification.new do |gem|
|
6
6
|
gem.name = "aws-rds-instance_types"
|
7
|
-
gem.version =
|
7
|
+
gem.version = DLDInternet::AWS::RDS::Instance_Types::VERSION
|
8
8
|
gem.description =
|
9
9
|
gem.summary = %q{Retrieve an up to date list of valid AWS RDS Instance Types directly from Amazon Web Services official site.}
|
10
10
|
gem.license = 'Apachev2'
|
@@ -21,8 +21,13 @@ Gem::Specification.new do |gem|
|
|
21
21
|
gem.add_development_dependency 'rake', '~> 0.8'
|
22
22
|
gem.add_development_dependency 'rubygems-tasks', '~> 0.2'
|
23
23
|
|
24
|
+
gem.add_runtime_dependency('thor', ['>= 0.19.1'])
|
24
25
|
gem.add_runtime_dependency(%q<mechanize>, ['>= 2.7.2'])
|
25
26
|
gem.add_runtime_dependency(%q<nokogiri>, ['>= 1.5.0'])
|
26
|
-
gem.
|
27
|
-
gem.
|
27
|
+
gem.add_dependency 'awesome_print', '> 0'
|
28
|
+
gem.add_dependency 'inifile', '> 0'
|
29
|
+
gem.add_dependency 'colorize', '>= 0'
|
30
|
+
gem.add_dependency 'dldinternet-mixlib-logging', '>= 0.4.3'
|
31
|
+
gem.add_dependency 'psych'
|
32
|
+
gem.add_dependency 'json'
|
28
33
|
end
|
data/bin/aws-rds-instance_types
CHANGED
@@ -3,20 +3,14 @@
|
|
3
3
|
# Adjust lib path
|
4
4
|
_lib=File.expand_path(File.dirname(__FILE__) + '/../lib')
|
5
5
|
$:.unshift(_lib) unless $:.include?(_lib)
|
6
|
+
require 'dldinternet/aws/rds/instance_types/cli'
|
6
7
|
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
require 'awesome_print'
|
16
|
-
require 'colorize'
|
17
|
-
|
18
|
-
begin
|
19
|
-
puts getInstanceTypes(:mechanize => mechanize).ai
|
20
|
-
rescue Timeout::Error => e
|
21
|
-
puts "Unable to retrieve instance type details in a reasonable time (#{mechanize.open_timeout}s). Giving up ...".light_red
|
8
|
+
# =====================================================================================================================
|
9
|
+
rc = DLDInternet::AWS::RDS::Instance_Types::Cli.start(ARGV)
|
10
|
+
if rc.is_a?(Fixnum)
|
11
|
+
exit rc
|
12
|
+
else
|
13
|
+
puts rc.ai
|
14
|
+
exit 0
|
22
15
|
end
|
16
|
+
|
@@ -1,119 +1,19 @@
|
|
1
1
|
module DLDInternet
|
2
2
|
module AWS
|
3
3
|
module RDS
|
4
|
-
|
5
|
-
HEADINGS = [
|
6
|
-
:Instance_Type,
|
7
|
-
:vCPU,
|
8
|
-
:Memory,
|
9
|
-
:PIOPS_Optimized,
|
10
|
-
:Network_Performance,
|
11
|
-
:Generation,
|
12
|
-
]
|
13
|
-
DEBUG = true
|
14
|
-
attr_reader :instance_types
|
4
|
+
module Instance_Types
|
15
5
|
|
16
|
-
|
17
|
-
def getInstanceTypes(options={})
|
18
|
-
unless @instance_types
|
19
|
-
@instance_types = {}
|
20
|
-
require 'mechanize'
|
21
|
-
mechanize = options[:mechanize]
|
22
|
-
unless mechanize
|
23
|
-
mechanize = Mechanize.new
|
24
|
-
mechanize.user_agent_alias = 'Mac Safari' # Pretend to use a Mac
|
25
|
-
end
|
26
|
-
url = options[:url] || 'http://aws.amazon.com/rds/details/'
|
6
|
+
class << self
|
27
7
|
|
28
|
-
|
8
|
+
require 'dldinternet/aws/rds/instance_types/mixins/RDS_instance_types'
|
9
|
+
include DLDInternet::AWS::RDS::Instance_Types::MixIns::RDS_Instance_Types
|
29
10
|
|
30
|
-
require 'nokogiri'
|
31
|
-
|
32
|
-
nk = Nokogiri::HTML(page.body)
|
33
|
-
div = nk.css('div.page-content')
|
34
|
-
# noinspection RubyAssignmentExpressionInConditionalInspection
|
35
|
-
if div = find_div(div, %r'^<div\s+class="nine columns content-with-nav')
|
36
|
-
# noinspection RubyAssignmentExpressionInConditionalInspection
|
37
|
-
if div = find_div(div, %r'^<div\s+class="content parsys')
|
38
|
-
divs = div.css('div').to_a
|
39
|
-
itm = nil
|
40
|
-
idx = 0
|
41
|
-
divs.each do |d|
|
42
|
-
as = d.css('div div h2 a')
|
43
|
-
as.each do |a|
|
44
|
-
# puts "'#{a.text}'"
|
45
|
-
if a.text.match %r'\s*DB Instance Classes\s*'
|
46
|
-
itm = d
|
47
|
-
break
|
48
|
-
end
|
49
|
-
end
|
50
|
-
break if itm
|
51
|
-
idx += 1
|
52
|
-
end
|
53
|
-
if idx < divs.count
|
54
|
-
divs = divs[idx..-1]
|
55
|
-
table = nil
|
56
|
-
divs.each do |d|
|
57
|
-
table = d.css('div.aws-table table')
|
58
|
-
break if table
|
59
|
-
end
|
60
|
-
|
61
|
-
@instance_types = scrapeTable(HEADINGS, table) if table
|
62
|
-
end
|
63
|
-
end
|
64
|
-
end
|
65
11
|
end
|
66
|
-
@instance_types
|
67
|
-
end
|
68
12
|
|
69
|
-
def find_div(nk,regex)
|
70
|
-
ret = nil
|
71
|
-
divs = nk.search('div')
|
72
|
-
if divs.count > 0
|
73
|
-
nine = divs.select { |div| div.to_s.match regex }
|
74
|
-
if nine.count >= 1
|
75
|
-
nine = nine.shift
|
76
|
-
ret = nine
|
77
|
-
end
|
78
|
-
end
|
79
|
-
ret
|
80
13
|
end
|
81
14
|
|
82
|
-
# ---------------------------------------------------------------------------------------------------------------
|
83
|
-
def scrapeTable(cHeadings,table)
|
84
|
-
raise Error.new 'Cannot find instance type table' unless (table.is_a?(Nokogiri::XML::Element) or table.is_a?(Nokogiri::XML::NodeSet))
|
85
|
-
rows = table.search('tr')[0..-1]
|
86
|
-
head = rows.shift
|
87
|
-
|
88
|
-
cols = head.search('td').collect { |td|
|
89
|
-
text = td.text.to_s
|
90
|
-
text = text.gsub(%r/(\r?\n)+/, ' ').strip
|
91
|
-
CGI.unescapeHTML(text)
|
92
|
-
}
|
93
|
-
instance_types = {
|
94
|
-
:headings => {},
|
95
|
-
:details => []
|
96
|
-
}
|
97
|
-
(0..cols.size-1).map { |i| instance_types[:headings][cHeadings[i]] = cols[i] }
|
98
|
-
instance_types[:headings][cHeadings[-1]] = 'Generation'
|
99
|
-
instance_set = nil
|
100
|
-
rows.each do |row|
|
101
|
-
|
102
|
-
cells = row.search('td').collect { |td|
|
103
|
-
CGI.unescapeHTML(td.text.to_s.gsub(%r/(\r?\n)+/, ' ').strip)
|
104
|
-
}
|
105
|
-
if cells.count == 1 or (cells[1] == cells[2] and cells[2] == cells[4])
|
106
|
-
instance_set = cells[0]
|
107
|
-
else
|
108
|
-
raise StandardError.new "This row does not have the same number of cells as the table header: #{row.text.to_s.strip}" unless cells.size == cols.size
|
109
|
-
instance = {}
|
110
|
-
(0..cells.size-1).map { |i| instance[cHeadings[i]] = cells[i] }
|
111
|
-
instance[cHeadings[-1]] = instance_set
|
112
|
-
instance_types[:details] << instance
|
113
|
-
end
|
114
|
-
end
|
115
|
-
instance_types
|
116
|
-
end
|
117
15
|
end
|
16
|
+
|
118
17
|
end
|
119
|
-
|
18
|
+
|
19
|
+
end
|
@@ -0,0 +1,87 @@
|
|
1
|
+
require 'thor'
|
2
|
+
require 'awesome_print'
|
3
|
+
require 'colorize'
|
4
|
+
require 'dldinternet/aws/rds/instance_types/version'
|
5
|
+
require 'dldinternet/aws/rds/instance_types/error'
|
6
|
+
require 'dldinternet/aws/rds/instance_types/scraper'
|
7
|
+
require 'dldinternet/aws/rds/instance_types/cli'
|
8
|
+
require 'dldinternet/aws/rds/instance_types'
|
9
|
+
|
10
|
+
module DLDInternet
|
11
|
+
module AWS
|
12
|
+
module RDS
|
13
|
+
module Instance_Types
|
14
|
+
class Cli < Thor
|
15
|
+
class_option :verbose, :type => :boolean
|
16
|
+
class_option :debug, :type => :boolean
|
17
|
+
class_option :log_level, :type => :string, :banner => 'Log level ([:trace, :debug, :info, :step, :warn, :error, :fatal, :todo])'
|
18
|
+
class_option :inifile, :type => :string
|
19
|
+
class_option :help, :type => :boolean
|
20
|
+
class_option :format, :type => :string, :default => 'pretty', :banner => '[:pretty, :yaml, :json]'
|
21
|
+
|
22
|
+
no_commands do
|
23
|
+
|
24
|
+
require 'dldinternet/aws/rds/instance_types/mixins/no_commands'
|
25
|
+
include DLDInternet::AWS::RDS::Instance_Types::MixIns::NoCommands
|
26
|
+
|
27
|
+
end
|
28
|
+
|
29
|
+
def help()
|
30
|
+
puts "Version: #{VERSION}"
|
31
|
+
super
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.start(argv = ARGV, config = {})
|
35
|
+
if argv.size == 0 or argv[0].match(%r'^--')
|
36
|
+
argv.unshift('get')
|
37
|
+
end
|
38
|
+
super(argv,config)
|
39
|
+
end
|
40
|
+
|
41
|
+
def initialize(args = [], local_options = {}, config = {})
|
42
|
+
super(args,local_options,config)
|
43
|
+
@log_level = :step
|
44
|
+
end
|
45
|
+
|
46
|
+
desc 'get', 'get instance types'
|
47
|
+
def get()
|
48
|
+
parse_options
|
49
|
+
puts 'get instance types' if options[:verbose]
|
50
|
+
|
51
|
+
it = DLDInternet::AWS::RDS::Instance_Types.getRDS_Instance_Types()
|
52
|
+
case options[:format]
|
53
|
+
when /yaml/
|
54
|
+
puts it.to_yaml line_width: 1024, indentation: 4, canonical: false
|
55
|
+
when /json/
|
56
|
+
puts JSON.pretty_generate( it, { indent: "\t", space: ' '})
|
57
|
+
else
|
58
|
+
puts it.ai
|
59
|
+
end
|
60
|
+
0
|
61
|
+
end
|
62
|
+
|
63
|
+
desc 'load ARGS', 'load instance types'
|
64
|
+
def load(path)
|
65
|
+
parse_options
|
66
|
+
puts 'load instance types' if options[:verbose]
|
67
|
+
|
68
|
+
it = DLDInternet::AWS::RDS::Instance_Types.loadRDS_Instance_Types(path)
|
69
|
+
ap it
|
70
|
+
end
|
71
|
+
|
72
|
+
desc 'save', 'save instance types'
|
73
|
+
def save(path)
|
74
|
+
parse_options
|
75
|
+
puts 'save instance types' if options[:verbose]
|
76
|
+
|
77
|
+
it = DLDInternet::AWS::Rds::Instance_Types.getRDS_Instance_Types()
|
78
|
+
DLDInternet::AWS::Rds::Instance_Types.saveRDS_Instance_Types(path, it)
|
79
|
+
end
|
80
|
+
|
81
|
+
default_task 'get'
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
@@ -0,0 +1,119 @@
|
|
1
|
+
require 'thor'
|
2
|
+
require 'awesome_print'
|
3
|
+
require 'inifile'
|
4
|
+
require 'colorize'
|
5
|
+
|
6
|
+
module DLDInternet
|
7
|
+
module AWS
|
8
|
+
module RDS
|
9
|
+
module Instance_Types
|
10
|
+
|
11
|
+
LOG_LEVELS = [:trace, :debug, :info, :step, :warn, :error, :fatal, :todo]
|
12
|
+
|
13
|
+
module MixIns
|
14
|
+
module NoCommands
|
15
|
+
require 'dldinternet/mixlib/logging'
|
16
|
+
include DLDInternet::Mixlib::Logging
|
17
|
+
|
18
|
+
def validate_options
|
19
|
+
_options = options.dup
|
20
|
+
if options[:log_level]
|
21
|
+
log_level = options[:log_level].to_sym
|
22
|
+
raise "Invalid log-level: #{log_level}" unless LOG_LEVELS.include?(log_level)
|
23
|
+
_options[:log_level] = log_level
|
24
|
+
else
|
25
|
+
_options[:log_level] ||= :step
|
26
|
+
end
|
27
|
+
@options = _options
|
28
|
+
end
|
29
|
+
|
30
|
+
def parse_options
|
31
|
+
if options[:help]
|
32
|
+
Cli.start(['help'])
|
33
|
+
exit 0
|
34
|
+
end
|
35
|
+
validate_options
|
36
|
+
|
37
|
+
lcs = ::Logging::ColorScheme.new( 'compiler', :levels => {
|
38
|
+
:trace => :blue,
|
39
|
+
:debug => :cyan,
|
40
|
+
:info => :green,
|
41
|
+
:step => :green,
|
42
|
+
:warn => :yellow,
|
43
|
+
:error => :red,
|
44
|
+
:fatal => :red,
|
45
|
+
:todo => :purple,
|
46
|
+
})
|
47
|
+
scheme = lcs.scheme
|
48
|
+
scheme['trace'] = "\e[38;5;33m"
|
49
|
+
scheme['fatal'] = "\e[38;5;89m"
|
50
|
+
scheme['todo'] = "\e[38;5;55m"
|
51
|
+
lcs.scheme scheme
|
52
|
+
@config = @options.dup
|
53
|
+
@config[:log_opts] = lambda{|mlll| {
|
54
|
+
:pattern => "%#{mlll}l: %m %g\n",
|
55
|
+
:date_pattern => '%Y-%m-%d %H:%M:%S',
|
56
|
+
:color_scheme => 'compiler',
|
57
|
+
:trace => (@config[:trace].nil? ? false : @config[:trace]),
|
58
|
+
# [2014-06-30 Christo] DO NOT do this ... it needs to be a FixNum!!!!
|
59
|
+
# If you want to do ::Logging.init first then fine ... go ahead :)
|
60
|
+
# :level => @config[:log_level],
|
61
|
+
}
|
62
|
+
}
|
63
|
+
@logger = getLogger(@config)
|
64
|
+
|
65
|
+
if @options[:inifile]
|
66
|
+
@options[:inifile] = File.expand_path(@options[:inifile])
|
67
|
+
unless File.exist?(@options[:inifile])
|
68
|
+
raise "#{@options[:inifile]} not found!"
|
69
|
+
end
|
70
|
+
begin
|
71
|
+
ini = ::IniFile.load(@options[:inifile])
|
72
|
+
ini['global'].each{ |key,value|
|
73
|
+
@options[key.to_s]=value
|
74
|
+
}
|
75
|
+
def _expand(k,v,regex,rerun)
|
76
|
+
matches = v.match(regex)
|
77
|
+
if matches
|
78
|
+
var = matches[1]
|
79
|
+
if options[var]
|
80
|
+
options[k]=v.gsub(/\%\(#{var}\)/,options[var]).gsub(/\%#{var}/,options[var])
|
81
|
+
else
|
82
|
+
rerun[var] = 1
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
pending = nil
|
88
|
+
rerun = {}
|
89
|
+
begin
|
90
|
+
pending = rerun
|
91
|
+
rerun = {}
|
92
|
+
options.to_hash.each{|k,v|
|
93
|
+
if v.to_s.match(/\%/)
|
94
|
+
_expand(k,v,%r'[^\\]\%\((\w+)\)', rerun)
|
95
|
+
_expand(k,v,%r'[^\\]\%(\w+)', rerun)
|
96
|
+
end
|
97
|
+
}
|
98
|
+
# Should break out the first time that we make no progress!
|
99
|
+
end while pending != rerun
|
100
|
+
rescue ::IniFile::Error => e
|
101
|
+
# noop
|
102
|
+
rescue ::Exception => e
|
103
|
+
@logger.error "#{e.class.name} #{e.message}"
|
104
|
+
raise e
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
if options[:verbose]
|
109
|
+
@logger.info "Options:\n#{options.ai}"
|
110
|
+
end
|
111
|
+
|
112
|
+
end
|
113
|
+
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
@@ -0,0 +1,91 @@
|
|
1
|
+
require 'thor'
|
2
|
+
require 'awesome_print'
|
3
|
+
require 'inifile'
|
4
|
+
require 'colorize'
|
5
|
+
require 'dldinternet/aws/rds/instance_types/scraper'
|
6
|
+
|
7
|
+
module DLDInternet
|
8
|
+
module AWS
|
9
|
+
module RDS
|
10
|
+
module Instance_Types
|
11
|
+
module MixIns
|
12
|
+
module RDS_Instance_Types
|
13
|
+
|
14
|
+
def getFileFormat(path)
|
15
|
+
format = case File.extname(File.basename(path)).downcase
|
16
|
+
when /json|js/
|
17
|
+
'json'
|
18
|
+
when /yaml|yml/
|
19
|
+
'yaml'
|
20
|
+
else
|
21
|
+
raise DLDInternet::AWS::RDS::Instance_Types::Error.new("Unsupported file type: #{path}")
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def saveRDS_Instance_Types(path,it)
|
26
|
+
format = getFileFormat(path)
|
27
|
+
begin
|
28
|
+
File.open path, File::CREAT|File::TRUNC|File::RDWR, 0644 do |f|
|
29
|
+
case format
|
30
|
+
when /yaml/
|
31
|
+
f.write it.to_yaml line_width: 1024, indentation: 4, canonical: false
|
32
|
+
when /json/
|
33
|
+
f.write JSON.pretty_generate(it, { indent: "\t", space: ' '})
|
34
|
+
else
|
35
|
+
f.write YAML::dump(it)
|
36
|
+
# abort! "Unsupported save format #{format}!"
|
37
|
+
end
|
38
|
+
f.close
|
39
|
+
end
|
40
|
+
rescue
|
41
|
+
abort! "!!! Could not write file #{path}: \nException: #{$!}\nParent directory exists? #{File.directory?(File.dirname(path))}\n"
|
42
|
+
end
|
43
|
+
0
|
44
|
+
end
|
45
|
+
|
46
|
+
def loadRDS_Instance_Types(path)
|
47
|
+
format = getFileFormat(path)
|
48
|
+
spec = File.read(path)
|
49
|
+
case format
|
50
|
+
when /json/
|
51
|
+
JSON.parse(spec)
|
52
|
+
#when /yaml/
|
53
|
+
else
|
54
|
+
begin
|
55
|
+
YAML.load(spec)
|
56
|
+
rescue Psych::SyntaxError => e
|
57
|
+
abort! "Error in the template specification: #{e.message}\n#{spec.split(/\n/).map{|l| "#{i+=1}: #{l}"}.join("\n")}"
|
58
|
+
end
|
59
|
+
# else
|
60
|
+
# abort! "Unsupported file type: #{path}"
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def getRDS_Instance_Types(mechanize=nil)
|
65
|
+
unless mechanize
|
66
|
+
require 'mechanize'
|
67
|
+
mechanize = ::Mechanize.new
|
68
|
+
mechanize.open_timeout = 5
|
69
|
+
mechanize.read_timeout = 10
|
70
|
+
end
|
71
|
+
|
72
|
+
scraper = DLDInternet::AWS::RDS::Instance_Types::Scraper.new()
|
73
|
+
|
74
|
+
begin
|
75
|
+
return scraper.getInstanceTypes(:mechanize => mechanize)
|
76
|
+
rescue Timeout::Error => e
|
77
|
+
puts "Unable to retrieve instance type details in a reasonable time (#{mechanize.open_timeout}s). Giving up ...".light_red
|
78
|
+
return nil
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
def abort!(msg)
|
83
|
+
raise DLDInternet::AWS::RDS::Instance_Types::Error.new msg
|
84
|
+
end
|
85
|
+
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
@@ -0,0 +1,127 @@
|
|
1
|
+
module DLDInternet
|
2
|
+
module AWS
|
3
|
+
module RDS
|
4
|
+
module Instance_Types
|
5
|
+
HEADINGS = [
|
6
|
+
:Instance_Type,
|
7
|
+
:vCPU,
|
8
|
+
:Memory,
|
9
|
+
:PIOPS_Optimized,
|
10
|
+
:Network_Performance,
|
11
|
+
:Generation,
|
12
|
+
]
|
13
|
+
DEBUG = true
|
14
|
+
class Scraper
|
15
|
+
attr_reader :instance_types
|
16
|
+
|
17
|
+
# ---------------------------------------------------------------------------------------------------------------
|
18
|
+
def initialize
|
19
|
+
@instance_types = {}
|
20
|
+
end
|
21
|
+
|
22
|
+
# ---------------------------------------------------------------------------------------------------------------
|
23
|
+
def getInstanceTypes(options={})
|
24
|
+
unless @instance_types.size > 0
|
25
|
+
require 'mechanize'
|
26
|
+
mechanize = options[:mechanize]
|
27
|
+
unless mechanize
|
28
|
+
mechanize = Mechanize.new
|
29
|
+
mechanize.user_agent_alias = 'Mac Safari' # Pretend to use a Mac
|
30
|
+
end
|
31
|
+
url = options[:url] || 'http://aws.amazon.com/rds/details/'
|
32
|
+
|
33
|
+
page = mechanize.get(url)
|
34
|
+
|
35
|
+
require 'nokogiri'
|
36
|
+
|
37
|
+
nk = Nokogiri::HTML(page.body)
|
38
|
+
div = nk.css('div.page-content')
|
39
|
+
# noinspection RubyAssignmentExpressionInConditionalInspection
|
40
|
+
if div = find_div(div, %r'^<div\s+class="nine columns content-with-nav')
|
41
|
+
# noinspection RubyAssignmentExpressionInConditionalInspection
|
42
|
+
if div = find_div(div, %r'^<div\s+class="content parsys')
|
43
|
+
divs = div.css('div').to_a
|
44
|
+
itm = nil
|
45
|
+
idx = 0
|
46
|
+
divs.each do |d|
|
47
|
+
as = d.css('div div h2 a')
|
48
|
+
as.each do |a|
|
49
|
+
# puts "'#{a.text}'"
|
50
|
+
if a.text.match %r'\s*DB Instance Classes\s*'
|
51
|
+
itm = d
|
52
|
+
break
|
53
|
+
end
|
54
|
+
end
|
55
|
+
break if itm
|
56
|
+
idx += 1
|
57
|
+
end
|
58
|
+
if idx < divs.count
|
59
|
+
divs = divs[idx..-1]
|
60
|
+
table = nil
|
61
|
+
divs.each do |d|
|
62
|
+
table = d.css('div.aws-table table')
|
63
|
+
break if table
|
64
|
+
end
|
65
|
+
|
66
|
+
@instance_types = scrapeTable(HEADINGS, table) if table
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
@instance_types
|
72
|
+
end
|
73
|
+
|
74
|
+
def find_div(nk,regex)
|
75
|
+
ret = nil
|
76
|
+
divs = nk.search('div')
|
77
|
+
if divs.count > 0
|
78
|
+
nine = divs.select { |div| div.to_s.match regex }
|
79
|
+
if nine.count >= 1
|
80
|
+
nine = nine.shift
|
81
|
+
ret = nine
|
82
|
+
end
|
83
|
+
end
|
84
|
+
ret
|
85
|
+
end
|
86
|
+
|
87
|
+
# ---------------------------------------------------------------------------------------------------------------
|
88
|
+
def scrapeTable(cHeadings,table)
|
89
|
+
raise Error.new 'Cannot find instance type table' unless (table.is_a?(Nokogiri::XML::Element) or table.is_a?(Nokogiri::XML::NodeSet))
|
90
|
+
rows = table.search('tr')[0..-1]
|
91
|
+
head = rows.shift
|
92
|
+
|
93
|
+
cols = head.search('td').collect { |td|
|
94
|
+
text = td.text.to_s
|
95
|
+
text = text.gsub(%r/(\r?\n)+/, ' ').strip
|
96
|
+
CGI.unescapeHTML(text)
|
97
|
+
}
|
98
|
+
instance_types = {
|
99
|
+
:headings => {},
|
100
|
+
:details => []
|
101
|
+
}
|
102
|
+
(0..cols.size-1).map { |i| instance_types[:headings][cHeadings[i]] = cols[i] }
|
103
|
+
instance_types[:headings][cHeadings[-1]] = 'Generation'
|
104
|
+
instance_set = nil
|
105
|
+
rows.each do |row|
|
106
|
+
|
107
|
+
cells = row.search('td').collect { |td|
|
108
|
+
CGI.unescapeHTML(td.text.to_s.gsub(%r/(\r?\n)+/, ' ').strip)
|
109
|
+
}
|
110
|
+
if cells.count == 1 or (cells[1] == cells[2] and cells[2] == cells[4])
|
111
|
+
instance_set = cells[0]
|
112
|
+
else
|
113
|
+
raise StandardError.new "This row does not have the same number of cells as the table header: #{row.text.to_s.strip}" unless cells.size == cols.size
|
114
|
+
instance = {}
|
115
|
+
(0..cells.size-1).map { |i| instance[cHeadings[i]] = cells[i] }
|
116
|
+
instance[cHeadings[-1]] = instance_set
|
117
|
+
instance_types[:details] << instance
|
118
|
+
end
|
119
|
+
end
|
120
|
+
instance_types
|
121
|
+
end
|
122
|
+
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: aws-rds-instance_types
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Christo De Lange
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-11-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -52,6 +52,20 @@ dependencies:
|
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0.2'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: thor
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 0.19.1
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 0.19.1
|
55
69
|
- !ruby/object:Gem::Dependency
|
56
70
|
name: mechanize
|
57
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -82,32 +96,88 @@ dependencies:
|
|
82
96
|
version: 1.5.0
|
83
97
|
- !ruby/object:Gem::Dependency
|
84
98
|
name: awesome_print
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: inifile
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">"
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :runtime
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">"
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: colorize
|
85
127
|
requirement: !ruby/object:Gem::Requirement
|
86
128
|
requirements:
|
87
129
|
- - ">="
|
88
130
|
- !ruby/object:Gem::Version
|
89
|
-
version:
|
131
|
+
version: '0'
|
90
132
|
type: :runtime
|
91
133
|
prerelease: false
|
92
134
|
version_requirements: !ruby/object:Gem::Requirement
|
93
135
|
requirements:
|
94
136
|
- - ">="
|
95
137
|
- !ruby/object:Gem::Version
|
96
|
-
version:
|
138
|
+
version: '0'
|
97
139
|
- !ruby/object:Gem::Dependency
|
98
|
-
name:
|
140
|
+
name: dldinternet-mixlib-logging
|
141
|
+
requirement: !ruby/object:Gem::Requirement
|
142
|
+
requirements:
|
143
|
+
- - ">="
|
144
|
+
- !ruby/object:Gem::Version
|
145
|
+
version: 0.4.3
|
146
|
+
type: :runtime
|
147
|
+
prerelease: false
|
148
|
+
version_requirements: !ruby/object:Gem::Requirement
|
149
|
+
requirements:
|
150
|
+
- - ">="
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: 0.4.3
|
153
|
+
- !ruby/object:Gem::Dependency
|
154
|
+
name: psych
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
156
|
+
requirements:
|
157
|
+
- - ">="
|
158
|
+
- !ruby/object:Gem::Version
|
159
|
+
version: '0'
|
160
|
+
type: :runtime
|
161
|
+
prerelease: false
|
162
|
+
version_requirements: !ruby/object:Gem::Requirement
|
163
|
+
requirements:
|
164
|
+
- - ">="
|
165
|
+
- !ruby/object:Gem::Version
|
166
|
+
version: '0'
|
167
|
+
- !ruby/object:Gem::Dependency
|
168
|
+
name: json
|
99
169
|
requirement: !ruby/object:Gem::Requirement
|
100
170
|
requirements:
|
101
171
|
- - ">="
|
102
172
|
- !ruby/object:Gem::Version
|
103
|
-
version: 0
|
173
|
+
version: '0'
|
104
174
|
type: :runtime
|
105
175
|
prerelease: false
|
106
176
|
version_requirements: !ruby/object:Gem::Requirement
|
107
177
|
requirements:
|
108
178
|
- - ">="
|
109
179
|
- !ruby/object:Gem::Version
|
110
|
-
version: 0
|
180
|
+
version: '0'
|
111
181
|
description: Retrieve an up to date list of valid AWS RDS Instance Types directly
|
112
182
|
from Amazon Web Services official site.
|
113
183
|
email: rubygems@dldinternet.com
|
@@ -131,9 +201,14 @@ files:
|
|
131
201
|
- features/step_definitions/.gitkeep
|
132
202
|
- features/step_definitions/aws-rds-instance_types_steps.rb
|
133
203
|
- lib/aws/rds/instance_types.rb
|
134
|
-
- lib/aws/rds/instance_types/version.rb
|
135
204
|
- lib/dldinternet/aws/rds.rb
|
136
205
|
- lib/dldinternet/aws/rds/instance_types.rb
|
206
|
+
- lib/dldinternet/aws/rds/instance_types/cli.rb
|
207
|
+
- lib/dldinternet/aws/rds/instance_types/error.rb
|
208
|
+
- lib/dldinternet/aws/rds/instance_types/mixins/no_commands.rb
|
209
|
+
- lib/dldinternet/aws/rds/instance_types/mixins/rds_instance_types.rb
|
210
|
+
- lib/dldinternet/aws/rds/instance_types/scraper.rb
|
211
|
+
- lib/dldinternet/aws/rds/instance_types/version.rb
|
137
212
|
homepage: https://rubygems.org/gems/aws-rds-instance_types
|
138
213
|
licenses:
|
139
214
|
- Apachev2
|