aws-rds-instance_types 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/aws-rds-instance_types.gemspec +9 -4
- data/bin/aws-rds-instance_types +9 -15
- data/lib/dldinternet/aws/rds/instance_types.rb +7 -107
- data/lib/dldinternet/aws/rds/instance_types/cli.rb +87 -0
- data/lib/dldinternet/aws/rds/instance_types/error.rb +9 -0
- data/lib/dldinternet/aws/rds/instance_types/mixins/no_commands.rb +119 -0
- data/lib/dldinternet/aws/rds/instance_types/mixins/rds_instance_types.rb +91 -0
- data/lib/dldinternet/aws/rds/instance_types/scraper.rb +127 -0
- data/lib/dldinternet/aws/rds/instance_types/version.rb +10 -0
- metadata +83 -8
- data/lib/aws/rds/instance_types/version.rb +0 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 10825fbd401fd97265d9c6cc789190576052127b
|
4
|
+
data.tar.gz: 1b962f5264e9c1dba4ad9277a8d03eccbf0cc182
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ee70bc544cb03aadc0861f044081ce2169ff0007a9d05d8ee39ee521df899490c92a85dd45f81ccd12220a59991f932766014c181f2a20f8a2d465fcc8f502f0
|
7
|
+
data.tar.gz: afb92c16617868792c599b4100ae3580a0d9e80d539565a5a98d13c158a42a3719839c0852356533f664523ac56e10da2b5b69161bb60f32cde11fe2122155cc
|
@@ -1,10 +1,10 @@
|
|
1
1
|
# -*- encoding: utf-8 -*-
|
2
2
|
|
3
|
-
require File.expand_path('../lib/aws/rds/instance_types/version', __FILE__)
|
3
|
+
require File.expand_path('../lib/dldinternet/aws/rds/instance_types/version', __FILE__)
|
4
4
|
|
5
5
|
Gem::Specification.new do |gem|
|
6
6
|
gem.name = "aws-rds-instance_types"
|
7
|
-
gem.version =
|
7
|
+
gem.version = DLDInternet::AWS::RDS::Instance_Types::VERSION
|
8
8
|
gem.description =
|
9
9
|
gem.summary = %q{Retrieve an up to date list of valid AWS RDS Instance Types directly from Amazon Web Services official site.}
|
10
10
|
gem.license = 'Apachev2'
|
@@ -21,8 +21,13 @@ Gem::Specification.new do |gem|
|
|
21
21
|
gem.add_development_dependency 'rake', '~> 0.8'
|
22
22
|
gem.add_development_dependency 'rubygems-tasks', '~> 0.2'
|
23
23
|
|
24
|
+
gem.add_runtime_dependency('thor', ['>= 0.19.1'])
|
24
25
|
gem.add_runtime_dependency(%q<mechanize>, ['>= 2.7.2'])
|
25
26
|
gem.add_runtime_dependency(%q<nokogiri>, ['>= 1.5.0'])
|
26
|
-
gem.
|
27
|
-
gem.
|
27
|
+
gem.add_dependency 'awesome_print', '> 0'
|
28
|
+
gem.add_dependency 'inifile', '> 0'
|
29
|
+
gem.add_dependency 'colorize', '>= 0'
|
30
|
+
gem.add_dependency 'dldinternet-mixlib-logging', '>= 0.4.3'
|
31
|
+
gem.add_dependency 'psych'
|
32
|
+
gem.add_dependency 'json'
|
28
33
|
end
|
data/bin/aws-rds-instance_types
CHANGED
@@ -3,20 +3,14 @@
|
|
3
3
|
# Adjust lib path
|
4
4
|
_lib=File.expand_path(File.dirname(__FILE__) + '/../lib')
|
5
5
|
$:.unshift(_lib) unless $:.include?(_lib)
|
6
|
+
require 'dldinternet/aws/rds/instance_types/cli'
|
6
7
|
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
require 'awesome_print'
|
16
|
-
require 'colorize'
|
17
|
-
|
18
|
-
begin
|
19
|
-
puts getInstanceTypes(:mechanize => mechanize).ai
|
20
|
-
rescue Timeout::Error => e
|
21
|
-
puts "Unable to retrieve instance type details in a reasonable time (#{mechanize.open_timeout}s). Giving up ...".light_red
|
8
|
+
# =====================================================================================================================
|
9
|
+
rc = DLDInternet::AWS::RDS::Instance_Types::Cli.start(ARGV)
|
10
|
+
if rc.is_a?(Fixnum)
|
11
|
+
exit rc
|
12
|
+
else
|
13
|
+
puts rc.ai
|
14
|
+
exit 0
|
22
15
|
end
|
16
|
+
|
@@ -1,119 +1,19 @@
|
|
1
1
|
module DLDInternet
|
2
2
|
module AWS
|
3
3
|
module RDS
|
4
|
-
|
5
|
-
HEADINGS = [
|
6
|
-
:Instance_Type,
|
7
|
-
:vCPU,
|
8
|
-
:Memory,
|
9
|
-
:PIOPS_Optimized,
|
10
|
-
:Network_Performance,
|
11
|
-
:Generation,
|
12
|
-
]
|
13
|
-
DEBUG = true
|
14
|
-
attr_reader :instance_types
|
4
|
+
module Instance_Types
|
15
5
|
|
16
|
-
|
17
|
-
def getInstanceTypes(options={})
|
18
|
-
unless @instance_types
|
19
|
-
@instance_types = {}
|
20
|
-
require 'mechanize'
|
21
|
-
mechanize = options[:mechanize]
|
22
|
-
unless mechanize
|
23
|
-
mechanize = Mechanize.new
|
24
|
-
mechanize.user_agent_alias = 'Mac Safari' # Pretend to use a Mac
|
25
|
-
end
|
26
|
-
url = options[:url] || 'http://aws.amazon.com/rds/details/'
|
6
|
+
class << self
|
27
7
|
|
28
|
-
|
8
|
+
require 'dldinternet/aws/rds/instance_types/mixins/RDS_instance_types'
|
9
|
+
include DLDInternet::AWS::RDS::Instance_Types::MixIns::RDS_Instance_Types
|
29
10
|
|
30
|
-
require 'nokogiri'
|
31
|
-
|
32
|
-
nk = Nokogiri::HTML(page.body)
|
33
|
-
div = nk.css('div.page-content')
|
34
|
-
# noinspection RubyAssignmentExpressionInConditionalInspection
|
35
|
-
if div = find_div(div, %r'^<div\s+class="nine columns content-with-nav')
|
36
|
-
# noinspection RubyAssignmentExpressionInConditionalInspection
|
37
|
-
if div = find_div(div, %r'^<div\s+class="content parsys')
|
38
|
-
divs = div.css('div').to_a
|
39
|
-
itm = nil
|
40
|
-
idx = 0
|
41
|
-
divs.each do |d|
|
42
|
-
as = d.css('div div h2 a')
|
43
|
-
as.each do |a|
|
44
|
-
# puts "'#{a.text}'"
|
45
|
-
if a.text.match %r'\s*DB Instance Classes\s*'
|
46
|
-
itm = d
|
47
|
-
break
|
48
|
-
end
|
49
|
-
end
|
50
|
-
break if itm
|
51
|
-
idx += 1
|
52
|
-
end
|
53
|
-
if idx < divs.count
|
54
|
-
divs = divs[idx..-1]
|
55
|
-
table = nil
|
56
|
-
divs.each do |d|
|
57
|
-
table = d.css('div.aws-table table')
|
58
|
-
break if table
|
59
|
-
end
|
60
|
-
|
61
|
-
@instance_types = scrapeTable(HEADINGS, table) if table
|
62
|
-
end
|
63
|
-
end
|
64
|
-
end
|
65
11
|
end
|
66
|
-
@instance_types
|
67
|
-
end
|
68
12
|
|
69
|
-
def find_div(nk,regex)
|
70
|
-
ret = nil
|
71
|
-
divs = nk.search('div')
|
72
|
-
if divs.count > 0
|
73
|
-
nine = divs.select { |div| div.to_s.match regex }
|
74
|
-
if nine.count >= 1
|
75
|
-
nine = nine.shift
|
76
|
-
ret = nine
|
77
|
-
end
|
78
|
-
end
|
79
|
-
ret
|
80
13
|
end
|
81
14
|
|
82
|
-
# ---------------------------------------------------------------------------------------------------------------
|
83
|
-
def scrapeTable(cHeadings,table)
|
84
|
-
raise Error.new 'Cannot find instance type table' unless (table.is_a?(Nokogiri::XML::Element) or table.is_a?(Nokogiri::XML::NodeSet))
|
85
|
-
rows = table.search('tr')[0..-1]
|
86
|
-
head = rows.shift
|
87
|
-
|
88
|
-
cols = head.search('td').collect { |td|
|
89
|
-
text = td.text.to_s
|
90
|
-
text = text.gsub(%r/(\r?\n)+/, ' ').strip
|
91
|
-
CGI.unescapeHTML(text)
|
92
|
-
}
|
93
|
-
instance_types = {
|
94
|
-
:headings => {},
|
95
|
-
:details => []
|
96
|
-
}
|
97
|
-
(0..cols.size-1).map { |i| instance_types[:headings][cHeadings[i]] = cols[i] }
|
98
|
-
instance_types[:headings][cHeadings[-1]] = 'Generation'
|
99
|
-
instance_set = nil
|
100
|
-
rows.each do |row|
|
101
|
-
|
102
|
-
cells = row.search('td').collect { |td|
|
103
|
-
CGI.unescapeHTML(td.text.to_s.gsub(%r/(\r?\n)+/, ' ').strip)
|
104
|
-
}
|
105
|
-
if cells.count == 1 or (cells[1] == cells[2] and cells[2] == cells[4])
|
106
|
-
instance_set = cells[0]
|
107
|
-
else
|
108
|
-
raise StandardError.new "This row does not have the same number of cells as the table header: #{row.text.to_s.strip}" unless cells.size == cols.size
|
109
|
-
instance = {}
|
110
|
-
(0..cells.size-1).map { |i| instance[cHeadings[i]] = cells[i] }
|
111
|
-
instance[cHeadings[-1]] = instance_set
|
112
|
-
instance_types[:details] << instance
|
113
|
-
end
|
114
|
-
end
|
115
|
-
instance_types
|
116
|
-
end
|
117
15
|
end
|
16
|
+
|
118
17
|
end
|
119
|
-
|
18
|
+
|
19
|
+
end
|
@@ -0,0 +1,87 @@
|
|
1
|
+
require 'thor'
|
2
|
+
require 'awesome_print'
|
3
|
+
require 'colorize'
|
4
|
+
require 'dldinternet/aws/rds/instance_types/version'
|
5
|
+
require 'dldinternet/aws/rds/instance_types/error'
|
6
|
+
require 'dldinternet/aws/rds/instance_types/scraper'
|
7
|
+
require 'dldinternet/aws/rds/instance_types/cli'
|
8
|
+
require 'dldinternet/aws/rds/instance_types'
|
9
|
+
|
10
|
+
module DLDInternet
|
11
|
+
module AWS
|
12
|
+
module RDS
|
13
|
+
module Instance_Types
|
14
|
+
class Cli < Thor
|
15
|
+
class_option :verbose, :type => :boolean
|
16
|
+
class_option :debug, :type => :boolean
|
17
|
+
class_option :log_level, :type => :string, :banner => 'Log level ([:trace, :debug, :info, :step, :warn, :error, :fatal, :todo])'
|
18
|
+
class_option :inifile, :type => :string
|
19
|
+
class_option :help, :type => :boolean
|
20
|
+
class_option :format, :type => :string, :default => 'pretty', :banner => '[:pretty, :yaml, :json]'
|
21
|
+
|
22
|
+
no_commands do
|
23
|
+
|
24
|
+
require 'dldinternet/aws/rds/instance_types/mixins/no_commands'
|
25
|
+
include DLDInternet::AWS::RDS::Instance_Types::MixIns::NoCommands
|
26
|
+
|
27
|
+
end
|
28
|
+
|
29
|
+
def help()
|
30
|
+
puts "Version: #{VERSION}"
|
31
|
+
super
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.start(argv = ARGV, config = {})
|
35
|
+
if argv.size == 0 or argv[0].match(%r'^--')
|
36
|
+
argv.unshift('get')
|
37
|
+
end
|
38
|
+
super(argv,config)
|
39
|
+
end
|
40
|
+
|
41
|
+
def initialize(args = [], local_options = {}, config = {})
|
42
|
+
super(args,local_options,config)
|
43
|
+
@log_level = :step
|
44
|
+
end
|
45
|
+
|
46
|
+
desc 'get', 'get instance types'
|
47
|
+
def get()
|
48
|
+
parse_options
|
49
|
+
puts 'get instance types' if options[:verbose]
|
50
|
+
|
51
|
+
it = DLDInternet::AWS::RDS::Instance_Types.getRDS_Instance_Types()
|
52
|
+
case options[:format]
|
53
|
+
when /yaml/
|
54
|
+
puts it.to_yaml line_width: 1024, indentation: 4, canonical: false
|
55
|
+
when /json/
|
56
|
+
puts JSON.pretty_generate( it, { indent: "\t", space: ' '})
|
57
|
+
else
|
58
|
+
puts it.ai
|
59
|
+
end
|
60
|
+
0
|
61
|
+
end
|
62
|
+
|
63
|
+
desc 'load ARGS', 'load instance types'
|
64
|
+
def load(path)
|
65
|
+
parse_options
|
66
|
+
puts 'load instance types' if options[:verbose]
|
67
|
+
|
68
|
+
it = DLDInternet::AWS::RDS::Instance_Types.loadRDS_Instance_Types(path)
|
69
|
+
ap it
|
70
|
+
end
|
71
|
+
|
72
|
+
desc 'save', 'save instance types'
|
73
|
+
def save(path)
|
74
|
+
parse_options
|
75
|
+
puts 'save instance types' if options[:verbose]
|
76
|
+
|
77
|
+
it = DLDInternet::AWS::Rds::Instance_Types.getRDS_Instance_Types()
|
78
|
+
DLDInternet::AWS::Rds::Instance_Types.saveRDS_Instance_Types(path, it)
|
79
|
+
end
|
80
|
+
|
81
|
+
default_task 'get'
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
@@ -0,0 +1,119 @@
|
|
1
|
+
require 'thor'
|
2
|
+
require 'awesome_print'
|
3
|
+
require 'inifile'
|
4
|
+
require 'colorize'
|
5
|
+
|
6
|
+
module DLDInternet
|
7
|
+
module AWS
|
8
|
+
module RDS
|
9
|
+
module Instance_Types
|
10
|
+
|
11
|
+
LOG_LEVELS = [:trace, :debug, :info, :step, :warn, :error, :fatal, :todo]
|
12
|
+
|
13
|
+
module MixIns
|
14
|
+
module NoCommands
|
15
|
+
require 'dldinternet/mixlib/logging'
|
16
|
+
include DLDInternet::Mixlib::Logging
|
17
|
+
|
18
|
+
def validate_options
|
19
|
+
_options = options.dup
|
20
|
+
if options[:log_level]
|
21
|
+
log_level = options[:log_level].to_sym
|
22
|
+
raise "Invalid log-level: #{log_level}" unless LOG_LEVELS.include?(log_level)
|
23
|
+
_options[:log_level] = log_level
|
24
|
+
else
|
25
|
+
_options[:log_level] ||= :step
|
26
|
+
end
|
27
|
+
@options = _options
|
28
|
+
end
|
29
|
+
|
30
|
+
def parse_options
|
31
|
+
if options[:help]
|
32
|
+
Cli.start(['help'])
|
33
|
+
exit 0
|
34
|
+
end
|
35
|
+
validate_options
|
36
|
+
|
37
|
+
lcs = ::Logging::ColorScheme.new( 'compiler', :levels => {
|
38
|
+
:trace => :blue,
|
39
|
+
:debug => :cyan,
|
40
|
+
:info => :green,
|
41
|
+
:step => :green,
|
42
|
+
:warn => :yellow,
|
43
|
+
:error => :red,
|
44
|
+
:fatal => :red,
|
45
|
+
:todo => :purple,
|
46
|
+
})
|
47
|
+
scheme = lcs.scheme
|
48
|
+
scheme['trace'] = "\e[38;5;33m"
|
49
|
+
scheme['fatal'] = "\e[38;5;89m"
|
50
|
+
scheme['todo'] = "\e[38;5;55m"
|
51
|
+
lcs.scheme scheme
|
52
|
+
@config = @options.dup
|
53
|
+
@config[:log_opts] = lambda{|mlll| {
|
54
|
+
:pattern => "%#{mlll}l: %m %g\n",
|
55
|
+
:date_pattern => '%Y-%m-%d %H:%M:%S',
|
56
|
+
:color_scheme => 'compiler',
|
57
|
+
:trace => (@config[:trace].nil? ? false : @config[:trace]),
|
58
|
+
# [2014-06-30 Christo] DO NOT do this ... it needs to be a FixNum!!!!
|
59
|
+
# If you want to do ::Logging.init first then fine ... go ahead :)
|
60
|
+
# :level => @config[:log_level],
|
61
|
+
}
|
62
|
+
}
|
63
|
+
@logger = getLogger(@config)
|
64
|
+
|
65
|
+
if @options[:inifile]
|
66
|
+
@options[:inifile] = File.expand_path(@options[:inifile])
|
67
|
+
unless File.exist?(@options[:inifile])
|
68
|
+
raise "#{@options[:inifile]} not found!"
|
69
|
+
end
|
70
|
+
begin
|
71
|
+
ini = ::IniFile.load(@options[:inifile])
|
72
|
+
ini['global'].each{ |key,value|
|
73
|
+
@options[key.to_s]=value
|
74
|
+
}
|
75
|
+
def _expand(k,v,regex,rerun)
|
76
|
+
matches = v.match(regex)
|
77
|
+
if matches
|
78
|
+
var = matches[1]
|
79
|
+
if options[var]
|
80
|
+
options[k]=v.gsub(/\%\(#{var}\)/,options[var]).gsub(/\%#{var}/,options[var])
|
81
|
+
else
|
82
|
+
rerun[var] = 1
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
pending = nil
|
88
|
+
rerun = {}
|
89
|
+
begin
|
90
|
+
pending = rerun
|
91
|
+
rerun = {}
|
92
|
+
options.to_hash.each{|k,v|
|
93
|
+
if v.to_s.match(/\%/)
|
94
|
+
_expand(k,v,%r'[^\\]\%\((\w+)\)', rerun)
|
95
|
+
_expand(k,v,%r'[^\\]\%(\w+)', rerun)
|
96
|
+
end
|
97
|
+
}
|
98
|
+
# Should break out the first time that we make no progress!
|
99
|
+
end while pending != rerun
|
100
|
+
rescue ::IniFile::Error => e
|
101
|
+
# noop
|
102
|
+
rescue ::Exception => e
|
103
|
+
@logger.error "#{e.class.name} #{e.message}"
|
104
|
+
raise e
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
if options[:verbose]
|
109
|
+
@logger.info "Options:\n#{options.ai}"
|
110
|
+
end
|
111
|
+
|
112
|
+
end
|
113
|
+
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
@@ -0,0 +1,91 @@
|
|
1
|
+
require 'thor'
|
2
|
+
require 'awesome_print'
|
3
|
+
require 'inifile'
|
4
|
+
require 'colorize'
|
5
|
+
require 'dldinternet/aws/rds/instance_types/scraper'
|
6
|
+
|
7
|
+
module DLDInternet
|
8
|
+
module AWS
|
9
|
+
module RDS
|
10
|
+
module Instance_Types
|
11
|
+
module MixIns
|
12
|
+
module RDS_Instance_Types
|
13
|
+
|
14
|
+
def getFileFormat(path)
|
15
|
+
format = case File.extname(File.basename(path)).downcase
|
16
|
+
when /json|js/
|
17
|
+
'json'
|
18
|
+
when /yaml|yml/
|
19
|
+
'yaml'
|
20
|
+
else
|
21
|
+
raise DLDInternet::AWS::RDS::Instance_Types::Error.new("Unsupported file type: #{path}")
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def saveRDS_Instance_Types(path,it)
|
26
|
+
format = getFileFormat(path)
|
27
|
+
begin
|
28
|
+
File.open path, File::CREAT|File::TRUNC|File::RDWR, 0644 do |f|
|
29
|
+
case format
|
30
|
+
when /yaml/
|
31
|
+
f.write it.to_yaml line_width: 1024, indentation: 4, canonical: false
|
32
|
+
when /json/
|
33
|
+
f.write JSON.pretty_generate(it, { indent: "\t", space: ' '})
|
34
|
+
else
|
35
|
+
f.write YAML::dump(it)
|
36
|
+
# abort! "Unsupported save format #{format}!"
|
37
|
+
end
|
38
|
+
f.close
|
39
|
+
end
|
40
|
+
rescue
|
41
|
+
abort! "!!! Could not write file #{path}: \nException: #{$!}\nParent directory exists? #{File.directory?(File.dirname(path))}\n"
|
42
|
+
end
|
43
|
+
0
|
44
|
+
end
|
45
|
+
|
46
|
+
def loadRDS_Instance_Types(path)
|
47
|
+
format = getFileFormat(path)
|
48
|
+
spec = File.read(path)
|
49
|
+
case format
|
50
|
+
when /json/
|
51
|
+
JSON.parse(spec)
|
52
|
+
#when /yaml/
|
53
|
+
else
|
54
|
+
begin
|
55
|
+
YAML.load(spec)
|
56
|
+
rescue Psych::SyntaxError => e
|
57
|
+
abort! "Error in the template specification: #{e.message}\n#{spec.split(/\n/).map{|l| "#{i+=1}: #{l}"}.join("\n")}"
|
58
|
+
end
|
59
|
+
# else
|
60
|
+
# abort! "Unsupported file type: #{path}"
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def getRDS_Instance_Types(mechanize=nil)
|
65
|
+
unless mechanize
|
66
|
+
require 'mechanize'
|
67
|
+
mechanize = ::Mechanize.new
|
68
|
+
mechanize.open_timeout = 5
|
69
|
+
mechanize.read_timeout = 10
|
70
|
+
end
|
71
|
+
|
72
|
+
scraper = DLDInternet::AWS::RDS::Instance_Types::Scraper.new()
|
73
|
+
|
74
|
+
begin
|
75
|
+
return scraper.getInstanceTypes(:mechanize => mechanize)
|
76
|
+
rescue Timeout::Error => e
|
77
|
+
puts "Unable to retrieve instance type details in a reasonable time (#{mechanize.open_timeout}s). Giving up ...".light_red
|
78
|
+
return nil
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
def abort!(msg)
|
83
|
+
raise DLDInternet::AWS::RDS::Instance_Types::Error.new msg
|
84
|
+
end
|
85
|
+
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
@@ -0,0 +1,127 @@
|
|
1
|
+
module DLDInternet
|
2
|
+
module AWS
|
3
|
+
module RDS
|
4
|
+
module Instance_Types
|
5
|
+
HEADINGS = [
|
6
|
+
:Instance_Type,
|
7
|
+
:vCPU,
|
8
|
+
:Memory,
|
9
|
+
:PIOPS_Optimized,
|
10
|
+
:Network_Performance,
|
11
|
+
:Generation,
|
12
|
+
]
|
13
|
+
DEBUG = true
|
14
|
+
class Scraper
|
15
|
+
attr_reader :instance_types
|
16
|
+
|
17
|
+
# ---------------------------------------------------------------------------------------------------------------
|
18
|
+
def initialize
|
19
|
+
@instance_types = {}
|
20
|
+
end
|
21
|
+
|
22
|
+
# ---------------------------------------------------------------------------------------------------------------
|
23
|
+
def getInstanceTypes(options={})
|
24
|
+
unless @instance_types.size > 0
|
25
|
+
require 'mechanize'
|
26
|
+
mechanize = options[:mechanize]
|
27
|
+
unless mechanize
|
28
|
+
mechanize = Mechanize.new
|
29
|
+
mechanize.user_agent_alias = 'Mac Safari' # Pretend to use a Mac
|
30
|
+
end
|
31
|
+
url = options[:url] || 'http://aws.amazon.com/rds/details/'
|
32
|
+
|
33
|
+
page = mechanize.get(url)
|
34
|
+
|
35
|
+
require 'nokogiri'
|
36
|
+
|
37
|
+
nk = Nokogiri::HTML(page.body)
|
38
|
+
div = nk.css('div.page-content')
|
39
|
+
# noinspection RubyAssignmentExpressionInConditionalInspection
|
40
|
+
if div = find_div(div, %r'^<div\s+class="nine columns content-with-nav')
|
41
|
+
# noinspection RubyAssignmentExpressionInConditionalInspection
|
42
|
+
if div = find_div(div, %r'^<div\s+class="content parsys')
|
43
|
+
divs = div.css('div').to_a
|
44
|
+
itm = nil
|
45
|
+
idx = 0
|
46
|
+
divs.each do |d|
|
47
|
+
as = d.css('div div h2 a')
|
48
|
+
as.each do |a|
|
49
|
+
# puts "'#{a.text}'"
|
50
|
+
if a.text.match %r'\s*DB Instance Classes\s*'
|
51
|
+
itm = d
|
52
|
+
break
|
53
|
+
end
|
54
|
+
end
|
55
|
+
break if itm
|
56
|
+
idx += 1
|
57
|
+
end
|
58
|
+
if idx < divs.count
|
59
|
+
divs = divs[idx..-1]
|
60
|
+
table = nil
|
61
|
+
divs.each do |d|
|
62
|
+
table = d.css('div.aws-table table')
|
63
|
+
break if table
|
64
|
+
end
|
65
|
+
|
66
|
+
@instance_types = scrapeTable(HEADINGS, table) if table
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
@instance_types
|
72
|
+
end
|
73
|
+
|
74
|
+
def find_div(nk,regex)
|
75
|
+
ret = nil
|
76
|
+
divs = nk.search('div')
|
77
|
+
if divs.count > 0
|
78
|
+
nine = divs.select { |div| div.to_s.match regex }
|
79
|
+
if nine.count >= 1
|
80
|
+
nine = nine.shift
|
81
|
+
ret = nine
|
82
|
+
end
|
83
|
+
end
|
84
|
+
ret
|
85
|
+
end
|
86
|
+
|
87
|
+
# ---------------------------------------------------------------------------------------------------------------
|
88
|
+
def scrapeTable(cHeadings,table)
|
89
|
+
raise Error.new 'Cannot find instance type table' unless (table.is_a?(Nokogiri::XML::Element) or table.is_a?(Nokogiri::XML::NodeSet))
|
90
|
+
rows = table.search('tr')[0..-1]
|
91
|
+
head = rows.shift
|
92
|
+
|
93
|
+
cols = head.search('td').collect { |td|
|
94
|
+
text = td.text.to_s
|
95
|
+
text = text.gsub(%r/(\r?\n)+/, ' ').strip
|
96
|
+
CGI.unescapeHTML(text)
|
97
|
+
}
|
98
|
+
instance_types = {
|
99
|
+
:headings => {},
|
100
|
+
:details => []
|
101
|
+
}
|
102
|
+
(0..cols.size-1).map { |i| instance_types[:headings][cHeadings[i]] = cols[i] }
|
103
|
+
instance_types[:headings][cHeadings[-1]] = 'Generation'
|
104
|
+
instance_set = nil
|
105
|
+
rows.each do |row|
|
106
|
+
|
107
|
+
cells = row.search('td').collect { |td|
|
108
|
+
CGI.unescapeHTML(td.text.to_s.gsub(%r/(\r?\n)+/, ' ').strip)
|
109
|
+
}
|
110
|
+
if cells.count == 1 or (cells[1] == cells[2] and cells[2] == cells[4])
|
111
|
+
instance_set = cells[0]
|
112
|
+
else
|
113
|
+
raise StandardError.new "This row does not have the same number of cells as the table header: #{row.text.to_s.strip}" unless cells.size == cols.size
|
114
|
+
instance = {}
|
115
|
+
(0..cells.size-1).map { |i| instance[cHeadings[i]] = cells[i] }
|
116
|
+
instance[cHeadings[-1]] = instance_set
|
117
|
+
instance_types[:details] << instance
|
118
|
+
end
|
119
|
+
end
|
120
|
+
instance_types
|
121
|
+
end
|
122
|
+
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: aws-rds-instance_types
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Christo De Lange
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-11-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -52,6 +52,20 @@ dependencies:
|
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0.2'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: thor
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 0.19.1
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 0.19.1
|
55
69
|
- !ruby/object:Gem::Dependency
|
56
70
|
name: mechanize
|
57
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -82,32 +96,88 @@ dependencies:
|
|
82
96
|
version: 1.5.0
|
83
97
|
- !ruby/object:Gem::Dependency
|
84
98
|
name: awesome_print
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: inifile
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">"
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :runtime
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">"
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: colorize
|
85
127
|
requirement: !ruby/object:Gem::Requirement
|
86
128
|
requirements:
|
87
129
|
- - ">="
|
88
130
|
- !ruby/object:Gem::Version
|
89
|
-
version:
|
131
|
+
version: '0'
|
90
132
|
type: :runtime
|
91
133
|
prerelease: false
|
92
134
|
version_requirements: !ruby/object:Gem::Requirement
|
93
135
|
requirements:
|
94
136
|
- - ">="
|
95
137
|
- !ruby/object:Gem::Version
|
96
|
-
version:
|
138
|
+
version: '0'
|
97
139
|
- !ruby/object:Gem::Dependency
|
98
|
-
name:
|
140
|
+
name: dldinternet-mixlib-logging
|
141
|
+
requirement: !ruby/object:Gem::Requirement
|
142
|
+
requirements:
|
143
|
+
- - ">="
|
144
|
+
- !ruby/object:Gem::Version
|
145
|
+
version: 0.4.3
|
146
|
+
type: :runtime
|
147
|
+
prerelease: false
|
148
|
+
version_requirements: !ruby/object:Gem::Requirement
|
149
|
+
requirements:
|
150
|
+
- - ">="
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: 0.4.3
|
153
|
+
- !ruby/object:Gem::Dependency
|
154
|
+
name: psych
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
156
|
+
requirements:
|
157
|
+
- - ">="
|
158
|
+
- !ruby/object:Gem::Version
|
159
|
+
version: '0'
|
160
|
+
type: :runtime
|
161
|
+
prerelease: false
|
162
|
+
version_requirements: !ruby/object:Gem::Requirement
|
163
|
+
requirements:
|
164
|
+
- - ">="
|
165
|
+
- !ruby/object:Gem::Version
|
166
|
+
version: '0'
|
167
|
+
- !ruby/object:Gem::Dependency
|
168
|
+
name: json
|
99
169
|
requirement: !ruby/object:Gem::Requirement
|
100
170
|
requirements:
|
101
171
|
- - ">="
|
102
172
|
- !ruby/object:Gem::Version
|
103
|
-
version: 0
|
173
|
+
version: '0'
|
104
174
|
type: :runtime
|
105
175
|
prerelease: false
|
106
176
|
version_requirements: !ruby/object:Gem::Requirement
|
107
177
|
requirements:
|
108
178
|
- - ">="
|
109
179
|
- !ruby/object:Gem::Version
|
110
|
-
version: 0
|
180
|
+
version: '0'
|
111
181
|
description: Retrieve an up to date list of valid AWS RDS Instance Types directly
|
112
182
|
from Amazon Web Services official site.
|
113
183
|
email: rubygems@dldinternet.com
|
@@ -131,9 +201,14 @@ files:
|
|
131
201
|
- features/step_definitions/.gitkeep
|
132
202
|
- features/step_definitions/aws-rds-instance_types_steps.rb
|
133
203
|
- lib/aws/rds/instance_types.rb
|
134
|
-
- lib/aws/rds/instance_types/version.rb
|
135
204
|
- lib/dldinternet/aws/rds.rb
|
136
205
|
- lib/dldinternet/aws/rds/instance_types.rb
|
206
|
+
- lib/dldinternet/aws/rds/instance_types/cli.rb
|
207
|
+
- lib/dldinternet/aws/rds/instance_types/error.rb
|
208
|
+
- lib/dldinternet/aws/rds/instance_types/mixins/no_commands.rb
|
209
|
+
- lib/dldinternet/aws/rds/instance_types/mixins/rds_instance_types.rb
|
210
|
+
- lib/dldinternet/aws/rds/instance_types/scraper.rb
|
211
|
+
- lib/dldinternet/aws/rds/instance_types/version.rb
|
137
212
|
homepage: https://rubygems.org/gems/aws-rds-instance_types
|
138
213
|
licenses:
|
139
214
|
- Apachev2
|