mystro-common 0.1.11 → 0.2.0

data/.gitignore

@@ -1,3 +1,6 @@
+config
+log
+
 *.gem
 *.rbc
 .bundle

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format documentation

data/CHANGELOG.md

@@ -1,5 +1,41 @@
 # Changelog
 
+## v0.2.0:
+* tests green.
+* conversion work completed... still want to test things a bit
+* custom create method for balancers
+* conversion work nearly completed. balancers working. environment creation cleaned up
+* checkin
+* fix tests to properly clean up after themselves
+* refactor some of the fog extension work. monkey patch fog/dynect support.
+* fixes for volume support
+* getting compute and records conversion completed
+* disable volumes to avoid fog bug
+* volume modification (changing root volume size) is working, but there's a bug in fog
+* conversion work
+* refactor data call to to_hash
+* dsl conversion
+* check in
+* add access to config, allow to disable in configuration
+* add raw attribute to listener
+* cleanup
+* add _raw to decoded object
+* dynect record destroy working. refactor tests to use configuration file. add _raw attribute to models, this can contain source object (fog object). reopen fog dynect records to add find_by_name.
+* working with shawncatz/fog fork
+* dynect
+* some refactoring, support for dynect started
+* refactor to shared examples
+* compute, balancers and records working. includes tests for cloud operations and models
+* converted to new format, organizations now use new connect framework, started support for DNS
+* balancers
+* rspec integration
+* add LIST to Version class. makes it easier to handle release tasks
+* providers support
+* orgs
+
+## v0.1.11:
+* fixes bug if disabled is not set
+
 ## v0.1.10:
 * allow for disabling plugins
 
@@ -18,17 +54,19 @@
 * changelog rake tasks
 
 ## v0.1.6:
-
-## v0.1.5:
 * update rails to 3.2.14. tweak plugins jobs output to match qujo. fix issue with getting zones through fog
 * smarter errors in dns connect. add and use Account#get instead of @list attr
-* hard code rake version in Gemfile to match mystro server
-* support for templates in userdata, handled like files, but processed through erubis
+* merge
 * merge release v0.1.2
 * new DSL framework
 * ruby version file
 * changelog rake tasks
 
+## v0.1.4:
+* merge
+* hard code rake version in Gemfile to match mystro server
+* support for templates in userdata, handled like files, but processed through erubis
+
 ## v0.1.3:
 * fixes bugs with dns usage and plugin methods when no plugins are defined
 * update ruby version

data/Gemfile

@@ -3,5 +3,11 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in mystro-common.gemspec
 gemspec
 
-gem "rake", '10.0.4'
+gem 'damsel', path: '../../damsel'
+
+gem 'rake', '10.0.4'
 gem 'awesome_print'
+gem 'rspec'
+gem 'terminal-table'
+gem 'factory_girl', '~> 4.0'
+gem 'net-dns'

data/Rakefile

@@ -1,4 +1,140 @@
-require "bundler/gem_tasks"
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+require 'mystro-common'
+require 'terminal-table'
+require 'awesome_print'
+
+RSpec::Core::RakeTask.new(:spec)
+task :default => :spec
+desc 'run tests and create computes and such that cost money'
+task :spend do
+  Mystro.config.test!.spend = true
+  Rake::Task['spec'].invoke
+end
+def table(head, rows=nil)
+  if rows
+    t = Terminal::Table.new :headings => head, :rows => rows
+  else
+    t = Terminal::Table.new :rows => rows
+  end
+  t
+end
+
+def list(keys, list)
+  rows = []
+  list.each do |l|
+    row = []
+    keys.each do |k|
+      row << (l[k] || l[k.downcase] || l[k.to_sym] || l[k.downcase.to_sym])
+    end
+    rows << row
+  end
+  table(keys, rows)
+end
+
+def show(obj)
+  keys = obj.keys
+  rows = []
+  keys.each do |k|
+    list = [obj[k]].flatten
+    list.each do |v|
+      if v.is_a?(Hash)
+        v = v.inject([]) {|s, e| s << e.join(": ")}.join("\n")
+      end
+      v
+      rows << [k, v]
+    end
+  end
+  table(%w{key value}, rows)
+end
+
+def options
+  {aws_access_key_id: 'AKIAIVMUCDVWWZFFLVGA', aws_secret_access_key: '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'}
+end
+
+#Mystro::Log.console_debug
+#Mystro::Log.debug "logging ... "
+
+desc 'get and show compute'
+task :compute do
+  x = Mystro.compute
+  o = x.find("i-69d32404")
+  e = o.to_hash
+  e.merge!(name: o.name)
+  puts show(e)
+end
+
+desc 'get list of computes'
+task :computes do
+  x = Mystro.compute
+  list = x.all
+  list.map! {|e| e.to_hash.merge(name: e.tags['Name'])}
+  puts list(%w{id name state ip dns}, list)
+end
+
+desc 'get and show balancer'
+task :balancer do
+  x = Mystro.balancer
+  o = x.find 'RG-EVENTS-1'
+  e = o.to_hash
+  puts show(e)
+end
+
+desc 'get and show zone'
+task :record do
+  o = Mystro::Organization.get('rg')
+  x = o.record
+  r = x.find_by_name 'mcstg1.rgops.com'
+  e = r.to_hash
+  puts show(e)
+  r = x.find '75069597'
+  e = r.to_hash
+  puts show(e)
+end
+
+desc 'get and show zone'
+task :records do
+  x = Mystro.record
+  o = x.all
+  puts list(%w{name type ttl values}, o)
+end
+
+
+desc 'show default organization'
+task :org do
+  puts Mystro.organization.to_hash.to_yaml
+end
+
+desc 'show configuration'
+task :config do
+  puts Mystro.config.to_hash.to_yaml
+end
+
+desc 'template'
+task :template, [:name] do |_, args|
+  name = args.name || 'hdp/live'
+  name.gsub!(/\.rb$/, '') if name =~ /\.rb$/
+  t = Mystro::Dsl.load("config/mystro/templates/#{name}.rb")
+  ap t
+end
+
+desc 'template cloud'
+task :cloud, [:name] do |_, args|
+  name = args.name || 'hdp/live'
+  ap Mystro::Dsl.load("config/mystro/templates/#{name}.rb").actions
+end
+
+desc 'compute volumes'
+task :volumes do
+  org = Mystro::Organization.get 'ops'
+  connect = org.compute
+  service = connect.service
+  template = Mystro::Dsl.load("config/mystro/templates/hdp/live.rb")
+  puts template.to_hash.to_yaml
+  #actions  = template.actions
+  #options = connect.encode(actions.first.data)
+  #ap options
+end
 
 def changelog(last=nil, single=false)
   command="git --no-pager log --format='%an::::%h::::%s'"

data/lib/fog/ext/dynect/dns.rb

@@ -0,0 +1,140 @@
+require 'fog/dynect'
+require 'fog/dns'
+
+module Fog
+  module DNS
+    class Dynect < Fog::Service
+
+      requires :dynect_customer, :dynect_username, :dynect_password
+      recognizes :timeout, :persistent
+      recognizes :provider # remove post deprecation
+
+      model_path 'fog/ext/dynect/models/dns'
+      model       :record
+      collection  :records
+      model       :zone
+      collection  :zones
+
+      request_path 'fog/ext/dynect/requests/dns'
+      request :delete_record
+      request :delete_zone
+      request :get_node_list
+      request :get_all_records
+      request :get_record
+      request :get_zone
+      request :post_record
+      request :post_session
+      request :post_zone
+      request :put_zone
+
+      class JobIncomplete < Error; end
+
+      class Mock
+        def initialize(options={})
+          @dynect_customer = options[:dynect_customer]
+          @dynect_username = options[:dynect_username]
+          @dynect_password = options[:dynect_password]
+        end
+
+        def self.data
+          @data ||= {
+            :zones => {}
+          }
+        end
+
+        def self.reset
+          @data = nil
+        end
+
+        def auth_token
+          @auth_token ||= Fog::Dynect::Mock.token
+        end
+
+        def data
+          self.class.data
+        end
+
+        def reset_data
+          self.class.reset
+        end
+      end
+
+      class Real
+        def initialize(options={})
+          @dynect_customer = options[:dynect_customer]
+          @dynect_username = options[:dynect_username]
+          @dynect_password = options[:dynect_password]
+
+          @connection_options = options[:connection_options] || {}
+          @host       = 'api-v4.dynect.net'
+          @port       = options[:port]        || 443
+          @path       = options[:path]        || '/REST'
+          @persistent = options[:persistent]  || false
+          @scheme     = options[:scheme]      || 'https'
+          @version    = options[:version]     || '3.5.2'
+          @connection = Fog::Connection.new("#{@scheme}://#{@host}:#{@port}", @persistent, @connection_options)
+        end
+
+        def auth_token
+          @auth_token ||= post_session.body['data']['token']
+        end
+
+        def request(params)
+          begin
+            # any request could redirect to a job
+            params[:expects] = Array(params[:expects]) | [307]
+
+            params[:headers] ||= {}
+            params[:headers]['Content-Type'] = 'application/json'
+            params[:headers]['API-Version'] = @version
+            params[:headers]['Auth-Token'] = auth_token unless params[:path] == 'Session'
+            params[:path] = "#{@path}/#{params[:path]}" unless params[:path] =~ %r{^#{Regexp.escape(@path)}/}
+
+            response = @connection.request(params.merge!({:host => @host}))
+
+            if response.body.empty?
+              response.body = {}
+            elsif response.headers['Content-Type'] == 'application/json'
+              response.body = Fog::JSON.decode(response.body)
+            end
+
+            if response.body['status'] == 'failure'
+              raise Error, response.body['msgs'].first['INFO']
+            end
+
+            if response.status == 307 && params[:path] !~ %r{^/REST/Job/}
+              response = poll_job(response, params[:expects])
+            end
+
+            response
+          rescue Excon::Errors::HTTPStatusError => error
+            if @auth_token && error.message =~ /login: (Bad or expired credentials|inactivity logout)/
+              @auth_token = nil
+              retry
+            else
+              raise error
+            end
+          end
+
+          response
+        end
+
+        def poll_job(response, original_expects, time_to_wait = 10)
+          job_location = response.headers['Location']
+
+          Fog.wait_for(time_to_wait) do
+            response = request(:expects => original_expects, :method => :get, :path => job_location)
+            response.body['status'] != 'incomplete'
+          end
+
+          if response.body['status'] == 'incomplete'
+            raise JobIncomplete.new("Job #{response.body['job_id']} is still incomplete")
+          end
+
+          response
+        end
+      end
+
+    end
+  end
+end

data/lib/fog/ext/dynect/models/dns/record.rb

@@ -0,0 +1,66 @@
+require 'fog/core/model'
+
+module Fog
+  module DNS
+    class Dynect
+
+      class Record < Fog::Model
+        extend Fog::Deprecation
+
+        identity  :id
+        attribute :name,        :aliases => [:fqdn, 'fqdn']
+        attribute :rdata
+        attribute :serial_style
+        attribute :ttl
+        attribute :type,        :aliases => 'record_type'
+
+        def destroy
+          requires :identity, :name, :type, :zone
+          service.delete_record(type, zone.identity, name, identity)
+          true
+        end
+
+        def save
+          requires :name, :type, :rdata, :zone
+
+          options = {
+            :ttl => ttl
+          }
+          options.delete_if {|key, value| value.nil?}
+
+          data = service.post_record(type, zone.identity, name, rdata, options).body['data']
+          # avoid overwriting zone object with zone string
+          data = data.reject {|key, value| key == 'zone'}
+          merge_attributes(data)
+
+          zone.publish
+          records = service.get_record(type, zone.identity, name).body['data']
+          # data in format ['/REST/xRecord/domain/fqdn/identity]
+          records.map! do |record|
+            tokens = record.split('/')
+            {
+              :identity => tokens.last,
+              :type     => tokens[2][0...-6] # everything before 'Record'
+            }
+          end
+          record = records.detect {|record| record[:type] == type}
+          merge_attributes(record)
+
+          true
+        end
+
+        def zone
+          @zone
+        end
+
+        private
+
+        def zone=(new_zone)
+          @zone = new_zone
+        end
+
+      end
+
+    end
+  end
+end

data/lib/fog/ext/dynect/models/dns/records.rb

@@ -0,0 +1,87 @@
+require 'fog/core/collection'
+require 'fog/ext/dynect/models/dns/record'
+
+module Fog
+  module DNS
+    class Dynect
+
+      class Records < Fog::Collection
+
+        attribute :zone
+
+        model Fog::DNS::Dynect::Record
+
+        def all(options = {})
+          requires :zone
+          data = []
+          service.get_all_records(zone.domain, options).body['data'].each do |url|
+            (_, _, t, _, fqdn, id) = url.split('/')
+            type = t.gsub(/Record$/, '')
+
+            # leave out the default, read only records
+            # by putting this here we don't make the secondary request for these records
+            next if ['NS', 'SOA'].include?(type)
+            record = service.get_record(type, zone.domain, fqdn, 'record_id' => id).body['data']
+
+            data << {
+              :identity => record['record_id'],
+              :fqdn => record['fqdn'],
+              :type => record['record_type'],
+              :rdata => record['rdata']
+            }
+          end
+
+          load(data)
+        end
+
+        def get(record_id)
+          requires :zone
+
+          list = service.get_all_records(zone.domain, {}).body['data']
+          url = list.detect { |e| e =~ /\/#{record_id}$/ }
+          return unless url
+          (_, _, t, _, fqdn, id) = url.split('/')
+          type = t.gsub(/Record$/, '')
+          record = service.get_record(type, zone.domain, fqdn, 'record_id' => id).body['data']
+
+          new({
+            :identity => record['record_id'],
+            :fqdn => record['fqdn'],
+            :type => record['record_type'],
+            :rdata => record['rdata']
+          })
+        end
+
+        def new(attributes = {})
+          requires :zone
+          super({:zone => zone}.merge!(attributes))
+        end
+
+        def find_by_name(name)
+          requires :zone
+          data = []
+          service.get_all_records(zone.domain, {}).body['data'].select { |url| url =~ /\/#{name}\// }.each do |url|
+            (_, _, t, _, fqdn, id) = url.split('/')
+            type = t.gsub(/Record$/, '')
+
+            # leave out the default, read only records
+            next if ['NS', 'SOA'].include?(type)
+
+            record = service.get_record(type, zone.domain, fqdn, 'record_id' => id).body['data']
+
+            data << {
+                :identity => record['record_id'],
+                :fqdn => record['fqdn'],
+                :type => record['record_type'],
+                :rdata => record['rdata']
+            }
+          end
+
+          load(data)
+        end
+
+      end
+
+    end
+  end
+end

data/lib/fog/ext/dynect/models/dns/zone.rb

@@ -0,0 +1,60 @@
+require 'fog/core/model'
+require 'fog/ext/dynect/models/dns/records'
+
+module Fog
+  module DNS
+    class Dynect
+
+      class Zone < Fog::Model
+
+        identity  :domain
+
+        attribute :domain,        :aliases => 'zone'
+        attribute :email,         :aliases => 'rname'
+        attribute :serial
+        attribute :serial_style
+        attribute :ttl
+        attribute :type,          :aliases => 'zone_type'
+
+        def initialize(attributes={})
+          super
+        end
+
+        def destroy
+          requires :domain
+          service.delete_zone(domain)
+          true
+        end
+
+        undef_method :domain=
+        def domain=(new_domain)
+          attributes[:domain] = new_domain.split('/').last
+        end
+
+        def publish
+          requires :identity
+          data = service.put_zone(identity, 'publish' => true)
+          true
+        end
+
+        def records
+          @records ||= Fog::DNS::Dynect::Records.new(:zone => self, :service => service)
+        end
+
+        def nameservers
+          raise 'nameservers Not Implemented'
+        end
+
+        def save
+          self.ttl ||= 3600
+          requires :domain, :email, :ttl
+          data = service.post_zone(email, ttl, domain).body['data']
+          merge_attributes(data)
+          true
+        end
+
+      end
+
+    end
+  end
+end

data/lib/fog/ext/dynect/models/dns/zones.rb

@@ -0,0 +1,29 @@
+require 'fog/core/collection'
+require 'fog/ext/dynect/models/dns/zone'
+
+module Fog
+  module DNS
+    class Dynect
+
+      class Zones < Fog::Collection
+
+        model Fog::DNS::Dynect::Zone
+
+        def all
+          data = service.get_zone.body['data'].map do |zone|
+            { :domain => zone }
+          end
+          load(data)
+        end
+
+        def get(zone_id)
+          new(service.get_zone('zone' => zone_id).body['data'])
+        rescue Excon::Errors::NotFound
+          nil
+        end
+
+      end
+
+    end
+  end
+end

data/lib/fog/ext/dynect/requests/dns/delete_record.rb

@@ -0,0 +1,56 @@
+module Fog
+  module DNS
+    class Dynect
+      class Real
+
+        # Delete a record
+        #
+        # ==== Parameters
+        # * type<~String> - type of record in ['AAAA', 'ANY', 'A', 'CNAME', 'DHCID', 'DNAME', 'DNSKEY', 'DS', 'KEY', 'LOC', 'MX', 'NSA', 'NS', 'PTR', 'PX', 'RP', 'SOA', 'SPF', 'SRV', 'SSHFP', 'TXT']
+        # * zone<~String> - zone of record
+        # * fqdn<~String> - fqdn of record
+        # * record_id<~String> - id of record
+
+        def delete_record(type, zone, fqdn, record_id)
+          request(
+            :expects  => 200,
+            :idempotent => true,
+            :method   => :delete,
+            :path     => ["#{type.to_s.upcase}Record", zone, fqdn, record_id].join('/')
+          )
+        end
+      end
+
+      class Mock
+        def delete_record(type, zone, fqdn, record_id)
+          raise Fog::DNS::Dynect::NotFound unless zone = self.data[:zones][zone]
+
+          raise Fog::DNS::Dynect::NotFound unless zone[:records][type].find { |record| record[:fqdn] == fqdn && record[:record_id] == record_id.to_i }
+
+          zone[:records_to_delete] << {
+            :type => type,
+            :fqdn => fqdn,
+            :record_id => record_id.to_i
+          }
+
+          response = Excon::Response.new
+          response.status = 200
+
+          response.body = {
+            "status" => "success",
+            "data" => {},
+            "job_id" => Fog::Dynect::Mock.job_id,
+            "msgs" => [{
+              "INFO" => "delete: Record will be deleted on zone publish",
+              "SOURCE" => "BLL",
+              "ERR_CD" => nil,
+              "LVL" => "INFO"
+            }]
+          }
+
+          response
+        end
+      end
+    end
+  end
+end