record_store 5.10.0 → 6.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 305f971c2df40732e6ed0a53798af1ef1ffe4bda959454beea2c9f2d4da09ee7
-  data.tar.gz: 4233420dd7d25ac624d52d643a4326762133f6531784de8174c1de15ca890a57
+  metadata.gz: f7a5da9483eaf8ce4eb82e8d974fefe82f7f481cde945a45ec6ae204b02524dc
+  data.tar.gz: 51e681688b6cc80c851f801ad00bb07ccf473ea2fb0b1c8a8e722d3cbc2ec196
 SHA512:
-  metadata.gz: 99034b1c2625066dd7c4d32ee33582ef58ad2203197c52650f2a810365799604969b49575699f70ad8a009ee443838cd2de8989e75724dc4dc8967fecfbc6e63
-  data.tar.gz: 150539a073dc355c83288f0356ba8db15d28287e17977dd7cce882487a439f19cfa614305cd67b3f1ffe1aa21f7359c170a168803248af9d8eb914f5fd844496
+  metadata.gz: 502acb53fcef150d5cb5442414c47552fc199efe66012e85d30745ae25c8af326064283c855ba12f4cead5b6f7b09f5a97c0b3bd6a4de1a010bba15664954335
+  data.tar.gz: 491ee9ff80ccb35ceb7ffbe70776948cd459467df8c213917c08ac6830e249585dbd993849a44e413bd6fc8e38589601eab27601cf3061c9b51abd09a94898a0

data/.rubocop-https---shopify-github-io-ruby-style-guide-rubocop-yml

@@ -803,7 +803,7 @@ Layout/SpaceInsideRangeLiteral:
 Style/SymbolLiteral:
   Enabled: true
 
-Layout/Tab:
+Layout/IndentationStyle:
   Enabled: true
 
 Layout/TrailingWhitespace:

data/CHANGELOG.md

@@ -1,5 +1,23 @@
 # CHANGELOG
 
+## 6.1.1
+- Emit messages when waiting for rate-limit to elapse for DNSimple and NS1 providers, so deployment does not timeout [BUGFIX]
+
+## 6.1.0
+- sort zone files [FEATURE]
+- CLI support for specifying zones for validate_authority [FEATURE]
+- retry failed lookup using another nameserver if unreachable [BUGFIX]
+- ignore records other than NS in authority section [BUGFIX]
+
+## 6.0.1
+- add API rate limiting to DNSimple provider [FEATURE]
+
+## 6.0.0
+- add `--all` option for `record-store diff` to compare ignored records too [FEATURE]
+
+## 5.11.0
+- support PTR record type [FEATURE]
+
 ## 5.10.0
 - add `record-store validate_authority` command to sanity check delegation [FEATURE]
 - fix handling of NXDOMAIN, etc. when fetching authoritative nameservers [BUGFIX]

data/lib/record_store.rb

@@ -19,6 +19,7 @@ require 'record_store/record/caa'
 require 'record_store/record/cname'
 require 'record_store/record/mx'
 require 'record_store/record/ns'
+require 'record_store/record/ptr'
 require 'record_store/record/sshfp'
 require 'record_store/record/txt'
 require 'record_store/record/spf'

data/lib/record_store/changeset.rb

@@ -36,12 +36,15 @@ module RecordStore
 
     attr_reader :current_records, :desired_records, :removals, :additions, :updates, :provider, :zone
 
-    def self.build_from(provider:, zone:)
+    def self.build_from(provider:, zone:, all: false)
       current_zone = provider.build_zone(zone_name: zone.unrooted_name, config: zone.config)
 
+      current_records = all ? current_zone.all : current_zone.records
+      desired_records = all ? zone.all : zone.records
+
       new(
-        current_records: current_zone.records,
-        desired_records: zone.records,
+        current_records: current_records,
+        desired_records: desired_records,
         provider: provider,
         zone: zone.unrooted_name
       )

data/lib/record_store/cli.rb

@@ -59,13 +59,18 @@ module RecordStore
       end
     end
 
-    desc 'diff', 'Displays the DNS differences between the zone files in this repo and production'
+    desc 'diff [ZONE ...]', 'Displays the DNS differences between the zone files in this repo and production'
+    option :all, desc: 'Include all records', aliases: '-a', type: :boolean, default: false
     option :verbose, desc: 'Print records that haven\'t diverged', aliases: '-v', type: :boolean, default: false
-    def diff
-      puts "Diffing #{Zone.defined.count} zones"
+    def diff(*zones)
+      puts "Diffing #{zones.any? ? zones.count : Zone.defined.count} zone(s)"
+
+      all = options.fetch('all')
 
       Zone.each do |name, zone|
-        changesets = zone.build_changesets
+        next unless zones.empty? || zones.include?(name)
+
+        changesets = zone.build_changesets(all: all)
 
         if !options.fetch('verbose') && changesets.all?(&:empty?)
           print_and_flush('.')
@@ -81,21 +86,21 @@ module RecordStore
           puts '-' * 20
           puts "Provider: #{changeset.provider}"
 
-          if !changeset.additions.empty? || options.fetch('verbose')
+          if changeset.additions.any?
             puts "Add:"
             changeset.additions.map(&:record).each do |record|
               puts " - #{record}"
             end
           end
 
-          if !changeset.removals.empty? || options.fetch('verbose')
+          if changeset.removals.any?
             puts "Remove:"
             changeset.removals.map(&:record).each do |record|
               puts " - #{record}"
             end
           end
 
-          if !changeset.updates.empty? || options.fetch('verbose')
+          if changeset.updates.any?
             puts "Update:"
             changeset.updates.map(&:record).each do |record|
               puts " - #{record}"
@@ -215,12 +220,14 @@ module RecordStore
       end
     end
 
-    desc 'validate_authority', 'Validates that authoritative nameservers match the providers'
+    desc 'validate_authority [ZONE ...]', 'Validates that authoritative nameservers match the providers'
     option :verbose, desc: 'Include valid zones in output', aliases: '-v', type: :boolean, default: false
-    def validate_authority
+    def validate_authority(*zones)
       verbose = options.fetch('verbose')
 
       Zone.each do |name, zone|
+        next unless zones.empty? || zones.include?(name)
+
         authority = zone.fetch_authority
 
         delegation = Hash.new { |h, k| h[k] = [] }

data/lib/record_store/provider/dnsimple.rb

@@ -1,10 +1,11 @@
 require 'dnsimple'
+require_relative 'dnsimple/patch_api_header'
 
 module RecordStore
   class Provider::DNSimple < Provider
     class << self
       def record_types
-        super | Set.new(['SSHFP'])
+        super | Set.new(%w(PTR SSHFP))
       end
 
       def supports_alias?
@@ -99,6 +100,8 @@ module RecordStore
           record.merge!(preference: api_record.priority, exchange: api_record.content)
         when 'NS'
           record.merge!(nsdname: api_record.content)
+        when 'PTR'
+          record.merge!(ptrdname: api_record.content)
         when 'SSHFP'
           algorithm, fptype, fingerprint = api_record.content.split(' ')
           record.merge!(
@@ -143,6 +146,8 @@ module RecordStore
           record_hash[:content] = record.exchange.chomp('.')
         when 'NS'
           record_hash[:content] = record.nsdname.chomp('.')
+        when 'PTR'
+          record_hash[:content] = record.ptrdname.chomp('.')
         when 'SSHFP'
           record_hash[:content] = record.rdata_txt
         when 'SPF', 'TXT'

data/lib/record_store/provider/dnsimple/patch_api_header.rb

@@ -0,0 +1,33 @@
+require_relative '../provider_utils/rate_limit'
+
+# Patch Dnsimple client method which retrieves headers for API rate limit dynamically
+module Dnsimple
+  class Client
+    def execute(method, path, data = nil, options = {})
+      response = request(method, path, data, options)
+      rate_limit_sleep(response.headers['x-ratelimit-reset'].to_i, response.headers['x-ratelimit-remaining'].to_i)
+
+      case response.code
+      when 200..299
+        response
+      when 401
+        raise AuthenticationFailed, response['message']
+      when 404
+        raise NotFoundError, response
+      else
+        raise RequestError, response
+      end
+    end
+
+    private
+
+    def rate_limit_sleep(rate_limit_reset, rate_limit_remaining)
+      rate_limit_reset_in = [0, rate_limit_reset - Time.now.to_i].max
+      rate_limit_periods = rate_limit_remaining + 1
+      sleep_time = rate_limit_reset_in / rate_limit_periods.to_f
+
+      rate_limit = RateLimit.new('DNSimple')
+      rate_limit.sleep_for(sleep_time)
+    end
+  end
+end

data/lib/record_store/provider/ns1.rb

@@ -41,6 +41,10 @@ module RecordStore
     end
 
     class << self
+      def record_types
+        super | Set.new(%w(PTR))
+      end
+
       def client
         Provider::NS1::Client.new(api_key: secrets['api_key'])
       end
@@ -229,6 +233,8 @@ module RecordStore
             )
           when 'NS'
             record.merge!(nsdname: answer.rrdata_string)
+          when 'PTR'
+            record.merge!(ptrdname: answer.rrdata_string)
           when 'SPF', 'TXT'
             record.merge!(txtdata: answer.rrdata_string.gsub(';', '\;'))
           when 'SRV'

data/lib/record_store/provider/ns1/patch_api_header.rb

@@ -1,11 +1,15 @@
 require 'net/http'
+require_relative '../provider_utils/rate_limit'
 
 # Patch the method which retrieves headers for API rate limit dynamically
 module NS1::Transport
   class NetHttp
     def process_response(response)
-      sleep(response.to_hash["x-ratelimit-period"].first.to_i /
-        [1, response.to_hash["x-ratelimit-remaining"].first.to_i].max.to_f)
+      sleep_time = response.to_hash['x-ratelimit-period'].first.to_i /
+                   [1, response.to_hash['x-ratelimit-remaining'].first.to_i].max.to_f
+
+      rate_limit = RateLimit.new('NS1')
+      rate_limit.sleep_for(sleep_time)
 
       body = JSON.parse(response.body)
       case response

data/lib/record_store/provider/provider_utils/rate_limit.rb

@@ -0,0 +1,14 @@
+class RateLimit
+  def initialize(provider)
+    @provider = provider
+  end
+
+  def sleep_for(sleep_time)
+    while sleep_time > 0
+      wait = [10, sleep_time].min
+      puts "Waiting on #{@provider} rate-limit" if wait > 1
+      sleep(wait)
+      sleep_time -= wait
+    end
+  end
+end

data/lib/record_store/record/ptr.rb

@@ -0,0 +1,42 @@
+module RecordStore
+  class Record::PTR < Record
+    attr_accessor :ptrdname
+
+    OCTET_LABEL_SEQUENCE_REGEX = /\A(?:([0-9]|[1-9][0-9]|[1-9][0-9][0-9])\.){1,4}/
+    IN_ADDR_ARPA_SUFFIX_REGEX = /in-addr\.arpa\.\z/
+    FQDN_FORMAT_REGEX = Regexp.new(OCTET_LABEL_SEQUENCE_REGEX.source + IN_ADDR_ARPA_SUFFIX_REGEX.source)
+
+    validates_format_of :fqdn, with: FQDN_FORMAT_REGEX
+
+    validate :validate_fqdn_octets_in_range
+    validate :validate_fqdn_is_in_addr_arpa_subzone
+
+    def initialize(record)
+      super
+
+      @ptrdname = Record.ensure_ends_with_dot(record.fetch(:ptrdname))
+    end
+
+    def rdata
+      { ptrdname: ptrdname }
+    end
+
+    def rdata_txt
+      ptrdname.to_s
+    end
+
+    def validate_fqdn_octets_in_range
+      OCTET_LABEL_SEQUENCE_REGEX.match(fqdn) do |m|
+        unless m.captures.all? { |o| o.to_d.between?(0, 255) }
+          errors.add(:fqdn, 'octet labels must be within the range 0-255')
+        end
+      end
+    end
+
+    def validate_fqdn_is_in_addr_arpa_subzone
+      unless IN_ADDR_ARPA_SUFFIX_REGEX.match?(fqdn)
+        errors.add(:fqdn, 'PTR records may only exist in the in-addr.arpa zone')
+      end
+    end
+  end
+end

data/lib/record_store/version.rb

@@ -1,3 +1,3 @@
 module RecordStore
-  VERSION = '5.10.0'.freeze
+  VERSION = '6.1.1'.freeze
 end

data/lib/record_store/zone.rb

@@ -70,10 +70,10 @@ module RecordStore
       @records = build_records(records)
     end
 
-    def build_changesets
+    def build_changesets(all: false)
       @changesets ||= begin
         providers.map do |provider|
-          Changeset.build_from(provider: provider, zone: self)
+          Changeset.build_from(provider: provider, zone: self, all: all)
         end
       end
     end
@@ -129,14 +129,12 @@ module RecordStore
     )
 
     def fetch_authority(nameserver = ROOT_SERVERS.sample)
-      authority = Resolv::DNS.open(nameserver: nameserver) do |resolv|
-        resolv.fetch_resource(name, Resolv::DNS::Resource::IN::SOA) do |reply, name|
-          break if reply.answer.any?
+      authority = fetch_soa(nameserver) do |reply, _name|
+        break if reply.answer.any?
 
-          raise "No authority found (#{name})" unless reply.authority.any?
+        raise "No authority found (#{name})" unless reply.authority.any?
 
-          break extract_authority(reply)
-        end
+        break extract_authority(reply.authority)
       end
 
       # candidate DNS name is returned instead when NXDomain or other error
@@ -147,18 +145,39 @@ module RecordStore
 
     private
 
-    def extract_authority(reply)
-      authority = reply.authority.sample
+    def fetch_soa(nameserver)
+      Resolv::DNS.open(nameserver: nameserver) do |resolv|
+        resolv.fetch_resource(name, Resolv::DNS::Resource::IN::SOA) do |reply, name|
+          yield reply, name
+        end
+      end
+    end
 
-      if unrooted_name.casecmp?(authority.first.to_s)
-        build_authority(reply.authority)
+    def resolve_authority(authority)
+      nameservers = authority.map { |a| a.last.name.to_s }
+
+      begin
+        nameserver = nameservers.shift
+        fetch_authority(nameserver)
+      rescue Errno::EHOSTUNREACH => e
+        $stderr.puts "Warning: #{e} [host=#{nameserver}]"
+        raise if nameservers.empty?
+        retry
+      end
+    end
+
+    def extract_authority(authority)
+      if unrooted_name.casecmp?(authority.first.first.to_s)
+        build_authority(authority)
       else
-        fetch_authority(authority.last.name.to_s) || build_authority(reply.authority)
+        resolve_authority(authority) || build_authority(authority)
       end
     end
 
     def build_authority(authority)
-      authority.map.with_index do |(name, ttl, data), index|
+      ns = authority.select { |_name, _ttl, data| data.is_a?(Resolv::DNS::Resource::IN::NS) }
+
+      ns.map.with_index do |(name, ttl, data), index|
         Record::NS.new(ttl: ttl, fqdn: name.to_s, nsdname: data.name.to_s, record_id: index)
       end
     end

data/lib/record_store/zone/yaml_definitions.rb

@@ -10,6 +10,7 @@ module RecordStore
       def defined
         @defined ||= yaml_files
           .map { |file| load_yml_zone_definition(file) }
+          .sort_by(&:unrooted_name)
           .index_by(&:unrooted_name)
       end
 

data/record_store.gemspec

@@ -36,7 +36,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'fog-xml'
   spec.add_runtime_dependency 'fog-dynect', '~> 0.4.0'
   spec.add_runtime_dependency 'dnsimple', '~> 4.4.0'
-  spec.add_runtime_dependency 'google-cloud-dns'
+  spec.add_runtime_dependency 'google-cloud-dns', '~> 0.31.0'
   spec.add_runtime_dependency 'ruby-limiter', '~> 1.0', '>= 1.0.1'
   spec.add_runtime_dependency 'ns1'
   spec.add_runtime_dependency 'oci', '~> 2.6.0'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: record_store
 version: !ruby/object:Gem::Version
-  version: 5.10.0
+  version: 6.1.1
 platform: ruby
 authors:
 - Willem van Bergen
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-15 00:00:00.000000000 Z
+date: 2020-06-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -141,16 +141,16 @@ dependencies:
   name: google-cloud-dns
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.31.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.31.0
 - !ruby/object:Gem::Dependency
   name: ruby-limiter
   requirement: !ruby/object:Gem::Requirement
@@ -355,12 +355,14 @@ files:
 - lib/record_store/cli.rb
 - lib/record_store/provider.rb
 - lib/record_store/provider/dnsimple.rb
+- lib/record_store/provider/dnsimple/patch_api_header.rb
 - lib/record_store/provider/dynect.rb
 - lib/record_store/provider/google_cloud_dns.rb
 - lib/record_store/provider/ns1.rb
 - lib/record_store/provider/ns1/client.rb
 - lib/record_store/provider/ns1/patch_api_header.rb
 - lib/record_store/provider/oracle_cloud_dns.rb
+- lib/record_store/provider/provider_utils/rate_limit.rb
 - lib/record_store/record.rb
 - lib/record_store/record/a.rb
 - lib/record_store/record/aaaa.rb
@@ -369,6 +371,7 @@ files:
 - lib/record_store/record/cname.rb
 - lib/record_store/record/mx.rb
 - lib/record_store/record/ns.rb
+- lib/record_store/record/ptr.rb
 - lib/record_store/record/spf.rb
 - lib/record_store/record/srv.rb
 - lib/record_store/record/sshfp.rb