record_store 6.0.1 → 6.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9c526e771b65c9da7f83836311bfc8c7b2af502a6801256ccdaa540b54df4138
-  data.tar.gz: f5e34d123f4dbef6d2206c4703310af91784deb0d0d2681de05f4aea78b83985
+  metadata.gz: f2933700da7288ff5085d9865364776031896ea9efada8cf75fea37e5edb27b1
+  data.tar.gz: 5d1420e6df2c086d7d98ce655f8b8512bd65467ff4825a6f00b628b16b5fa180
 SHA512:
-  metadata.gz: d5c507ba11cdefb048f6ceb74f30c1edef6a0f4bd0f0558126f7be02de3b9b38bab5c9d02bf8c8fec9f07ac59207dd24115fac7d0819141029f2691da989ada2
-  data.tar.gz: d688116ba3470d5ff1a4ab4d74b8d38d8d274d1180988839679d1d9b8aeb2e813a20b3fbc0634077e9a216b74d28ca1b0e3da125c43240488f5626a9c6fbd265
+  metadata.gz: eee575d27faa042e9db956aa842b5d0f8a35aac56784cff7369a56970554f0430cdec4dbd67d904364a97bf5f53d5769dbf3ff8f46db4c5ce3822a8d1c39412d
+  data.tar.gz: 796fcd383e7df7621f21b63356c7634afb0a8a71447f0775801ec93297bd5754f48250fd28390cbfde412fe1a51d85329d8f0fd0f5358073810a98fa1ad08e2b

data/.rubocop-https---shopify-github-io-ruby-style-guide-rubocop-yml

@@ -671,7 +671,7 @@ Style/LineEndConcatenation:
 Style/MethodCallWithoutArgsParentheses:
   Enabled: true
 
-Style/MethodMissingSuper:
+Lint/MissingSuper:
   Enabled: true
 
 Style/MissingRespondToMissing:
@@ -964,7 +964,7 @@ Lint/UselessAccessModifier:
 Lint/UselessAssignment:
   Enabled: true
 
-Lint/UselessComparison:
+Lint/BinaryOperatorWithIdenticalOperands:
   Enabled: true
 
 Lint/UselessElseWithoutRescue:

data/CHANGELOG.md

@@ -1,6 +1,16 @@
 # CHANGELOG
+
+## 6.1.1
+- Emit messages when waiting for rate-limit to elapse for DNSimple and NS1 providers, so deployment does not timeout [BUGFIX]
+
+## 6.1.0
+- sort zone files [FEATURE]
+- CLI support for specifying zones for validate_authority [FEATURE]
+- retry failed lookup using another nameserver if unreachable [BUGFIX]
+- ignore records other than NS in authority section [BUGFIX]
+
 ## 6.0.1
-- add API rate limiting to DNSimple provider
+- add API rate limiting to DNSimple provider [FEATURE]
 
 ## 6.0.0
 - add `--all` option for `record-store diff` to compare ignored records too [FEATURE]

data/lib/record_store/cli.rb

@@ -220,12 +220,14 @@ module RecordStore
       end
     end
 
-    desc 'validate_authority', 'Validates that authoritative nameservers match the providers'
+    desc 'validate_authority [ZONE ...]', 'Validates that authoritative nameservers match the providers'
     option :verbose, desc: 'Include valid zones in output', aliases: '-v', type: :boolean, default: false
-    def validate_authority
+    def validate_authority(*zones)
       verbose = options.fetch('verbose')
 
       Zone.each do |name, zone|
+        next unless zones.empty? || zones.include?(name)
+
         authority = zone.fetch_authority
 
         delegation = Hash.new { |h, k| h[k] = [] }

data/lib/record_store/provider.rb

@@ -52,6 +52,10 @@ module RecordStore
         false
       end
 
+      def empty_non_terminal_over_wildcard?
+        true
+      end
+
       def build_zone(zone_name:, config:)
         zone = Zone.new(name: zone_name)
         zone.records = retrieve_current_records(zone: zone_name)
@@ -128,6 +132,37 @@ module RecordStore
 
         dns.getresource(zone_name, Resolv::DNS::Resource::IN::SOA).mname.to_s
       end
+
+      def retry_on_connection_errors(
+        max_timeouts: 5,
+        max_conn_resets: 5,
+        delay: 1,
+        backoff_multiplier: 2,
+        max_backoff: 10
+      )
+        waiter = BackoffWaiter.new(
+          "Waiting to retry after a connection reset",
+          initial_delay: delay,
+          multiplier: backoff_multiplier,
+          max_delay: max_backoff,
+        )
+
+        loop do
+          begin
+            return yield
+          rescue Net::OpenTimeout, Errno::ETIMEDOUT
+            raise if max_timeouts <= 0
+            max_timeouts -= 1
+
+            $stderr.puts("Retrying after a connection timeout")
+          rescue Errno::ECONNRESET
+            raise if max_conn_resets <= 0
+            max_conn_resets -= 1
+
+            waiter.wait
+          end
+        end
+      end
     end
   end
 end

data/lib/record_store/provider/dnsimple.rb

@@ -12,21 +12,29 @@ module RecordStore
         true
       end
 
+      def empty_non_terminal_over_wildcard?
+        false
+      end
+
       # returns an array of Record objects that match the records which exist in the provider
       def retrieve_current_records(zone:, stdout: $stdout)
-        session.zones.all_records(account_id, zone).data.map do |record|
-          begin
-            build_from_api(record, zone)
-          rescue StandardError
-            stdout.puts "Cannot build record: #{record}"
-            raise
-          end
-        end.compact
+        retry_on_connection_errors do
+          session.zones.all_records(account_id, zone).data.map do |record|
+            begin
+              build_from_api(record, zone)
+            rescue StandardError
+              stdout.puts "Cannot build record: #{record}"
+              raise
+            end
+          end.compact
+        end
       end
 
       # Returns an array of the zones managed by provider as strings
       def zones
-        session.zones.all_zones(account_id).data.map(&:name)
+        retry_on_connection_errors do
+          session.zones.all_zones(account_id).data.map(&:name)
+        end
       end
 
       private

data/lib/record_store/provider/dnsimple/patch_api_header.rb

@@ -1,15 +1,17 @@
+require_relative '../provider_utils/waiter'
+
 # Patch Dnsimple client method which retrieves headers for API rate limit dynamically
 module Dnsimple
   class Client
     def execute(method, path, data = nil, options = {})
       response = request(method, path, data, options)
-      rate_limit_sleep(response.headers["x-ratelimit-reset"].to_i, response.headers["x-ratelimit-remaining"].to_i)
+      rate_limit_sleep(response.headers['x-ratelimit-reset'].to_i, response.headers['x-ratelimit-remaining'].to_i)
 
       case response.code
       when 200..299
         response
       when 401
-        raise AuthenticationFailed, response["message"]
+        raise AuthenticationFailed, response['message']
       when 404
         raise NotFoundError, response
       else
@@ -22,9 +24,10 @@ module Dnsimple
     def rate_limit_sleep(rate_limit_reset, rate_limit_remaining)
       rate_limit_reset_in = [0, rate_limit_reset - Time.now.to_i].max
       rate_limit_periods = rate_limit_remaining + 1
-      wait_time = rate_limit_reset_in / rate_limit_periods.to_f
+      sleep_time = rate_limit_reset_in / rate_limit_periods.to_f
 
-      sleep(wait_time) if wait_time > 0
+      rate_limit = RateLimitWaiter.new('DNSimple')
+      rate_limit.wait(sleep_time)
     end
   end
 end

data/lib/record_store/provider/ns1.rb

@@ -60,14 +60,18 @@ module RecordStore
 
       # Returns an array of the zones managed by provider as strings
       def zones
-        client.zones.map { |zone| zone['zone'] }
+        retry_on_connection_errors do
+          client.zones.map { |zone| zone['zone'] }
+        end
       end
 
       private
 
       # Fetches simplified records for the provided zone
       def records_for_zone(zone)
-        client.zone(zone)['records']
+        retry_on_connection_errors do
+          client.zone(zone)['records']
+        end
       end
 
       # Creates a new record to the zone. It is expected this call modifies external state.

data/lib/record_store/provider/ns1/patch_api_header.rb

@@ -1,11 +1,22 @@
 require 'net/http'
+require_relative '../provider_utils/waiter'
 
 # Patch the method which retrieves headers for API rate limit dynamically
 module NS1::Transport
   class NetHttp
+    X_RATELIMIT_PERIOD = 'x-ratelimit-period'.freeze
+    X_RATELIMIT_REMAINING = 'x-ratelimit-remaining'.freeze
+
     def process_response(response)
-      sleep(response.to_hash["x-ratelimit-period"].first.to_i /
-        [1, response.to_hash["x-ratelimit-remaining"].first.to_i].max.to_f)
+      response_hash = response.to_hash
+
+      if response_hash.key?(X_RATELIMIT_PERIOD) && response_hash.key?(X_RATELIMIT_REMAINING)
+        sleep_time = response_hash[X_RATELIMIT_PERIOD].first.to_i /
+                     [1, response_hash[X_RATELIMIT_REMAINING].first.to_i].max.to_f
+
+        rate_limit = RateLimitWaiter.new('NS1')
+        rate_limit.wait(sleep_time)
+      end
 
       body = JSON.parse(response.body)
       case response

data/lib/record_store/provider/provider_utils/waiter.rb

@@ -0,0 +1,41 @@
+class Waiter
+  def initialize(message = nil)
+    @message = message || 'Waiting'
+  end
+
+  attr_accessor :message
+
+  def wait(sleep_time)
+    while sleep_time > 0
+      wait_time = [10, sleep_time].min
+      puts "#{message} (#{sleep_time}s left)" if wait_time > 1
+      sleep(wait_time)
+      sleep_time -= wait_time
+    end
+  end
+end
+
+class RateLimitWaiter < Waiter
+  def initialize(provider)
+    super("Waiting on #{provider} rate-limit")
+  end
+end
+
+class BackoffWaiter < Waiter
+  def initialize(message, initial_delay:, multiplier:, max_delay: nil)
+    super(message)
+
+    @initial_delay = @current_delay = initial_delay
+    @multiplier = multiplier
+    @max_delay = max_delay
+  end
+
+  def reset
+    @current_delay = @initial_delay
+  end
+
+  def wait
+    super(@current_delay)
+    @current_delay = [@current_delay * @multiplier, @max_delay].compact.min
+  end
+end

data/lib/record_store/record.rb

@@ -112,6 +112,10 @@ module RecordStore
       "[#{type}Record] #{fqdn} #{ttl} IN #{type} #{rdata_txt}"
     end
 
+    def wildcard?
+      fqdn.match?(/^\*\./)
+    end
+
     protected
 
     def validate_label_length

data/lib/record_store/version.rb

@@ -1,3 +1,3 @@
 module RecordStore
-  VERSION = '6.0.1'.freeze
+  VERSION = '6.2.1'.freeze
 end

data/lib/record_store/zone.rb

@@ -18,6 +18,7 @@ module RecordStore
     validate :validate_cname_records_dont_point_to_root
     validate :validate_same_ttl_for_records_sharing_fqdn_and_type
     validate :validate_provider_can_handle_zone_records
+    validate :validate_no_empty_non_terminal
     validate :validate_can_handle_alias_records
 
     class << self
@@ -129,14 +130,12 @@ module RecordStore
     )
 
     def fetch_authority(nameserver = ROOT_SERVERS.sample)
-      authority = Resolv::DNS.open(nameserver: nameserver) do |resolv|
-        resolv.fetch_resource(name, Resolv::DNS::Resource::IN::SOA) do |reply, name|
-          break if reply.answer.any?
+      authority = fetch_soa(nameserver) do |reply, _name|
+        break if reply.answer.any?
 
-          raise "No authority found (#{name})" unless reply.authority.any?
+        raise "No authority found (#{name})" unless reply.authority.any?
 
-          break extract_authority(reply)
-        end
+        break extract_authority(reply.authority)
       end
 
       # candidate DNS name is returned instead when NXDomain or other error
@@ -147,18 +146,39 @@ module RecordStore
 
     private
 
-    def extract_authority(reply)
-      authority = reply.authority.sample
+    def fetch_soa(nameserver)
+      Resolv::DNS.open(nameserver: nameserver) do |resolv|
+        resolv.fetch_resource(name, Resolv::DNS::Resource::IN::SOA) do |reply, name|
+          yield reply, name
+        end
+      end
+    end
+
+    def resolve_authority(authority)
+      nameservers = authority.map { |a| a.last.name.to_s }
+
+      begin
+        nameserver = nameservers.shift
+        fetch_authority(nameserver)
+      rescue Errno::EHOSTUNREACH => e
+        $stderr.puts "Warning: #{e} [host=#{nameserver}]"
+        raise if nameservers.empty?
+        retry
+      end
+    end
 
-      if unrooted_name.casecmp?(authority.first.to_s)
-        build_authority(reply.authority)
+    def extract_authority(authority)
+      if unrooted_name.casecmp?(authority.first.first.to_s)
+        build_authority(authority)
       else
-        fetch_authority(authority.last.name.to_s) || build_authority(reply.authority)
+        resolve_authority(authority) || build_authority(authority)
       end
     end
 
     def build_authority(authority)
-      authority.map.with_index do |(name, ttl, data), index|
+      ns = authority.select { |_name, _ttl, data| data.is_a?(Resolv::DNS::Resource::IN::NS) }
+
+      ns.map.with_index do |(name, ttl, data), index|
         Record::NS.new(ttl: ttl, fqdn: name.to_s, nsdname: data.name.to_s, record_id: index)
       end
     end
@@ -239,6 +259,28 @@ module RecordStore
       end
     end
 
+    def validate_no_empty_non_terminal
+      return unless config.empty_non_terminal_over_wildcard?
+
+      wildcards = records.select(&:wildcard?).map(&:fqdn).uniq
+      wildcards.each do |wildcard|
+        suffix = wildcard[1..-1]
+
+        terminal_records = records.map(&:fqdn)
+          .select { |record| record.match?(/^([a-zA-Z0-9-_]+\.[a-zA-Z0-9-_])#{Regexp.escape(suffix)}$/) }
+        next unless terminal_records.any?
+
+        intermediate_records = records.map(&:fqdn)
+          .select { |record| record.match?(/^([a-zA-Z0-9-_]+)#{Regexp.escape(suffix)}$/) }
+        terminal_records.each do |terminal_record|
+          non_terminal = terminal_record.partition('.').last
+          errors.add(:records, "found empty non-terminal #{non_terminal} "\
+                     "(caused by existing records #{wildcard} and #{terminal_record})")\
+            unless intermediate_records.include?(non_terminal)
+        end
+      end
+    end
+
     def validate_can_handle_alias_records
       return unless records.any? { |record| record.is_a?(Record::ALIAS) }
       return if config.supports_alias?

data/lib/record_store/zone/config.rb

@@ -24,6 +24,10 @@ module RecordStore
         end
       end
 
+      def empty_non_terminal_over_wildcard?
+        valid_providers? && providers.any? { |provider| Provider.const_get(provider).empty_non_terminal_over_wildcard? }
+      end
+
       def to_hash
         config_hash = {
           providers: providers,

data/lib/record_store/zone/yaml_definitions.rb

@@ -10,6 +10,7 @@ module RecordStore
       def defined
         @defined ||= yaml_files
           .map { |file| load_yml_zone_definition(file) }
+          .sort_by(&:unrooted_name)
           .index_by(&:unrooted_name)
       end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: record_store
 version: !ruby/object:Gem::Version
-  version: 6.0.1
+  version: 6.2.1
 platform: ruby
 authors:
 - Willem van Bergen
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-13 00:00:00.000000000 Z
+date: 2020-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -362,6 +362,7 @@ files:
 - lib/record_store/provider/ns1/client.rb
 - lib/record_store/provider/ns1/patch_api_header.rb
 - lib/record_store/provider/oracle_cloud_dns.rb
+- lib/record_store/provider/provider_utils/waiter.rb
 - lib/record_store/record.rb
 - lib/record_store/record/a.rb
 - lib/record_store/record/aaaa.rb