record_store 6.1.1 → 6.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f7a5da9483eaf8ce4eb82e8d974fefe82f7f481cde945a45ec6ae204b02524dc
-  data.tar.gz: 51e681688b6cc80c851f801ad00bb07ccf473ea2fb0b1c8a8e722d3cbc2ec196
+  metadata.gz: f6875c27f0cf86510b733a58f668d44560eeb40e7c8b6c9438183fee175bc514
+  data.tar.gz: 3599cc9903a5afe46d544f8aeef349961fdc2d95011973325bbdcc75972c56c5
 SHA512:
-  metadata.gz: 502acb53fcef150d5cb5442414c47552fc199efe66012e85d30745ae25c8af326064283c855ba12f4cead5b6f7b09f5a97c0b3bd6a4de1a010bba15664954335
-  data.tar.gz: 491ee9ff80ccb35ceb7ffbe70776948cd459467df8c213917c08ac6830e249585dbd993849a44e413bd6fc8e38589601eab27601cf3061c9b51abd09a94898a0
+  metadata.gz: 9a6def9342919457cc71fdda4691c92310450dfc28a476eee6d353b73798b8d1b612b4775982704b0c44b2e657149a02f0e96330796b03d591619ad5930509ae
+  data.tar.gz: 42183ae0ecd31e27a2d40977ecdda6cb51c0368ff24ac3a2de4411ab016809c5551ee3faeb50eb53503be059ece4bc41889e6c73d3e1f0ab53e7ab9290563628

data/.rubocop-https---shopify-github-io-ruby-style-guide-rubocop-yml

@@ -671,7 +671,7 @@ Style/LineEndConcatenation:
 Style/MethodCallWithoutArgsParentheses:
   Enabled: true
 
-Style/MethodMissingSuper:
+Lint/MissingSuper:
   Enabled: true
 
 Style/MissingRespondToMissing:
@@ -964,7 +964,7 @@ Lint/UselessAccessModifier:
 Lint/UselessAssignment:
   Enabled: true
 
-Lint/UselessComparison:
+Lint/BinaryOperatorWithIdenticalOperands:
   Enabled: true
 
 Lint/UselessElseWithoutRescue:

data/.travis.yml

@@ -1,7 +1,7 @@
 cache: bundler
 language: ruby
 rvm:
-  - 2.4.3
+  - 2.6.4
 
 before_install:
   - gem update --system

data/CHANGELOG.md

@@ -1,5 +1,20 @@
 # CHANGELOG
 
+## 6.3.1
+- Improve resiliency in the face of temporary provider outages [BUGFIX]
+
+## 6.3.0
+- Support for configurable number of threads via environment variable [FEATURE]
+
+## 6.2.1
+- Improved error reporting after timeouts [FEATURE]
+
+## 6.2.0
+- Add validation for non-terminal conflict with wildcard [FEATURE]
+
+## 6.1.2
+- Retry on connection errors [FEATURE]
+
 ## 6.1.1
 - Emit messages when waiting for rate-limit to elapse for DNSimple and NS1 providers, so deployment does not timeout [BUGFIX]
 

data/README.md

@@ -112,6 +112,10 @@ Changesets are how Record Store knows what updates to make. A `Changeset` is gen
 
 When running `bin/record-store apply`, a `Changeset` is generated by comparing the current records in a zone's YAML file with the records the provider defines. A zone's YAML file is always considered the primary source of truth.
 
+### Parallelism
+
+Record store attempts to parallelize some of the bulk zone fetching operations. It does so by spawning multiple threads (default: 10). This value can be configured by setting the RECORD_STORE_MAX_THREADS environment variable to a positive integer value.
+
 ----
 
 # Development

data/dev.yml

@@ -2,7 +2,7 @@
 name: recordstore
 
 up:
-  - ruby: 2.5.0
+  - ruby: 2.6.4
   - bundler
 
 commands:

data/lib/record_store/provider.rb

@@ -2,6 +2,9 @@ require 'resolv'
 
 module RecordStore
   class Provider
+    class Error < StandardError; end
+    class UnparseableBodyError < Error; end
+
     class << self
       def provider_for(object)
         ns_server =
@@ -52,6 +55,10 @@ module RecordStore
         false
       end
 
+      def empty_non_terminal_over_wildcard?
+        true
+      end
+
       def build_zone(zone_name:, config:)
         zone = Zone.new(name: zone_name)
         zone.records = retrieve_current_records(zone: zone_name)
@@ -128,6 +135,43 @@ module RecordStore
 
         dns.getresource(zone_name, Resolv::DNS::Resource::IN::SOA).mname.to_s
       end
+
+      def retry_on_connection_errors(
+        max_timeouts: 5,
+        max_conn_resets: 5,
+        max_retries: 5,
+        delay: 1,
+        backoff_multiplier: 2,
+        max_backoff: 10
+      )
+        waiter = BackoffWaiter.new(
+          'Waiting to retry after a connection reset',
+          initial_delay: delay,
+          multiplier: backoff_multiplier,
+          max_delay: max_backoff,
+        )
+
+        loop do
+          begin
+            return yield
+          rescue UnparseableBodyError
+            raise if max_retries <= 0
+            max_retries -= 1
+
+            waiter.wait(message: 'Waiting to retry after receiving an unparseable response')
+          rescue Net::OpenTimeout, Errno::ETIMEDOUT
+            raise if max_timeouts <= 0
+            max_timeouts -= 1
+
+            $stderr.puts('Retrying after a connection timeout')
+          rescue Errno::ECONNRESET
+            raise if max_conn_resets <= 0
+            max_conn_resets -= 1
+
+            waiter.wait
+          end
+        end
+      end
     end
   end
 end

data/lib/record_store/provider/dnsimple.rb

@@ -12,21 +12,29 @@ module RecordStore
         true
       end
 
+      def empty_non_terminal_over_wildcard?
+        false
+      end
+
       # returns an array of Record objects that match the records which exist in the provider
       def retrieve_current_records(zone:, stdout: $stdout)
-        session.zones.all_records(account_id, zone).data.map do |record|
-          begin
-            build_from_api(record, zone)
-          rescue StandardError
-            stdout.puts "Cannot build record: #{record}"
-            raise
-          end
-        end.compact
+        retry_on_connection_errors do
+          session.zones.all_records(account_id, zone).data.map do |record|
+            begin
+              build_from_api(record, zone)
+            rescue StandardError
+              stdout.puts "Cannot build record: #{record}"
+              raise
+            end
+          end.compact
+        end
       end
 
       # Returns an array of the zones managed by provider as strings
       def zones
-        session.zones.all_zones(account_id).data.map(&:name)
+        retry_on_connection_errors do
+          session.zones.all_zones(account_id).data.map(&:name)
+        end
       end
 
       private

data/lib/record_store/provider/dnsimple/patch_api_header.rb

@@ -1,4 +1,4 @@
-require_relative '../provider_utils/rate_limit'
+require_relative '../provider_utils/waiter'
 
 # Patch Dnsimple client method which retrieves headers for API rate limit dynamically
 module Dnsimple
@@ -26,8 +26,8 @@ module Dnsimple
       rate_limit_periods = rate_limit_remaining + 1
       sleep_time = rate_limit_reset_in / rate_limit_periods.to_f
 
-      rate_limit = RateLimit.new('DNSimple')
-      rate_limit.sleep_for(sleep_time)
+      rate_limit = RateLimitWaiter.new('DNSimple')
+      rate_limit.wait(sleep_time)
     end
   end
 end

data/lib/record_store/provider/ns1.rb

@@ -3,8 +3,6 @@ require_relative 'ns1/patch_api_header'
 
 module RecordStore
   class Provider::NS1 < Provider
-    class Error < StandardError; end
-
     class ApiAnswer
       class << self
         def from_full_api_answer(type:, record_id:, answer:)
@@ -60,14 +58,18 @@ module RecordStore
 
       # Returns an array of the zones managed by provider as strings
       def zones
-        client.zones.map { |zone| zone['zone'] }
+        retry_on_connection_errors do
+          client.zones.map { |zone| zone['zone'] }
+        end
       end
 
       private
 
       # Fetches simplified records for the provided zone
       def records_for_zone(zone)
-        client.zone(zone)['records']
+        retry_on_connection_errors do
+          client.zone(zone)['records']
+        end
       end
 
       # Creates a new record to the zone. It is expected this call modifies external state.
@@ -176,7 +178,7 @@ module RecordStore
         unless updated
           error = +'while trying to update a record, could not find answer with fqdn: '
           error << "#{record.fqdn}, type; #{record.type}, id: #{id}"
-          raise Error, error
+          raise RecordStore::Provider::Error, error
         end
 
         client.modify_record(

data/lib/record_store/provider/ns1/client.rb

@@ -1,46 +1,59 @@
+require 'net/http'
 require 'ns1'
 
 module RecordStore
   class Provider::NS1 < Provider
-    class Error < StandardError; end
-
     class Client < ::NS1::Client
       def initialize(api_key:)
         super(api_key)
       end
 
       def zones
-        super
+        zones = super
+        raise_if_error!(zones)
+        zones
       end
 
       def zone(name)
-        super(name)
+        zone = super(name)
+        raise_if_error!(zone)
+        zone
       end
 
       def record(zone:, fqdn:, type:, must_exist: false)
         result = super(zone, fqdn, type)
-        raise(Error, result.to_s) if must_exist && result.is_a?(NS1::Response::Error)
+        raise_if_error!(result) if must_exist
         return nil if result.is_a?(NS1::Response::Error)
         result
       end
 
       def create_record(zone:, fqdn:, type:, params:)
         result = super(zone, fqdn, type, params)
-        raise(Error, result.to_s) if result.is_a?(NS1::Response::Error)
+        raise_if_error!(result)
         nil
       end
 
       def modify_record(zone:, fqdn:, type:, params:)
         result = super(zone, fqdn, type, params)
-        raise(Error, result.to_s) if result.is_a?(NS1::Response::Error)
+        raise_if_error!(result)
         nil
       end
 
       def delete_record(zone:, fqdn:, type:)
         result = super(zone, fqdn, type)
-        raise(Error, result.to_s) if result.is_a?(NS1::Response::Error)
+        raise_if_error!(result)
         nil
       end
+
+      private
+
+      def raise_if_error!(result)
+        return unless result.is_a?(NS1::Response::Error)
+        if result.is_a?(NS1::Response::UnparsableBodyError)
+          raise RecordStore::Provider::UnparseableBodyError, result.to_s
+        end
+        raise RecordStore::Provider::Error, result.to_s
+      end
     end
   end
 end

data/lib/record_store/provider/ns1/patch_api_header.rb

@@ -1,25 +1,41 @@
 require 'net/http'
-require_relative '../provider_utils/rate_limit'
+require_relative '../provider_utils/waiter'
+
+class NS1::Response::UnparsableBodyError < NS1::Response::Error
+  def initialize(status)
+    @status = status
+    super({}, status)
+  end
+end
 
 # Patch the method which retrieves headers for API rate limit dynamically
 module NS1::Transport
   class NetHttp
+    X_RATELIMIT_PERIOD = 'x-ratelimit-period'.freeze
+    X_RATELIMIT_REMAINING = 'x-ratelimit-remaining'.freeze
+
     def process_response(response)
-      sleep_time = response.to_hash['x-ratelimit-period'].first.to_i /
-                   [1, response.to_hash['x-ratelimit-remaining'].first.to_i].max.to_f
+      response_hash = response.to_hash
+
+      if response_hash.key?(X_RATELIMIT_PERIOD) && response_hash.key?(X_RATELIMIT_REMAINING)
+        sleep_time = response_hash[X_RATELIMIT_PERIOD].first.to_i /
+                     [1, response_hash[X_RATELIMIT_REMAINING].first.to_i].max.to_f
 
-      rate_limit = RateLimit.new('NS1')
-      rate_limit.sleep_for(sleep_time)
+        rate_limit = RateLimitWaiter.new('NS1')
+        rate_limit.wait(sleep_time)
+      end
 
-      body = JSON.parse(response.body)
-      case response
-      when Net::HTTPOK
-        NS1::Response::Success.new(body, response.code.to_i)
-      else
-        NS1::Response::Error.new(body, response.code.to_i)
+      begin
+        body = JSON.parse(response.body)
+        case response
+        when Net::HTTPOK
+          NS1::Response::Success.new(body, response.code.to_i)
+        else
+          NS1::Response::Error.new(body, response.code.to_i)
+        end
+      rescue JSON::ParserError
+        NS1::Response::UnparsableBodyError.new(response.code.to_i)
       end
-    rescue JSON::ParserError
-      raise NS1::Transport::ResponseParseError
     end
   end
 end

data/lib/record_store/provider/provider_utils/waiter.rb

@@ -0,0 +1,41 @@
+class Waiter
+  def initialize(message = nil)
+    @message = message || 'Waiting'
+  end
+
+  attr_accessor :message
+
+  def wait(sleep_time, message: @message)
+    while sleep_time > 0
+      wait_time = [10, sleep_time].min
+      puts "#{message} (#{sleep_time}s left)" if wait_time > 1
+      sleep(wait_time)
+      sleep_time -= wait_time
+    end
+  end
+end
+
+class RateLimitWaiter < Waiter
+  def initialize(provider)
+    super("Waiting on #{provider} rate-limit")
+  end
+end
+
+class BackoffWaiter < Waiter
+  def initialize(message, initial_delay:, multiplier:, max_delay: nil)
+    super(message)
+
+    @initial_delay = @current_delay = initial_delay
+    @multiplier = multiplier
+    @max_delay = max_delay
+  end
+
+  def reset
+    @current_delay = @initial_delay
+  end
+
+  def wait(message: @message)
+    super(@current_delay, message: message)
+    @current_delay = [@current_delay * @multiplier, @max_delay].compact.min
+  end
+end

data/lib/record_store/record.rb

@@ -112,6 +112,10 @@ module RecordStore
       "[#{type}Record] #{fqdn} #{ttl} IN #{type} #{rdata_txt}"
     end
 
+    def wildcard?
+      fqdn.match?(/^\*\./)
+    end
+
     protected
 
     def validate_label_length

data/lib/record_store/version.rb

@@ -1,3 +1,3 @@
 module RecordStore
-  VERSION = '6.1.1'.freeze
+  VERSION = '6.3.1'.freeze
 end

data/lib/record_store/zone.rb

@@ -18,6 +18,7 @@ module RecordStore
     validate :validate_cname_records_dont_point_to_root
     validate :validate_same_ttl_for_records_sharing_fqdn_and_type
     validate :validate_provider_can_handle_zone_records
+    validate :validate_no_empty_non_terminal
     validate :validate_can_handle_alias_records
 
     class << self
@@ -44,13 +45,18 @@ module RecordStore
         end
       end
 
-      MAX_PARALLEL_THREADS = 10
+      DEFAULT_MAX_PARALLEL_THREADS = 10
+
+      def max_parallel_threads
+        (ENV['RECORD_STORE_MAX_THREADS'] || DEFAULT_MAX_PARALLEL_THREADS).to_i
+      end
+
       def modified(verbose: false) # rubocop:disable Lint/UnusedMethodArgument
         modified_zones = []
         mutex = Mutex.new
         zones = all
 
-        (1..MAX_PARALLEL_THREADS).map do
+        (1..max_parallel_threads).map do
           Thread.new do
             current_zone = nil
             while zones.any?
@@ -258,6 +264,28 @@ module RecordStore
       end
     end
 
+    def validate_no_empty_non_terminal
+      return unless config.empty_non_terminal_over_wildcard?
+
+      wildcards = records.select(&:wildcard?).map(&:fqdn).uniq
+      wildcards.each do |wildcard|
+        suffix = wildcard[1..-1]
+
+        terminal_records = records.map(&:fqdn)
+          .select { |record| record.match?(/^([a-zA-Z0-9\-_]+\.[a-zA-Z0-9\-_])#{Regexp.escape(suffix)}$/) }
+        next unless terminal_records.any?
+
+        intermediate_records = records.map(&:fqdn)
+          .select { |record| record.match?(/^([a-zA-Z0-9\-_]+)#{Regexp.escape(suffix)}$/) }
+        terminal_records.each do |terminal_record|
+          non_terminal = terminal_record.partition('.').last
+          errors.add(:records, "found empty non-terminal #{non_terminal} "\
+                     "(caused by existing records #{wildcard} and #{terminal_record})")\
+            unless intermediate_records.include?(non_terminal)
+        end
+      end
+    end
+
     def validate_can_handle_alias_records
       return unless records.any? { |record| record.is_a?(Record::ALIAS) }
       return if config.supports_alias?

data/lib/record_store/zone/config.rb

@@ -24,6 +24,10 @@ module RecordStore
         end
       end
 
+      def empty_non_terminal_over_wildcard?
+        valid_providers? && providers.any? { |provider| Provider.const_get(provider).empty_non_terminal_over_wildcard? }
+      end
+
       def to_hash
         config_hash = {
           providers: providers,

data/record_store.gemspec

@@ -48,6 +48,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'vcr'
   spec.add_development_dependency 'pry'
   spec.add_development_dependency 'webmock'
-  spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'rubocop', '0.89.1'
   spec.add_development_dependency 'minitest-focus'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: record_store
 version: !ruby/object:Gem::Version
-  version: 6.1.1
+  version: 6.3.1
 platform: ruby
 authors:
 - Willem van Bergen
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-11 00:00:00.000000000 Z
+date: 2020-10-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -301,16 +301,16 @@ dependencies:
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.89.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.89.1
 - !ruby/object:Gem::Dependency
   name: minitest-focus
   requirement: !ruby/object:Gem::Requirement
@@ -362,7 +362,7 @@ files:
 - lib/record_store/provider/ns1/client.rb
 - lib/record_store/provider/ns1/patch_api_header.rb
 - lib/record_store/provider/oracle_cloud_dns.rb
-- lib/record_store/provider/provider_utils/rate_limit.rb
+- lib/record_store/provider/provider_utils/waiter.rb
 - lib/record_store/record.rb
 - lib/record_store/record/a.rb
 - lib/record_store/record/aaaa.rb

data/lib/record_store/provider/provider_utils/rate_limit.rb

@@ -1,14 +0,0 @@
-class RateLimit
-  def initialize(provider)
-    @provider = provider
-  end
-
-  def sleep_for(sleep_time)
-    while sleep_time > 0
-      wait = [10, sleep_time].min
-      puts "Waiting on #{@provider} rate-limit" if wait > 1
-      sleep(wait)
-      sleep_time -= wait
-    end
-  end
-end