record_store 6.2.0 → 6.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7c401c0dafc01955edd90912d793cdd0f27bd698df86f18ed4132e3ae0a8d1bc
-  data.tar.gz: 261f5a6a0c768d417e247746fe95955393075c643322fb7a5c701d39edef218d
+  metadata.gz: 2537c52a9350b969e64d8fab36f1281813e3d1db528accd4ee0700cdc6217b80
+  data.tar.gz: 8561a6ea4212f180a47240d5f1234f43e1d1896e620f2ea0acb427e06b2306fc
 SHA512:
-  metadata.gz: 7160e11eec16c8ecea7fa7cf02ae7437a5093ec58f6f932a7e9e63feb85413ff5d9c28e0a90f8960ff76a94dd0bc7992e0712a0a6f40a2158062755079a16bac
-  data.tar.gz: 7d81f1268810c177bd785b8ab5610175d2e9b7f20db743d12996df2ab6f145c522a354a1f7e94e86e91b90c7a44b1b63bcbe1a92a1e93e0f1f5eefb13d76a659
+  metadata.gz: 81048f68e0c0710fa7d61bcc25f97583e0b73dd7a0debea7281a8e22403a46f8fb245607d1f9a74ee8a6412915184d4c1faf7a5178fd3a666b64217201715115
+  data.tar.gz: 88ed41158b07af0b2efed051d2794e54efacc90e36457b51aacb9e53e14307c15fcde259f5beda9b4df5056d7fd517fa52852e5b828d264d90433ccf62464977

data/.rubocop-https---shopify-github-io-ruby-style-guide-rubocop-yml

@@ -195,6 +195,7 @@ Style/FrozenStringLiteralComment:
   SupportedStyles:
     - always
     - never
+  SafeAutoCorrect: true
 
 Style/GlobalVars:
   AllowedVariables: []
@@ -264,7 +265,7 @@ Style/MethodCallWithArgsParentheses:
   - raise
   - puts
   Exclude:
-  - Gemfile
+  - '**/Gemfile'
 
 Style/MethodDefParentheses:
   EnforcedStyle: require_parentheses
@@ -577,6 +578,7 @@ Layout/BlockEndNewline:
 
 Style/CaseEquality:
   Enabled: true
+  AllowOnConstant: true
 
 Style/CharacterLiteral:
   Enabled: true
@@ -659,6 +661,9 @@ Style/IfWithSemicolon:
 Style/IdenticalConditionalBranches:
   Enabled: true
 
+Layout/IndentationStyle:
+  Enabled: true
+
 Style/InfiniteLoop:
   Enabled: true
 
@@ -803,9 +808,6 @@ Layout/SpaceInsideRangeLiteral:
 Style/SymbolLiteral:
   Enabled: true
 
-Layout/IndentationStyle:
-  Enabled: true
-
 Layout/TrailingWhitespace:
   Enabled: true
 
@@ -834,7 +836,7 @@ Style/ZeroLengthPredicate:
   Enabled: true
 
 Layout/HeredocIndentation:
-  EnforcedStyle: squiggly
+  Enabled: true
 
 Lint/AmbiguousOperator:
   Enabled: true
@@ -1015,3 +1017,6 @@ Style/ModuleFunction:
 
 Lint/OrderedMagicComments:
   Enabled: true
+
+Lint/DeprecatedOpenSSLConstant:
+  Enabled: true

data/.travis.yml

@@ -1,7 +1,7 @@
 cache: bundler
 language: ruby
 rvm:
-  - 2.4.3
+  - 2.6.4
 
 before_install:
   - gem update --system

data/CHANGELOG.md

@@ -1,5 +1,28 @@
 # CHANGELOG
 
+## 6.4.0
+
+Add support for injecting implicit records into a zone based on a pre-configured template. Brief overview of template keys:
+
+- `each_record`: the template will generate all `injected_records` in the template for each Record in the Zone that matches criteria listed here.
+- `conflict_with`: if any Record in the Zone matches the criteria listed here, the template will avoid injecting any of its generated implicit records into the zone
+- `injected_records`: the implicit records that will be generated by the template for every Zone Record that matches the `each_record` criteria
+
+## 6.3.1
+- Improve resiliency in the face of temporary provider outages [BUGFIX]
+
+## 6.3.0
+- Support for configurable number of threads via environment variable [FEATURE]
+
+## 6.2.1
+- Improved error reporting after timeouts [FEATURE]
+
+## 6.2.0
+- Add validation for non-terminal conflict with wildcard [FEATURE]
+
+## 6.1.2
+- Retry on connection errors [FEATURE]
+
 ## 6.1.1
 - Emit messages when waiting for rate-limit to elapse for DNSimple and NS1 providers, so deployment does not timeout [BUGFIX]
 

data/README.md

@@ -112,6 +112,17 @@ Changesets are how Record Store knows what updates to make. A `Changeset` is gen
 
 When running `bin/record-store apply`, a `Changeset` is generated by comparing the current records in a zone's YAML file with the records the provider defines. A zone's YAML file is always considered the primary source of truth.
 
+### Parallelism
+
+Record store attempts to parallelize some of the bulk zone fetching operations. It does so by spawning multiple threads (default: 10). This value can be configured by setting the RECORD_STORE_MAX_THREADS environment variable to a positive integer value.
+
+
+### Templates
+
+Record Store supports injecting implicit records into a zone based on templates. Each `Zone` can have a list of `Zone::Config::ImplicitRecordTemplate` in its `Zone::Config` that can define their own criteria for which records they use for generating implicit template records, which records conflict with the template-generated records, and what the template-generated records look like. These records are injected at the time of `Zone` initialization within `Zone#build_records`.
+
+Templates can help reduce zone file bloat where instead of defining many generic literals within the zone file for a given criteria zone record, these generic records can be implicitly injected into the `Zone` at the time of initialization based on information provided within the template.
+
 ----
 
 # Development

data/bin/console

@@ -6,6 +6,7 @@ require 'record_store'
 
 RecordStore.zones_path = File.expand_path('../../dev/zones', __FILE__)
 RecordStore.config_path = File.expand_path('../../dev/config.yml', __FILE__)
+RecordStore.implicit_records_templates_path = File.expand_path('../../dev/templates/implicit_records', __FILE__)
 
 require 'pry'
 binding.pry(RecordStore) # rubocop:disable Lint/Debugger

data/dev.yml

@@ -2,7 +2,7 @@
 name: recordstore
 
 up:
-  - ruby: 2.5.0
+  - ruby: 2.6.4
   - bundler
 
 commands:

data/lib/record_store.rb

@@ -28,6 +28,7 @@ require 'record_store/zone/yaml_definitions'
 require 'record_store/zone'
 require 'record_store/zone/config'
 require 'record_store/zone/config/ignore_pattern'
+require 'record_store/zone/config/implicit_record_template'
 require 'record_store/changeset'
 require 'record_store/provider'
 require 'record_store/provider/dynect'
@@ -59,6 +60,13 @@ module RecordStore
       @config_path ||= File.expand_path('config.yml', Dir.pwd)
     end
 
+    def implicit_records_templates_path
+      @implicit_records_templates_path ||= Pathname.new(
+        File.expand_path(config.fetch('implicit_records_templates_path'),
+        File.dirname(config_path)),
+      ).realpath.to_s
+    end
+
     def config_path=(config_path)
       @config = @zones_path = @secrets_path = nil
       @config_path = config_path

data/lib/record_store/provider.rb

@@ -2,6 +2,9 @@ require 'resolv'
 
 module RecordStore
   class Provider
+    class Error < StandardError; end
+    class UnparseableBodyError < Error; end
+
     class << self
       def provider_for(object)
         ns_server =
@@ -136,12 +139,13 @@ module RecordStore
       def retry_on_connection_errors(
         max_timeouts: 5,
         max_conn_resets: 5,
+        max_retries: 5,
         delay: 1,
         backoff_multiplier: 2,
         max_backoff: 10
       )
         waiter = BackoffWaiter.new(
-          "Waiting to retry after a connection reset",
+          'Waiting to retry after a connection reset',
           initial_delay: delay,
           multiplier: backoff_multiplier,
           max_delay: max_backoff,
@@ -150,11 +154,16 @@ module RecordStore
         loop do
           begin
             return yield
-          rescue Net::OpenTimeout
+          rescue UnparseableBodyError
+            raise if max_retries <= 0
+            max_retries -= 1
+
+            waiter.wait(message: 'Waiting to retry after receiving an unparseable response')
+          rescue Net::OpenTimeout, Errno::ETIMEDOUT
             raise if max_timeouts <= 0
             max_timeouts -= 1
 
-            $stderr.puts("Retrying after a connection timeout")
+            $stderr.puts('Retrying after a connection timeout')
           rescue Errno::ECONNRESET
             raise if max_conn_resets <= 0
             max_conn_resets -= 1

data/lib/record_store/provider/ns1.rb

@@ -3,8 +3,6 @@ require_relative 'ns1/patch_api_header'
 
 module RecordStore
   class Provider::NS1 < Provider
-    class Error < StandardError; end
-
     class ApiAnswer
       class << self
         def from_full_api_answer(type:, record_id:, answer:)
@@ -180,7 +178,7 @@ module RecordStore
         unless updated
           error = +'while trying to update a record, could not find answer with fqdn: '
           error << "#{record.fqdn}, type; #{record.type}, id: #{id}"
-          raise Error, error
+          raise RecordStore::Provider::Error, error
         end
 
         client.modify_record(

data/lib/record_store/provider/ns1/client.rb

@@ -1,46 +1,59 @@
+require 'net/http'
 require 'ns1'
 
 module RecordStore
   class Provider::NS1 < Provider
-    class Error < StandardError; end
-
     class Client < ::NS1::Client
       def initialize(api_key:)
         super(api_key)
       end
 
       def zones
-        super
+        zones = super
+        raise_if_error!(zones)
+        zones
       end
 
       def zone(name)
-        super(name)
+        zone = super(name)
+        raise_if_error!(zone)
+        zone
       end
 
       def record(zone:, fqdn:, type:, must_exist: false)
         result = super(zone, fqdn, type)
-        raise(Error, result.to_s) if must_exist && result.is_a?(NS1::Response::Error)
+        raise_if_error!(result) if must_exist
         return nil if result.is_a?(NS1::Response::Error)
         result
       end
 
       def create_record(zone:, fqdn:, type:, params:)
         result = super(zone, fqdn, type, params)
-        raise(Error, result.to_s) if result.is_a?(NS1::Response::Error)
+        raise_if_error!(result)
         nil
       end
 
       def modify_record(zone:, fqdn:, type:, params:)
         result = super(zone, fqdn, type, params)
-        raise(Error, result.to_s) if result.is_a?(NS1::Response::Error)
+        raise_if_error!(result)
         nil
       end
 
       def delete_record(zone:, fqdn:, type:)
         result = super(zone, fqdn, type)
-        raise(Error, result.to_s) if result.is_a?(NS1::Response::Error)
+        raise_if_error!(result)
         nil
       end
+
+      private
+
+      def raise_if_error!(result)
+        return unless result.is_a?(NS1::Response::Error)
+        if result.is_a?(NS1::Response::UnparsableBodyError)
+          raise RecordStore::Provider::UnparseableBodyError, result.to_s
+        end
+        raise RecordStore::Provider::Error, result.to_s
+      end
     end
   end
 end

data/lib/record_store/provider/ns1/patch_api_header.rb

@@ -1,6 +1,13 @@
 require 'net/http'
 require_relative '../provider_utils/waiter'
 
+class NS1::Response::UnparsableBodyError < NS1::Response::Error
+  def initialize(status)
+    @status = status
+    super({}, status)
+  end
+end
+
 # Patch the method which retrieves headers for API rate limit dynamically
 module NS1::Transport
   class NetHttp
@@ -18,15 +25,17 @@ module NS1::Transport
         rate_limit.wait(sleep_time)
       end
 
-      body = JSON.parse(response.body)
-      case response
-      when Net::HTTPOK
-        NS1::Response::Success.new(body, response.code.to_i)
-      else
-        NS1::Response::Error.new(body, response.code.to_i)
+      begin
+        body = JSON.parse(response.body)
+        case response
+        when Net::HTTPOK
+          NS1::Response::Success.new(body, response.code.to_i)
+        else
+          NS1::Response::Error.new(body, response.code.to_i)
+        end
+      rescue JSON::ParserError
+        NS1::Response::UnparsableBodyError.new(response.code.to_i)
       end
-    rescue JSON::ParserError
-      raise NS1::Transport::ResponseParseError
     end
   end
 end

data/lib/record_store/provider/provider_utils/waiter.rb

@@ -5,7 +5,7 @@ class Waiter
 
   attr_accessor :message
 
-  def wait(sleep_time)
+  def wait(sleep_time, message: @message)
     while sleep_time > 0
       wait_time = [10, sleep_time].min
       puts "#{message} (#{sleep_time}s left)" if wait_time > 1
@@ -34,8 +34,8 @@ class BackoffWaiter < Waiter
     @current_delay = @initial_delay
   end
 
-  def wait
-    super(@current_delay)
+  def wait(message: @message)
+    super(@current_delay, message: message)
     @current_delay = [@current_delay * @multiplier, @max_delay].compact.min
   end
 end

data/lib/record_store/version.rb

@@ -1,3 +1,3 @@
 module RecordStore
-  VERSION = '6.2.0'.freeze
+  VERSION = '6.4.1'.freeze
 end

data/lib/record_store/zone.rb

@@ -45,13 +45,18 @@ module RecordStore
         end
       end
 
-      MAX_PARALLEL_THREADS = 10
+      DEFAULT_MAX_PARALLEL_THREADS = 10
+
+      def max_parallel_threads
+        (ENV['RECORD_STORE_MAX_THREADS'] || DEFAULT_MAX_PARALLEL_THREADS).to_i
+      end
+
       def modified(verbose: false) # rubocop:disable Lint/UnusedMethodArgument
         modified_zones = []
         mutex = Mutex.new
         zones = all
 
-        (1..MAX_PARALLEL_THREADS).map do
+        (1..max_parallel_threads).map do
           Thread.new do
             current_zone = nil
             while zones.any?
@@ -184,7 +189,13 @@ module RecordStore
     end
 
     def build_records(records)
-      records.map { |record| Record.build_from_yaml_definition(record) }
+      all_records = records.map { |record| Record.build_from_yaml_definition(record) }
+
+      config.implicit_records_templates.each do |template|
+        all_records.push(*template.generate_records_to_inject(current_records: all_records))
+      end
+
+      all_records
     end
 
     def validate_records
@@ -267,11 +278,11 @@ module RecordStore
         suffix = wildcard[1..-1]
 
         terminal_records = records.map(&:fqdn)
-          .select { |record| record.match?(/^([a-zA-Z0-9-_]+\.[a-zA-Z0-9-_])#{Regexp.escape(suffix)}$/) }
+          .select { |record| record.match?(/^([a-zA-Z0-9\-_]+\.[a-zA-Z0-9\-_])#{Regexp.escape(suffix)}$/) }
         next unless terminal_records.any?
 
         intermediate_records = records.map(&:fqdn)
-          .select { |record| record.match?(/^([a-zA-Z0-9-_]+)#{Regexp.escape(suffix)}$/) }
+          .select { |record| record.match?(/^([a-zA-Z0-9\-_]+)#{Regexp.escape(suffix)}$/) }
         terminal_records.each do |terminal_record|
           non_terminal = terminal_record.partition('.').last
           errors.add(:records, "found empty non-terminal #{non_terminal} "\

data/lib/record_store/zone/config.rb

@@ -3,15 +3,17 @@ module RecordStore
     class Config
       include ActiveModel::Validations
 
-      attr_reader :ignore_patterns, :providers, :supports_alias
+      attr_reader :ignore_patterns, :providers, :supports_alias, :implicit_records_templates
 
       validate :validate_zone_config
 
-      def initialize(ignore_patterns: [], providers: nil, supports_alias: nil)
+      def initialize(ignore_patterns: [], providers: nil, supports_alias: nil, implicit_records_templates: [])
         @ignore_patterns = ignore_patterns.map do |ignore_pattern|
           Zone::Config::IgnorePattern.new(ignore_pattern)
         end
-
+        @implicit_records_templates = implicit_records_templates.map do |filename|
+          Zone::Config::ImplicitRecordTemplate.from_file(filename: filename)
+        end
         @providers = providers
         @supports_alias = supports_alias
       end

data/lib/record_store/zone/config/implicit_record_template.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+module RecordStore
+  class Zone
+    class Config
+      class TemplateContext
+        def self.build(record:, current_records:)
+          new(record: record, current_records: current_records)
+        end
+
+        def initialize(record:, current_records:)
+          @record = record
+          @current_records = current_records
+        end
+
+        def fetch_binding
+          binding
+        end
+
+        private
+
+        attr_reader :record, :current_records
+      end
+
+      class ImplicitRecordTemplate
+        class << self
+          def from_file(filename:)
+            filepath = template_filepath_for(filename: filename)
+            template_file = File.read(filepath)
+            filters_for_records_to_template = YAML.load(template_file).deep_symbolize_keys[:each_record]
+
+            new(template: ERB.new(template_file), filters_for_records_to_template: filters_for_records_to_template)
+          end
+
+          private
+
+          def template_filepath_for(filename:)
+            "#{RecordStore.implicit_records_templates_path}/#{filename}"
+          end
+        end
+
+        def initialize(template:, filters_for_records_to_template:)
+          @template = template
+          @filters_for_records_to_template = filters_for_records_to_template
+        end
+
+        def generate_records_to_inject(current_records:)
+          current_records
+            .select { |record| should_template?(record: record) }
+            .map { |record| template_record_for(record: record, current_records: current_records) }
+            .each_with_object([]) do |template_records, records_to_inject|
+              next unless should_inject?(
+                template_records: template_records,
+                current_records: current_records + records_to_inject
+              )
+
+              records_to_inject.push(
+                *template_records.fetch(:injected_records, []).map { |r_yml| Record.build_from_yaml_definition(r_yml) }
+              )
+            end
+        end
+
+        private
+
+        attr_reader :template, :filters_for_records_to_template
+
+        def should_inject?(template_records:, current_records:)
+          current_records.none? do |record|
+            template_records[:conflict_with].any? do |filter|
+              record_match?(record: record, filter: filter)
+            end
+          end
+        end
+
+        def should_template?(record:)
+          filters_for_records_to_template.any? { |filter| record_match?(record: record, filter: filter) }
+        end
+
+        def record_match?(record:, filter:)
+          filter.all? do |key, value|
+            record.public_send(key) == value
+          end
+        end
+
+        def template_record_for(record:, current_records:)
+          context = TemplateContext.build(record: record, current_records: current_records)
+
+          YAML.load(
+            template.result(context.fetch_binding)
+          ).deep_symbolize_keys
+        end
+      end
+    end
+  end
+end

data/record_store.gemspec

@@ -48,6 +48,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'vcr'
   spec.add_development_dependency 'pry'
   spec.add_development_dependency 'webmock'
-  spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'rubocop', '~> 1.0.0'
   spec.add_development_dependency 'minitest-focus'
 end

data/template/config.yml

@@ -1,2 +1,3 @@
 zones_path: zones/
 secrets_path: secrets.json
+implicit_records_templates_path: templates/implicit_records/

data/template/templates/implicit_records/implicit_example.yml.erb

@@ -0,0 +1,13 @@
+each_record:
+- type: A
+  fqdn: abc.123.com.
+- type: TXT
+conflict_with:
+- type: TXT
+  fqdn: <%= record.fqdn %>.more.domain.com.
+injected_records:
+- type: CNAME
+  ttl: 3600
+  fqdn: <%= record.fqdn %>.added.information.com
+  cname: <%= record.fqdn %>.more.added.information.com.
+

data/template/zones/dynect.example.com.yml

@@ -5,6 +5,8 @@ dynect.example.com:
     ignore_patterns:
     - type: NS
       fqdn: dynect.example.com.
+    implicit_record_templates:
+    - implicit_example.yml.erb
   records:
   - type: A
     fqdn: a-record.dynect.example.com.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: record_store
 version: !ruby/object:Gem::Version
-  version: 6.2.0
+  version: 6.4.1
 platform: ruby
 authors:
 - Willem van Bergen
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-08-18 00:00:00.000000000 Z
+date: 2020-12-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -301,16 +301,16 @@ dependencies:
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: minitest-focus
   requirement: !ruby/object:Gem::Requirement
@@ -380,6 +380,7 @@ files:
 - lib/record_store/zone.rb
 - lib/record_store/zone/config.rb
 - lib/record_store/zone/config/ignore_pattern.rb
+- lib/record_store/zone/config/implicit_record_template.rb
 - lib/record_store/zone/yaml_definitions.rb
 - record_store.gemspec
 - shipit.rubygems.yml
@@ -389,6 +390,7 @@ files:
 - template/bin/test
 - template/config.yml
 - template/secrets.json
+- template/templates/implicit_records/implicit_example.yml.erb
 - template/zones/dnsimple.example.com.yml
 - template/zones/dnsimple.example.com/A__a-record.yml
 - template/zones/dnsimple.example.com/TXT__marco.yml