record_store 6.3.1 → 6.7.1

data/lib/record_store/provider/ns1.rb

@@ -13,6 +13,9 @@ module RecordStore
           rrdata_fields = case type
           when 'SPF', 'TXT'
             [answer]
+          when 'SRV'
+            priority, weight, port, host = answer.split
+            [priority.to_i, weight.to_i, port.to_i, Record.ensure_ends_with_dot(host)]
           else
             answer.split
           end
@@ -50,7 +53,7 @@ module RecordStore
       # Downloads all the records from the provider.
       #
       # Returns: an array of `Record` for each record in the provider's zone
-      def retrieve_current_records(zone:, stdout: $stdout) # rubocop:disable Lint/UnusedMethodArgument
+      def retrieve_current_records(zone:, stdout: $stdout)
         records_for_zone(zone)
           .flat_map { |short_record| build_from_api(short_record) }
           .compact
@@ -84,7 +87,7 @@ module RecordStore
         existing_record = client.record(
           zone: zone,
           fqdn: record_fqdn,
-          type: record.type
+          type: record.type,
         )
 
         if existing_record.nil?
@@ -96,7 +99,7 @@ module RecordStore
               answers: new_answers,
               ttl: record.ttl,
               use_client_subnet: false, # only required for filter chains that are not supported by record_store
-            }
+            },
           )
           return
         end
@@ -106,7 +109,7 @@ module RecordStore
           zone: zone,
           fqdn: record_fqdn,
           type: record.type,
-          params: { answers: existing_answers + new_answers, ttl: record.ttl }
+          params: { answers: existing_answers + new_answers, ttl: record.ttl },
         )
       end
 
@@ -120,7 +123,7 @@ module RecordStore
         existing_record = client.record(
           zone: zone,
           fqdn: record_fqdn,
-          type: record.type
+          type: record.type,
         )
         return if existing_record.nil?
 
@@ -132,7 +135,7 @@ module RecordStore
           client.delete_record(
             zone: zone,
             fqdn: record_fqdn,
-            type: record.type
+            type: record.type,
           )
           return
         end
@@ -141,7 +144,7 @@ module RecordStore
           zone: zone,
           fqdn: record_fqdn,
           type: record.type,
-          params: { answers: pruned_answers }
+          params: { answers: pruned_answers },
         )
       end
 
@@ -185,7 +188,7 @@ module RecordStore
           zone: zone,
           fqdn: record_fqdn,
           type: record.type,
-          params: { answers: existing_record['answers'], ttl: record.ttl }
+          params: { answers: existing_record['answers'], ttl: record.ttl },
         )
       end
 

data/lib/record_store/provider/oracle_cloud_dns.rb

@@ -55,7 +55,7 @@ module RecordStore
 
         client.patch_zone_records(
           zone,
-          OCI::Dns::Models::PatchZoneRecordsDetails.new(items: patch_add_record)
+          OCI::Dns::Models::PatchZoneRecordsDetails.new(items: patch_add_record),
         )
       end
 
@@ -72,6 +72,7 @@ module RecordStore
         ).data.items.select { |r| r.rdata == record.rdata_txt }
 
         return unless found_record
+
         begin
           record_hash = found_record.first.record_hash if found_record.length == 1
         rescue NoMethodError
@@ -90,7 +91,7 @@ module RecordStore
 
           client.patch_zone_records(
             zone,
-            OCI::Dns::Models::PatchZoneRecordsDetails.new(items: patch_remove_record)
+            OCI::Dns::Models::PatchZoneRecordsDetails.new(items: patch_remove_record),
           )
         end
       end
@@ -122,7 +123,7 @@ module RecordStore
           domain: record_fqdn,
           ttl: record.ttl,
           rtype: record.type,
-          rdata: record.rdata_txt
+          rdata: record.rdata_txt,
         )
         update_zone_record_items.delete_if { |r| id == r.record_hash }
 

data/lib/record_store/provider.rb

@@ -3,10 +3,12 @@ require 'resolv'
 module RecordStore
   class Provider
     class Error < StandardError; end
+
     class UnparseableBodyError < Error; end
 
     class << self
       def provider_for(object)
+        lookup_error = false
         ns_server =
           case object
           when Record::NS
@@ -16,9 +18,10 @@ module RecordStore
               master_nameserver_for(object)
             rescue Resolv::ResolvError
               $stderr.puts "Domain doesn't exist (#{object})"
-              return
+              lookup_error = true
             end
           end
+        return if lookup_error
 
         case ns_server
         when /\.dnsimple\.com\z/
@@ -152,24 +155,25 @@ module RecordStore
         )
 
         loop do
-          begin
-            return yield
-          rescue UnparseableBodyError
-            raise if max_retries <= 0
-            max_retries -= 1
-
-            waiter.wait(message: 'Waiting to retry after receiving an unparseable response')
-          rescue Net::OpenTimeout, Errno::ETIMEDOUT
-            raise if max_timeouts <= 0
-            max_timeouts -= 1
-
-            $stderr.puts('Retrying after a connection timeout')
-          rescue Errno::ECONNRESET
-            raise if max_conn_resets <= 0
-            max_conn_resets -= 1
-
-            waiter.wait
-          end
+          return yield
+        rescue UnparseableBodyError
+          raise if max_retries <= 0
+
+          max_retries -= 1
+
+          waiter.wait(message: 'Waiting to retry after receiving an unparseable response')
+        rescue Net::OpenTimeout, Errno::ETIMEDOUT
+          raise if max_timeouts <= 0
+
+          max_timeouts -= 1
+
+          $stderr.puts('Retrying after a connection timeout')
+        rescue Errno::ECONNRESET
+          raise if max_conn_resets <= 0
+
+          max_conn_resets -= 1
+
+          waiter.wait
         end
       end
     end

data/lib/record_store/record/alias.rb

@@ -23,7 +23,7 @@ module RecordStore
     end
 
     def validate_circular_reference
-      errors.add(:fqdn, 'An ALIAS should not point to itself') unless fqdn != @alias
+      errors.add(:fqdn, 'An ALIAS should not point to itself') if fqdn == @alias
     end
   end
 end

data/lib/record_store/record/caa.rb

@@ -7,7 +7,8 @@ module RecordStore
 
     validates :flags, presence: true, numericality:
       {
-        only_integer: true, greater_than_or_equal_to: 0,
+        only_integer: true,
+        greater_than_or_equal_to: 0,
         less_than_or_equal_to: 255
       }
     validates :tag, inclusion: { in: %w(issue issuewild iodef) }, presence: true
@@ -42,6 +43,7 @@ module RecordStore
     def validate_uri_value
       uri = URI(value)
       return if uri.is_a?(URI::MailTo) || uri.is_a?(URI::HTTP)
+
       errors.add(:value, "URL scheme should be mailto, http, or https")
     rescue URI::Error
       errors.add(:value, "Value should be a valid URI")

data/lib/record_store/record/cname.rb

@@ -23,7 +23,7 @@ module RecordStore
     end
 
     def validate_circular_reference
-      errors.add(:fqdn, 'A CNAME should not point to itself') unless fqdn != cname
+      errors.add(:fqdn, 'A CNAME should not point to itself') if fqdn == cname
     end
   end
 end

data/lib/record_store/record.rb

@@ -61,12 +61,14 @@ module RecordStore
       @id   = record.fetch(:record_id, nil)
     end
 
-    def self.build_from_yaml_definition(yaml_definition)
-      record_type = yaml_definition.fetch(:type)
-      Record.const_get(record_type).new(yaml_definition)
+    class << self
+      def build_from_yaml_definition(yaml_definition)
+        record_type = yaml_definition.fetch(:type)
+        Record.const_get(record_type).new(yaml_definition)
+      end
     end
 
-    def log!(logger = STDOUT)
+    def log!(logger = $stdout)
       logger.puts to_s
     end
 

data/lib/record_store/version.rb

@@ -1,3 +1,3 @@
 module RecordStore
-  VERSION = '6.3.1'.freeze
+  VERSION = '6.7.1'.freeze
 end

data/lib/record_store/zone/config/implicit_record_template.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module RecordStore
+  class Zone
+    class Config
+      class TemplateContext
+        class << self
+          def build(record:, current_records:)
+            new(record: record, current_records: current_records)
+          end
+        end
+
+        def initialize(record:, current_records:)
+          @record = record
+          @current_records = current_records
+        end
+
+        def fetch_binding
+          binding
+        end
+
+        private
+
+        attr_reader :record, :current_records
+      end
+
+      class ImplicitRecordTemplate
+        class << self
+          def from_file(filename:)
+            filepath = template_filepath_for(filename: filename)
+            template_file = File.read(filepath)
+
+            template_file_yaml = YAML.safe_load(template_file, permitted_classes: [Regexp]).deep_symbolize_keys
+            filters_for_records_to_template = template_file_yaml[:each_record]
+            filters_for_records_to_exclude = template_file_yaml[:except_record] || []
+
+            new(
+              template: ERB.new(template_file),
+              filters_for_records_to_template: filters_for_records_to_template,
+              filters_for_records_to_exclude: filters_for_records_to_exclude,
+            )
+          end
+
+          private
+
+          def template_filepath_for(filename:)
+            "#{RecordStore.implicit_records_templates_path}/#{filename}"
+          end
+        end
+
+        def initialize(template:, filters_for_records_to_template:, filters_for_records_to_exclude:)
+          @template = template
+          @filters_for_records_to_template = filters_for_records_to_template
+          @filters_for_records_to_exclude = filters_for_records_to_exclude
+        end
+
+        def generate_records_to_inject(current_records:)
+          current_records
+            .select { |record| should_template?(record: record) }
+            .map { |record| template_record_for(record: record, current_records: current_records) }
+            .each_with_object([]) do |template_records, records_to_inject|
+              next unless should_inject?(
+                template_records: template_records,
+                current_records: current_records + records_to_inject,
+              )
+
+              records_to_inject.push(
+                *template_records.fetch(:injected_records, []).map { |r_yml| Record.build_from_yaml_definition(r_yml) },
+              )
+            end
+        end
+
+        private
+
+        attr_reader :template, :filters_for_records_to_template, :filters_for_records_to_exclude
+
+        def should_inject?(template_records:, current_records:)
+          conflict_with = template_records[:conflict_with] || []
+          current_records.none? do |record|
+            conflict_with.any? do |filter|
+              record_match?(record: record, filter: filter)
+            end
+          end
+        end
+
+        def should_template?(record:)
+          filters_for_records_to_template.any? { |filter| record_match?(record: record, filter: filter) } &&
+            filters_for_records_to_exclude.none? { |filter| record_match?(record: record, filter: filter) }
+        end
+
+        def record_match?(record:, filter:)
+          filter.all? do |key, value|
+            if value.is_a?(Regexp)
+              value.match(record.public_send(key))
+            else
+              record.public_send(key) == value
+            end
+          end
+        end
+
+        def template_record_for(record:, current_records:)
+          context = TemplateContext.build(record: record, current_records: current_records)
+
+          YAML.safe_load(
+            template.result(context.fetch_binding),
+            permitted_classes: [Regexp],
+          ).deep_symbolize_keys
+        end
+      end
+    end
+  end
+end

data/lib/record_store/zone/config.rb

@@ -3,15 +3,17 @@ module RecordStore
     class Config
       include ActiveModel::Validations
 
-      attr_reader :ignore_patterns, :providers, :supports_alias
+      attr_reader :ignore_patterns, :providers, :supports_alias, :implicit_records_templates
 
       validate :validate_zone_config
 
-      def initialize(ignore_patterns: [], providers: nil, supports_alias: nil)
+      def initialize(ignore_patterns: [], providers: nil, supports_alias: nil, implicit_records_templates: [])
         @ignore_patterns = ignore_patterns.map do |ignore_pattern|
           Zone::Config::IgnorePattern.new(ignore_pattern)
         end
-
+        @implicit_records_templates = implicit_records_templates.map do |filename|
+          Zone::Config::ImplicitRecordTemplate.from_file(filename: filename)
+        end
         @providers = providers
         @supports_alias = supports_alias
       end

data/lib/record_store/zone/yaml_definitions.rb

@@ -31,7 +31,8 @@ module RecordStore
       end
 
       def write(name, config:, records:, format: :file)
-        raise ArgumentError, "format must be :directory or :file" unless %i(file directory).include?(format)
+        raise ArgumentError, "format must be :directory or :file" unless [:file, :directory].include?(format)
+
         name = name.chomp('.')
         zone_file = "#{RecordStore.zones_path}/#{name}.yml"
         zone = { name => { config: config.to_hash } }
@@ -60,13 +61,17 @@ module RecordStore
         dir = File.dirname(filename)
         data = YAML.load_file(filename)
         raise 'more than one zone in file' if data.size > 1
+
         name, definition = data.first
         definition['records'] ||= []
         definition['records'] = definition['records'].map(&:deep_symbolize_keys)
         Dir["#{dir}/#{name}/*__*.yml"].each do |record_file|
           definition['records'] += load_yml_record_definitions(name, record_file)
         end
-        Zone.new(name: name, records: definition['records'], config: definition['config'])
+
+        asts = { filename => Psych.parse_file(filename) }
+
+        Zone.new(name: name, records: definition['records'], config: definition['config'], abstract_syntax_trees: asts)
       end
 
       def load_yml_record_definitions(name, record_file)

data/lib/record_store/zone.rb

@@ -20,6 +20,8 @@ module RecordStore
     validate :validate_provider_can_handle_zone_records
     validate :validate_no_empty_non_terminal
     validate :validate_can_handle_alias_records
+    validate :validate_no_duplicate_keys
+    validate :validate_zone_record_not_shadowed
 
     class << self
       def download(name, provider_name, **write_options)
@@ -31,7 +33,7 @@ module RecordStore
         zone.config = Zone::Config.new(
           providers: [provider_name],
           ignore_patterns: [{ type: "NS", fqdn: "#{name}." }],
-          supports_alias: (zone.records.map(&:type).include?('ALIAS') || nil)
+          supports_alias: zone.records.map(&:type).include?('ALIAS') || nil,
         )
 
         zone.write(**write_options)
@@ -51,7 +53,7 @@ module RecordStore
         (ENV['RECORD_STORE_MAX_THREADS'] || DEFAULT_MAX_PARALLEL_THREADS).to_i
       end
 
-      def modified(verbose: false) # rubocop:disable Lint/UnusedMethodArgument
+      def modified(verbose: false)
         modified_zones = []
         mutex = Mutex.new
         zones = all
@@ -61,7 +63,12 @@ module RecordStore
             current_zone = nil
             while zones.any?
               mutex.synchronize { current_zone = zones.shift }
-              mutex.synchronize { modified_zones << current_zone } unless current_zone.unchanged?
+              break if current_zone.nil? # account for the race between `zones.any?` and `zones.shift`
+
+              # `unchanged?` is deliberately outside locked context since it's a bit CPU/time heavy
+              unless current_zone.unchanged?
+                mutex.synchronize { modified_zones << current_zone }
+              end
             end
           end
         end.each(&:join)
@@ -70,17 +77,16 @@ module RecordStore
       end
     end
 
-    def initialize(name:, records: [], config: {})
+    def initialize(name:, records: [], config: {}, abstract_syntax_trees: {})
       @name = Record.ensure_ends_with_dot(name)
-      @config = RecordStore::Zone::Config.new(config.deep_symbolize_keys)
+      @config = RecordStore::Zone::Config.new(**config.deep_symbolize_keys)
       @records = build_records(records)
+      @abstract_syntax_trees = abstract_syntax_trees
     end
 
     def build_changesets(all: false)
-      @changesets ||= begin
-        providers.map do |provider|
-          Changeset.build_from(provider: provider, zone: self, all: all)
-        end
+      @changesets ||= providers.map do |provider|
+        Changeset.build_from(provider: provider, zone: self, all: all)
       end
     end
 
@@ -138,24 +144,22 @@ module RecordStore
       authority = fetch_soa(nameserver) do |reply, _name|
         break if reply.answer.any?
 
-        raise "No authority found (#{name})" unless reply.authority.any?
+        raise "No authority found (#{name})" if reply.authority.none?
 
         break extract_authority(reply.authority)
       end
 
       # candidate DNS name is returned instead when NXDomain or other error
-      return nil if unrooted_name.casecmp?(Array(authority).first.to_s)
+      return if unrooted_name.casecmp?(Array(authority).first.to_s)
 
       authority
     end
 
     private
 
-    def fetch_soa(nameserver)
+    def fetch_soa(nameserver, &block)
       Resolv::DNS.open(nameserver: nameserver) do |resolv|
-        resolv.fetch_resource(name, Resolv::DNS::Resource::IN::SOA) do |reply, name|
-          yield reply, name
-        end
+        resolv.fetch_resource(name, Resolv::DNS::Resource::IN::SOA, &block)
       end
     end
 
@@ -168,6 +172,7 @@ module RecordStore
       rescue Errno::EHOSTUNREACH => e
         $stderr.puts "Warning: #{e} [host=#{nameserver}]"
         raise if nameservers.empty?
+
         retry
       end
     end
@@ -189,7 +194,13 @@ module RecordStore
     end
 
     def build_records(records)
-      records.map { |record| Record.build_from_yaml_definition(record) }
+      all_records = records.map { |record| Record.build_from_yaml_definition(record) }
+
+      config.implicit_records_templates.each do |template|
+        all_records.push(*template.generate_records_to_inject(current_records: all_records))
+      end
+
+      all_records
     end
 
     def validate_records
@@ -232,6 +243,7 @@ module RecordStore
       cname_records.each do |cname_record|
         records.each do |record|
           next unless record.fqdn == cname_record.fqdn && record != cname_record
+
           case record.type
           when 'SIG', 'NXT', 'KEY'
             # this is fine
@@ -264,6 +276,27 @@ module RecordStore
       end
     end
 
+    def validate_zone_record_not_shadowed
+      nameserver_fqdns = records
+        .select { |record| record.is_a?(Record::NS) && name != record.fqdn }
+        .map { |record| record.fqdn.delete_suffix(".") }
+        .uniq
+
+      nameserver_fqdns.each do |ns_record|
+        selected_records = records.reject do |record|
+          record.is_a?(Record::NS) &&
+            record.fqdn.delete_suffix(".") == ns_record
+        end
+        selected_records.each do |record|
+          normalized_record = record.fqdn.delete_suffix(".")
+          next unless normalized_record.end_with?(".#{ns_record}") || normalized_record == ns_record
+
+          errors.add(:records, "Record #{record.fqdn} #{record.type} in Zone #{name} " \
+            "is shadowed by #{ns_record} and will be ignored")
+        end
+      end
+    end
+
     def validate_no_empty_non_terminal
       return unless config.empty_non_terminal_over_wildcard?
 
@@ -273,14 +306,14 @@ module RecordStore
 
         terminal_records = records.map(&:fqdn)
           .select { |record| record.match?(/^([a-zA-Z0-9\-_]+\.[a-zA-Z0-9\-_])#{Regexp.escape(suffix)}$/) }
-        next unless terminal_records.any?
+        next if terminal_records.none?
 
         intermediate_records = records.map(&:fqdn)
           .select { |record| record.match?(/^([a-zA-Z0-9\-_]+)#{Regexp.escape(suffix)}$/) }
         terminal_records.each do |terminal_record|
           non_terminal = terminal_record.partition('.').last
           errors.add(:records, "found empty non-terminal #{non_terminal} "\
-                     "(caused by existing records #{wildcard} and #{terminal_record})")\
+            "(caused by existing records #{wildcard} and #{terminal_record})")\
             unless intermediate_records.include?(non_terminal)
         end
       end
@@ -301,5 +334,34 @@ module RecordStore
 
       errors.add(:records, "ALIAS record should be defined on the root of the zone: #{alias_record}")
     end
+
+    def validate_no_duplicate_keys
+      @abstract_syntax_trees.each do |filename, ast|
+        validate_no_duplicate_keys_in_node(filename, ast)
+      end
+    end
+
+    def validate_no_duplicate_keys_in_node(filename, node)
+      if node.mapping?
+        keys = node
+          .children
+          .each_slice(2)
+          .map(&:first)
+          .map(&:value)
+          .sort
+        dup_keys = keys
+          .find_all { |k| keys.count(k) > 1 }
+          .uniq
+        unless dup_keys.empty?
+          location = "#{File.basename(filename)}:#{node.start_line}"
+          description = "multiple definitions for keys #{dup_keys}"
+          errors.add(:records, "#{location}: #{description}")
+        end
+      end
+
+      node.children&.each do |child|
+        validate_no_duplicate_keys_in_node(filename, child)
+      end
+    end
   end
 end

data/lib/record_store.rb

@@ -28,6 +28,7 @@ require 'record_store/zone/yaml_definitions'
 require 'record_store/zone'
 require 'record_store/zone/config'
 require 'record_store/zone/config/ignore_pattern'
+require 'record_store/zone/config/implicit_record_template'
 require 'record_store/changeset'
 require 'record_store/provider'
 require 'record_store/provider/dynect'
@@ -43,6 +44,7 @@ module RecordStore
   class << self
     attr_writer :secrets_path
     attr_writer :zones_path
+    attr_writer :implicit_records_templates_path
 
     def secrets_path
       @secrets_path ||= File.expand_path(config.fetch('secrets_path'), File.dirname(config_path))
@@ -50,8 +52,10 @@ module RecordStore
 
     def zones_path
       @zones_path ||= Pathname.new(
-        File.expand_path(config.fetch('zones_path'),
-        File.dirname(config_path)),
+        File.expand_path(
+          config.fetch('zones_path'),
+          File.dirname(config_path),
+        ),
       ).realpath.to_s
     end
 
@@ -59,6 +63,15 @@ module RecordStore
       @config_path ||= File.expand_path('config.yml', Dir.pwd)
     end
 
+    def implicit_records_templates_path
+      @implicit_records_templates_path ||= Pathname.new(
+        File.expand_path(
+          config.fetch('implicit_records_templates_path'),
+          File.dirname(config_path),
+        ),
+      ).realpath.to_s
+    end
+
     def config_path=(config_path)
       @config = @zones_path = @secrets_path = nil
       @config_path = config_path