record_store 9.0.0 → 9.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ce291e6fbcef26df114b7bbf86e5e2ebd21bfdccf8734f5a9c68976ef92bdf2b
-  data.tar.gz: 2138a7322d533f1de25e2bd7dc69a5046acfda2f352520a64cbccf209f8a7b35
+  metadata.gz: effd0589fd20b2602042b321bd4f82d00b1f9fe08aa67f02b88e7b798d217559
+  data.tar.gz: 051e18aa4b65ac9ff67824027c1fcf83ebc66222dbd9b244ed7ae8f5914f8fce
 SHA512:
-  metadata.gz: ec4b883bb2bde7c4803ddd623a4ff95866a7d0505e1493f76af61abd6d904f2f81f40791a7902bbb380def97e82320cc0d1c57ba54fe02d87c726f41e6cb7f51
-  data.tar.gz: 5f2e9b36f150961bbb40df71c3aa917e8572d509ecb28aa547b75442adacba8c2f3cc7d642469a0b63402b09334d747c7bdf3245b264b233dae37daea22fc96c
+  metadata.gz: 237530be7cc7ee854275117be9d21d3e039eaf17fd45b73bb7cc56ecc7723bff2c531712021c263db2c87ef2eeca3d7a84fae18341a33615c4cf70af0ce66f77
+  data.tar.gz: 9ed803f7f7aad03e8a45bd46e0e0a8e1f398fdf7f70479b5154cb06b2efebd36cbc77cbeb2d150ed0374610dc995c14afd7ee057d9d28ce7bff3e96a71240623

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # CHANGELOG
 
+## 9.1.0
+- Load zones faster. Implicit record templates no longer rescan every record for each templated record, the duplicate record check no longer compares every pair of records, and `Record#type` is computed once per class. Zones, records and validation errors are unchanged. Loading and validating a repository with several hundred zones is several times faster.
+- Cloudflare: look up zone ids from one listing of all zones, fetched once per run and shared by all threads, instead of one request per zone each time a zone is read or written. A zone that is not in the listing is looked up on its own. A zone that does not exist now fails with `Zone not found for <name>`; before, it failed with `undefined method 'size' for nil`.
+
 ## 9.0.0
 - Only load implicit record templates by plain file name from directly inside `implicit_records_templates_path`. Template names containing path separators, `.` or `..`, and symlinks that resolve outside that directory now raise an error. Zone definitions that referenced templates this way must move them into the templates directory and use the plain file name.
 

data/lib/record_store/provider/cloudflare.rb

@@ -2,6 +2,12 @@ require_relative 'cloudflare/client'
 
 module RecordStore
   class Provider::Cloudflare < Provider
+    # Cloudflare returns at most 50 zones per page.
+    ZONE_LIST_PER_PAGE = 50
+
+    ZONE_IDS_MUTEX = Mutex.new
+    private_constant :ZONE_IDS_MUTEX
+
     class << self
       def record_types
         super | Set.new(%w(PTR))
@@ -45,25 +51,13 @@ module RecordStore
       # Cloudflare returns zones across all accounts accessible by the API token
       # Can implement filtering in request if needed
       def zones
-        all_zones = []
-        page = 1
-        per_page = 50
-
-        loop do
-          response = nil
-          retry_on_connection_errors do
-            response = client.get('/client/v4/zones', page: page, per_page: per_page)
-          end
-
-          zones = response.result_raw || []
-          all_zones.concat(zones)
-
-          break if page * per_page >= response.result_info_raw['total_count']
-
-          page += 1
-        end
+        list_zones.map { |zone| zone['name'] }
+      end
 
-        all_zones.map { |zone| zone['name'] }
+      # Forgets the zone name => id mapping, so the next lookup lists the zones again. The mapping is kept for the
+      # life of the process, which is a single run.
+      def clear_zone_id_cache
+        ZONE_IDS_MUTEX.synchronize { @zone_ids = nil }
       end
 
       def apply_changeset(changeset, stdout = $stdout)
@@ -204,20 +198,76 @@ module RecordStore
         )
       end
 
+      # Returns every zone the token can see, as the raw API hashes.
+      def list_zones
+        all_zones = []
+        page = 1
+
+        loop do
+          response = nil
+          retry_on_connection_errors do
+            response = client.get('/client/v4/zones', page: page, per_page: ZONE_LIST_PER_PAGE)
+          end
+
+          zones = response.result_raw || []
+          all_zones.concat(zones)
+
+          break if page * ZONE_LIST_PER_PAGE >= response.result_info_raw['total_count']
+
+          page += 1
+        end
+
+        all_zones
+      end
+
+      # Zone ids are looked up from one listing of all zones, which is fetched on first use and shared by every
+      # thread. A zone that is not in that listing (for example one created after it was fetched) is looked up on its
+      # own, and the answer is remembered.
       def zone_name_to_id(zone_name)
+        key = zone_id_cache_key(zone_name)
+        ids = zone_ids_by_name[key]
+
+        if ids.nil?
+          ids = zone_ids_from_lookup(zone_name)
+          remember_zone_ids(key, ids) if ids.size == 1
+        end
+
+        case ids.size
+        when 0
+          raise "Zone not found for #{zone_name}"
+        when 1
+          ids.first
+        else
+          raise "Multiple zones found for #{zone_name}. " \
+            "API key must only return the zone on which records are to be managed."
+        end
+      end
+
+      def zone_id_cache_key(zone_name)
+        zone_name.to_s.chomp('.').downcase
+      end
+
+      # name => [id, ...] for every zone in the account(s). Several ids under one name means the name is ambiguous.
+      # A zone created while the listing is being paged can show up on two pages, hence the uniq.
+      def zone_ids_by_name
+        @zone_ids || ZONE_IDS_MUTEX.synchronize do
+          @zone_ids ||= list_zones
+            .group_by { |zone| zone_id_cache_key(zone['name']) }
+            .transform_values { |zones| zones.map { |zone| zone['id'] }.uniq }
+            .freeze
+        end
+      end
+
+      def remember_zone_ids(key, ids)
+        ZONE_IDS_MUTEX.synchronize do
+          @zone_ids = @zone_ids.merge(key => ids.freeze).freeze if @zone_ids
+        end
+      end
+
+      def zone_ids_from_lookup(zone_name)
         retry_on_connection_errors do
-          params = { name: zone_name }
-          matching_zones = client.get("/client/v4/zones", params).result_raw
-
-          case matching_zones.size
-          when 0
-            raise "Zone not found for #{zone_name}"
-          when 1
-            matching_zones.first['id']
-          else
-            raise "Multiple zones found for #{zone_name}. " \
-              "API key must only return the zone on which records are to be managed."
-          end
+          matching_zones = client.get("/client/v4/zones", name: zone_name).result_raw || []
+          matching_zones.map { |zone| zone['id'] }
         end
       end
     end

data/lib/record_store/record.rb

@@ -42,6 +42,12 @@ module RecordStore
         unescape(value.sub(/\A"(.*)"\z/, '\1'))
       end
 
+      # The record type, e.g. "A" for Record::A. Called millions of times when loading zones, so it is computed once
+      # per class.
+      def type_name
+        @type_name ||= name.demodulize.freeze
+      end
+
       def ensure_ends_with_dot(fqdn)
         fqdn.end_with?(".") ? fqdn : "#{fqdn}."
       end
@@ -80,8 +86,9 @@ module RecordStore
       }.merge(rdata)
     end
 
+    # Returns a new string each time, as it always has: callers such as fog-dynect modify it in place.
     def type
-      self.class.name.demodulize
+      self.class.type_name.dup
     end
 
     def ==(other)

data/lib/record_store/version.rb

@@ -1,3 +1,3 @@
 module RecordStore
-  VERSION = '9.0.0'.freeze
+  VERSION = '9.1.0'.freeze
 end

data/lib/record_store/zone/config/implicit_record_template.rb

@@ -75,18 +75,26 @@ module RecordStore
         end
 
         def generate_records_to_inject(current_records:)
+          # Every [type, fqdn] pair currently in the zone, including what has been injected so far. Conflict filters
+          # that only name a type and an fqdn are answered from this set instead of scanning every record.
+          present = Set.new(current_records.map { |record| [record.type, record.fqdn] })
+
           current_records
             .select { |record| should_template?(record: record) }
             .map { |record| template_record_for(record: record, current_records: current_records) }
             .each_with_object([]) do |template_records, records_to_inject|
               next unless should_inject?(
                 template_records: template_records,
-                current_records: current_records + records_to_inject,
+                current_records: current_records,
+                records_to_inject: records_to_inject,
+                present: present,
               )
 
-              records_to_inject.push(
-                *template_records.fetch(:injected_records, []).map { |r_yml| Record.build_from_yaml_definition(r_yml) },
-              )
+              new_records = template_records.fetch(:injected_records, []).map do |r_yml|
+                Record.build_from_yaml_definition(r_yml)
+              end
+              new_records.each { |record| present << [record.type, record.fqdn] }
+              records_to_inject.push(*new_records)
             end
         end
 
@@ -94,15 +102,33 @@ module RecordStore
 
         attr_reader :template, :filters_for_records_to_template, :filters_for_records_to_exclude
 
-        def should_inject?(template_records:, current_records:)
+        def should_inject?(template_records:, current_records:, records_to_inject:, present:)
           conflict_with = template_records[:conflict_with] || []
-          current_records.none? do |record|
+
+          if conflict_with.all? { |filter| type_and_fqdn_filter?(filter) }
+            conflict_with.none? { |filter| present.include?([filter[:type], filter[:fqdn]]) }
+          else
+            !conflicts_with_any?(conflict_with: conflict_with, records: current_records + records_to_inject)
+          end
+        end
+
+        # The general check: any record matching any filter is a conflict. Used for filters that match on anything
+        # other than an exact type and fqdn (regexps, other attributes).
+        def conflicts_with_any?(conflict_with:, records:)
+          records.any? do |record|
             conflict_with.any? do |filter|
               record_match?(record: record, filter: filter)
             end
           end
         end
 
+        def type_and_fqdn_filter?(filter)
+          filter.is_a?(Hash) &&
+            filter.keys.sort == [:fqdn, :type] &&
+            filter[:type].is_a?(String) &&
+            filter[:fqdn].is_a?(String)
+        end
+
         def should_template?(record:)
           filters_for_records_to_template.any? { |filter| record_match?(record: record, filter: filter) } &&
             filters_for_records_to_exclude.none? { |filter| record_match?(record: record, filter: filter) }

data/lib/record_store/zone.rb

@@ -259,9 +259,24 @@ module RecordStore
     end
 
     def validate_all_records_are_unique
-      duplicates = records.select { |r| records.count(r) > 1 }
-      duplicates.uniq.each do |record|
-        errors.add(:records, "Duplicate record: #{record}")
+      if records.all? { |record| comparable_by_hash?(record) }
+        # Hash keys keep first-seen order, so each duplicated record is reported once, in the order it first appears.
+        records.tally.each do |record, count|
+          errors.add(:records, "Duplicate record: #{record}") if count > 1
+        end
+      else
+        duplicates = records.select { |r| records.count(r) > 1 }
+        duplicates.uniq.each do |record|
+          errors.add(:records, "Duplicate record: #{record}")
+        end
+      end
+    end
+
+    # Record#== compares to_hash with ==, but tally and uniq use Record#hash and eql?, which tell 1 from 1.0. The two
+    # agree when every value is a string, an Integer, a boolean or nil, so only then is tallying safe.
+    def comparable_by_hash?(record)
+      record.to_hash.each_value.all? do |value|
+        value.is_a?(String) || value.is_a?(Integer) || value == true || value == false || value.nil?
       end
     end
 

data/lib/record_store.rb

@@ -1,4 +1,5 @@
 require 'json'
+require 'set'
 require 'yaml'
 require 'active_support'
 require 'active_support/core_ext/array/wrap'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: record_store
 version: !ruby/object:Gem::Version
-  version: 9.0.0
+  version: 9.1.0
 platform: ruby
 authors:
 - Willem van Bergen
@@ -461,7 +461,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.21
+rubygems_version: 4.0.23
 specification_version: 4
 summary: Manage DNS using git
 test_files: []