record_store 8.0.11 → 9.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 43d93495d6304ff99a4c2335be9452a7dd8a222580d702536e21d3cf836e0607
-  data.tar.gz: f1f948d4136c09ecca1a6199eda17aea70bb726a3a75e8aa224d0ee3b2df13a3
+  metadata.gz: effd0589fd20b2602042b321bd4f82d00b1f9fe08aa67f02b88e7b798d217559
+  data.tar.gz: 051e18aa4b65ac9ff67824027c1fcf83ebc66222dbd9b244ed7ae8f5914f8fce
 SHA512:
-  metadata.gz: 7e143587f1505ee799b3849d1fdcec2e58563fd5ffb9e75154e4277bef465bf8e76aec116aeb9e418eb81441a46132ab34c8806c39266cc4c7db4f92c6125907
-  data.tar.gz: c92a3b36d67f4cec13ba82aa3c7679803188c1e56e79917fadccc82cd0c856d6bdefd0beb4892ca050c7005579d4e0ec888fe35cc5e9850c464d3cde23a5c542
+  metadata.gz: 237530be7cc7ee854275117be9d21d3e039eaf17fd45b73bb7cc56ecc7723bff2c531712021c263db2c87ef2eeca3d7a84fae18341a33615c4cf70af0ce66f77
+  data.tar.gz: 9ed803f7f7aad03e8a45bd46e0e0a8e1f398fdf7f70479b5154cb06b2efebd36cbc77cbeb2d150ed0374610dc995c14afd7ee057d9d28ce7bff3e96a71240623

data/.github/workflows/ci.yml

@@ -14,7 +14,7 @@ jobs:
         ruby: [ 3.2, 3.3, 3.4 ]
     name: Test Ruby ${{ matrix.ruby }}
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - name: Install dependencies
         run: |
           sudo apt-get update && sudo apt-get install build-essential libcurl4-openssl-dev

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # CHANGELOG
 
+## 9.1.0
+- Load zones faster. Implicit record templates no longer rescan every record for each templated record, the duplicate record check no longer compares every pair of records, and `Record#type` is computed once per class. Zones, records and validation errors are unchanged. Loading and validating a repository with several hundred zones is several times faster.
+- Cloudflare: look up zone ids from one listing of all zones, fetched once per run and shared by all threads, instead of one request per zone each time a zone is read or written. A zone that is not in the listing is looked up on its own. A zone that does not exist now fails with `Zone not found for <name>`; before, it failed with `undefined method 'size' for nil`.
+
+## 9.0.0
+- Only load implicit record templates by plain file name from directly inside `implicit_records_templates_path`. Template names containing path separators, `.` or `..`, and symlinks that resolve outside that directory now raise an error. Zone definitions that referenced templates this way must move them into the templates directory and use the plain file name.
+
 ## 8.0.11
 - Modified Zone#fetch_authority to handle SOA answer sections from authoritative servers
 

data/README.md

@@ -135,6 +135,8 @@ Record Store supports injecting implicit records into a zone based on templates.
 
 Templates can help reduce zone file bloat where instead of defining many generic literals within the zone file for a given criteria zone record, these generic records can be implicitly injected into the `Zone` at the time of initialization based on information provided within the template.
 
+Zone definitions reference templates by plain file name (for example `implicit_records_templates: [acme-challenge.yml.erb]`), and only files directly inside `implicit_records_templates_path` can be loaded. Names containing path separators, `.` or `..`, and symlinks that resolve outside that directory are rejected. Templates are rendered as ERB, so treat the templates directory as code: restrict who can change it to people you would trust to change Record Store itself.
+
 ----
 
 # Development

data/lib/record_store/provider/cloudflare.rb

@@ -2,6 +2,12 @@ require_relative 'cloudflare/client'
 
 module RecordStore
   class Provider::Cloudflare < Provider
+    # Cloudflare returns at most 50 zones per page.
+    ZONE_LIST_PER_PAGE = 50
+
+    ZONE_IDS_MUTEX = Mutex.new
+    private_constant :ZONE_IDS_MUTEX
+
     class << self
       def record_types
         super | Set.new(%w(PTR))
@@ -45,25 +51,13 @@ module RecordStore
       # Cloudflare returns zones across all accounts accessible by the API token
       # Can implement filtering in request if needed
       def zones
-        all_zones = []
-        page = 1
-        per_page = 50
-
-        loop do
-          response = nil
-          retry_on_connection_errors do
-            response = client.get('/client/v4/zones', page: page, per_page: per_page)
-          end
-
-          zones = response.result_raw || []
-          all_zones.concat(zones)
-
-          break if page * per_page >= response.result_info_raw['total_count']
-
-          page += 1
-        end
+        list_zones.map { |zone| zone['name'] }
+      end
 
-        all_zones.map { |zone| zone['name'] }
+      # Forgets the zone name => id mapping, so the next lookup lists the zones again. The mapping is kept for the
+      # life of the process, which is a single run.
+      def clear_zone_id_cache
+        ZONE_IDS_MUTEX.synchronize { @zone_ids = nil }
       end
 
       def apply_changeset(changeset, stdout = $stdout)
@@ -204,20 +198,76 @@ module RecordStore
         )
       end
 
+      # Returns every zone the token can see, as the raw API hashes.
+      def list_zones
+        all_zones = []
+        page = 1
+
+        loop do
+          response = nil
+          retry_on_connection_errors do
+            response = client.get('/client/v4/zones', page: page, per_page: ZONE_LIST_PER_PAGE)
+          end
+
+          zones = response.result_raw || []
+          all_zones.concat(zones)
+
+          break if page * ZONE_LIST_PER_PAGE >= response.result_info_raw['total_count']
+
+          page += 1
+        end
+
+        all_zones
+      end
+
+      # Zone ids are looked up from one listing of all zones, which is fetched on first use and shared by every
+      # thread. A zone that is not in that listing (for example one created after it was fetched) is looked up on its
+      # own, and the answer is remembered.
       def zone_name_to_id(zone_name)
+        key = zone_id_cache_key(zone_name)
+        ids = zone_ids_by_name[key]
+
+        if ids.nil?
+          ids = zone_ids_from_lookup(zone_name)
+          remember_zone_ids(key, ids) if ids.size == 1
+        end
+
+        case ids.size
+        when 0
+          raise "Zone not found for #{zone_name}"
+        when 1
+          ids.first
+        else
+          raise "Multiple zones found for #{zone_name}. " \
+            "API key must only return the zone on which records are to be managed."
+        end
+      end
+
+      def zone_id_cache_key(zone_name)
+        zone_name.to_s.chomp('.').downcase
+      end
+
+      # name => [id, ...] for every zone in the account(s). Several ids under one name means the name is ambiguous.
+      # A zone created while the listing is being paged can show up on two pages, hence the uniq.
+      def zone_ids_by_name
+        @zone_ids || ZONE_IDS_MUTEX.synchronize do
+          @zone_ids ||= list_zones
+            .group_by { |zone| zone_id_cache_key(zone['name']) }
+            .transform_values { |zones| zones.map { |zone| zone['id'] }.uniq }
+            .freeze
+        end
+      end
+
+      def remember_zone_ids(key, ids)
+        ZONE_IDS_MUTEX.synchronize do
+          @zone_ids = @zone_ids.merge(key => ids.freeze).freeze if @zone_ids
+        end
+      end
+
+      def zone_ids_from_lookup(zone_name)
         retry_on_connection_errors do
-          params = { name: zone_name }
-          matching_zones = client.get("/client/v4/zones", params).result_raw
-
-          case matching_zones.size
-          when 0
-            raise "Zone not found for #{zone_name}"
-          when 1
-            matching_zones.first['id']
-          else
-            raise "Multiple zones found for #{zone_name}. " \
-              "API key must only return the zone on which records are to be managed."
-          end
+          matching_zones = client.get("/client/v4/zones", name: zone_name).result_raw || []
+          matching_zones.map { |zone| zone['id'] }
         end
       end
     end

data/lib/record_store/record.rb

@@ -42,6 +42,12 @@ module RecordStore
         unescape(value.sub(/\A"(.*)"\z/, '\1'))
       end
 
+      # The record type, e.g. "A" for Record::A. Called millions of times when loading zones, so it is computed once
+      # per class.
+      def type_name
+        @type_name ||= name.demodulize.freeze
+      end
+
       def ensure_ends_with_dot(fqdn)
         fqdn.end_with?(".") ? fqdn : "#{fqdn}."
       end
@@ -80,8 +86,9 @@ module RecordStore
       }.merge(rdata)
     end
 
+    # Returns a new string each time, as it always has: callers such as fog-dynect modify it in place.
     def type
-      self.class.name.demodulize
+      self.class.type_name.dup
     end
 
     def ==(other)

data/lib/record_store/version.rb

@@ -1,3 +1,3 @@
 module RecordStore
-  VERSION = '8.0.11'.freeze
+  VERSION = '9.1.0'.freeze
 end

data/lib/record_store/zone/config/implicit_record_template.rb

@@ -25,6 +25,8 @@ module RecordStore
       end
 
       class ImplicitRecordTemplate
+        class InvalidTemplateFilename < ArgumentError; end
+
         class << self
           def from_file(filename:)
             filepath = template_filepath_for(filename: filename)
@@ -43,8 +45,26 @@ module RecordStore
 
           private
 
+          # Templates are rendered as ERB, so only files directly inside the
+          # configured templates directory may be loaded. Zone definitions name a
+          # template by its plain file name; anything that could resolve elsewhere
+          # (path separators, `.`/`..`, or a symlink out of the directory) is rejected.
           def template_filepath_for(filename:)
-            "#{RecordStore.implicit_records_templates_path}/#{filename}"
+            name = filename.to_s
+            base = Pathname.new(RecordStore.implicit_records_templates_path).realpath
+
+            if name.empty? || name == '.' || name == '..' || name != File.basename(name)
+              raise InvalidTemplateFilename,
+                "implicit record template #{filename.inspect} must be a plain file name in #{base}"
+            end
+
+            path = base.join(name).realpath
+            unless path.dirname == base
+              raise InvalidTemplateFilename,
+                "implicit record template #{filename.inspect} resolves outside #{base}"
+            end
+
+            path.to_s
           end
         end
 
@@ -55,18 +75,26 @@ module RecordStore
         end
 
         def generate_records_to_inject(current_records:)
+          # Every [type, fqdn] pair currently in the zone, including what has been injected so far. Conflict filters
+          # that only name a type and an fqdn are answered from this set instead of scanning every record.
+          present = Set.new(current_records.map { |record| [record.type, record.fqdn] })
+
           current_records
             .select { |record| should_template?(record: record) }
             .map { |record| template_record_for(record: record, current_records: current_records) }
             .each_with_object([]) do |template_records, records_to_inject|
               next unless should_inject?(
                 template_records: template_records,
-                current_records: current_records + records_to_inject,
+                current_records: current_records,
+                records_to_inject: records_to_inject,
+                present: present,
               )
 
-              records_to_inject.push(
-                *template_records.fetch(:injected_records, []).map { |r_yml| Record.build_from_yaml_definition(r_yml) },
-              )
+              new_records = template_records.fetch(:injected_records, []).map do |r_yml|
+                Record.build_from_yaml_definition(r_yml)
+              end
+              new_records.each { |record| present << [record.type, record.fqdn] }
+              records_to_inject.push(*new_records)
             end
         end
 
@@ -74,15 +102,33 @@ module RecordStore
 
         attr_reader :template, :filters_for_records_to_template, :filters_for_records_to_exclude
 
-        def should_inject?(template_records:, current_records:)
+        def should_inject?(template_records:, current_records:, records_to_inject:, present:)
           conflict_with = template_records[:conflict_with] || []
-          current_records.none? do |record|
+
+          if conflict_with.all? { |filter| type_and_fqdn_filter?(filter) }
+            conflict_with.none? { |filter| present.include?([filter[:type], filter[:fqdn]]) }
+          else
+            !conflicts_with_any?(conflict_with: conflict_with, records: current_records + records_to_inject)
+          end
+        end
+
+        # The general check: any record matching any filter is a conflict. Used for filters that match on anything
+        # other than an exact type and fqdn (regexps, other attributes).
+        def conflicts_with_any?(conflict_with:, records:)
+          records.any? do |record|
             conflict_with.any? do |filter|
               record_match?(record: record, filter: filter)
             end
           end
         end
 
+        def type_and_fqdn_filter?(filter)
+          filter.is_a?(Hash) &&
+            filter.keys.sort == [:fqdn, :type] &&
+            filter[:type].is_a?(String) &&
+            filter[:fqdn].is_a?(String)
+        end
+
         def should_template?(record:)
           filters_for_records_to_template.any? { |filter| record_match?(record: record, filter: filter) } &&
             filters_for_records_to_exclude.none? { |filter| record_match?(record: record, filter: filter) }

data/lib/record_store/zone.rb

@@ -80,7 +80,7 @@ module RecordStore
 
     def initialize(name:, records: [], config: {}, abstract_syntax_trees: {})
       @name = Record.ensure_ends_with_dot(name)
-      @config = RecordStore::Zone::Config.new(**config.deep_symbolize_keys)
+      @config = build_config(config)
       @records = build_records(records)
       @abstract_syntax_trees = abstract_syntax_trees
     end
@@ -224,6 +224,12 @@ module RecordStore
       end
     end
 
+    def build_config(config)
+      RecordStore::Zone::Config.new(**config.deep_symbolize_keys)
+    rescue Config::ImplicitRecordTemplate::InvalidTemplateFilename => e
+      raise e.exception("zone #{unrooted_name}: #{e.message}")
+    end
+
     def build_records(records)
       all_records = records.map { |record| Record.build_from_yaml_definition(record) }
 
@@ -253,9 +259,24 @@ module RecordStore
     end
 
     def validate_all_records_are_unique
-      duplicates = records.select { |r| records.count(r) > 1 }
-      duplicates.uniq.each do |record|
-        errors.add(:records, "Duplicate record: #{record}")
+      if records.all? { |record| comparable_by_hash?(record) }
+        # Hash keys keep first-seen order, so each duplicated record is reported once, in the order it first appears.
+        records.tally.each do |record, count|
+          errors.add(:records, "Duplicate record: #{record}") if count > 1
+        end
+      else
+        duplicates = records.select { |r| records.count(r) > 1 }
+        duplicates.uniq.each do |record|
+          errors.add(:records, "Duplicate record: #{record}")
+        end
+      end
+    end
+
+    # Record#== compares to_hash with ==, but tally and uniq use Record#hash and eql?, which tell 1 from 1.0. The two
+    # agree when every value is a string, an Integer, a boolean or nil, so only then is tallying safe.
+    def comparable_by_hash?(record)
+      record.to_hash.each_value.all? do |value|
+        value.is_a?(String) || value.is_a?(Integer) || value == true || value == false || value.nil?
       end
     end
 

data/lib/record_store.rb

@@ -1,4 +1,5 @@
 require 'json'
+require 'set'
 require 'yaml'
 require 'active_support'
 require 'active_support/core_ext/array/wrap'

data/record_store.gemspec

@@ -32,7 +32,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'ejson'
   spec.add_runtime_dependency 'thor', '>= 1.4.0'
 
-  spec.add_runtime_dependency 'dnsimple', '>= 4.4', '< 11.2'
+  spec.add_runtime_dependency 'dnsimple', '>= 4.4', '< 12.3'
   spec.add_runtime_dependency 'fog-dynect', '>= 0.4', '< 0.6'
   spec.add_runtime_dependency 'fog-json'
   spec.add_runtime_dependency 'fog-xml'
@@ -47,7 +47,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'mocha'
   spec.add_development_dependency 'pry'
   spec.add_development_dependency 'rake'
-  spec.add_development_dependency 'rubocop', '~> 1.82.1'
+  spec.add_development_dependency 'rubocop', '~> 1.88.0'
   spec.add_development_dependency 'rubocop-shopify', '~> 2.15.1'
   spec.add_development_dependency 'vcr'
   spec.add_development_dependency 'webmock'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: record_store
 version: !ruby/object:Gem::Version
-  version: 8.0.11
+  version: 9.1.0
 platform: ruby
 authors:
 - Willem van Bergen
@@ -89,7 +89,7 @@ dependencies:
         version: '4.4'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '11.2'
+        version: '12.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -99,7 +99,7 @@ dependencies:
         version: '4.4'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '11.2'
+        version: '12.3'
 - !ruby/object:Gem::Dependency
   name: fog-dynect
   requirement: !ruby/object:Gem::Requirement
@@ -312,14 +312,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.82.1
+        version: 1.88.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.82.1
+        version: 1.88.0
 - !ruby/object:Gem::Dependency
   name: rubocop-shopify
   requirement: !ruby/object:Gem::Requirement
@@ -461,7 +461,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.7.2
+rubygems_version: 4.0.23
 specification_version: 4
 summary: Manage DNS using git
 test_files: []