record_store 6.1.0 → 6.3.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d6686f890657b7a250384d189c2c377519616711a8295f510bbb140abaefde22
4
- data.tar.gz: 00066a49bd826aded557ba3c0a6cb60216ce53feb0c33efd89de9ae7792a5d7c
3
+ metadata.gz: 6476b30f3ef1df05642b1645783e2186e8ca6a3f489d99c4724fb252d6b26b97
4
+ data.tar.gz: 84ddb2d0e34076198bc1df633ab51714253c8c4ced05ef8b5b60eb777714eacb
5
5
  SHA512:
6
- metadata.gz: 642f48f0c639da2f9c3f5cb16bc3265504c647797586bbc6d7aa06f3667dbfaefe8977a4744f8a390ffd7bb45b2567e0964e8e719793182bbca05ccb9143d1e2
7
- data.tar.gz: 34e8d630d6b53dc22370f1472cd292b218e79136772151c403b46cc4eb027aa9377beebd17524dac6b37d47645ed7597a756f40f78511ebf0f5678ea9a61c74d
6
+ metadata.gz: '01136420252835b0ce65abf03d0466f88f879cf52c55f1db5e3db0753121bd55610c78fec31ee552b0d95c02b1922e8aefd4d91679872b1f16af16188fff0cde'
7
+ data.tar.gz: 512eb4f464314c298c07367e52dc9a3e6cec126da174f19c52cd4d32c12be7ab1d1a565c653b6f853a346582db9076e078ee530914272c1fbf6317306c911176
@@ -671,7 +671,7 @@ Style/LineEndConcatenation:
671
671
  Style/MethodCallWithoutArgsParentheses:
672
672
  Enabled: true
673
673
 
674
- Style/MethodMissingSuper:
674
+ Lint/MissingSuper:
675
675
  Enabled: true
676
676
 
677
677
  Style/MissingRespondToMissing:
@@ -964,7 +964,7 @@ Lint/UselessAccessModifier:
964
964
  Lint/UselessAssignment:
965
965
  Enabled: true
966
966
 
967
- Lint/UselessComparison:
967
+ Lint/BinaryOperatorWithIdenticalOperands:
968
968
  Enabled: true
969
969
 
970
970
  Lint/UselessElseWithoutRescue:
@@ -1,7 +1,7 @@
1
1
  cache: bundler
2
2
  language: ruby
3
3
  rvm:
4
- - 2.4.3
4
+ - 2.6.4
5
5
 
6
6
  before_install:
7
7
  - gem update --system
@@ -1,5 +1,20 @@
1
1
  # CHANGELOG
2
2
 
3
+ ## 6.3.0
4
+ - Support for configurable number of threads via environment variable [FEATURE]
5
+
6
+ ## 6.2.1
7
+ - Improved error reporting after timeouts [FEATURE]
8
+
9
+ ## 6.2.0
10
+ - Add validation for non-terminal conflict with wildcard [FEATURE]
11
+
12
+ ## 6.1.2
13
+ - Retry on connection errors [FEATURE]
14
+
15
+ ## 6.1.1
16
+ - Emit messages when waiting for rate-limit to elapse for DNSimple and NS1 providers, so deployment does not timeout [BUGFIX]
17
+
3
18
  ## 6.1.0
4
19
  - sort zone files [FEATURE]
5
20
  - CLI support for specifying zones for validate_authority [FEATURE]
data/README.md CHANGED
@@ -112,6 +112,10 @@ Changesets are how Record Store knows what updates to make. A `Changeset` is gen
112
112
 
113
113
  When running `bin/record-store apply`, a `Changeset` is generated by comparing the current records in a zone's YAML file with the records the provider defines. A zone's YAML file is always considered the primary source of truth.
114
114
 
115
+ ### Parallelism
116
+
117
+ Record store attempts to parallelize some of the bulk zone fetching operations. It does so by spawning multiple threads (default: 10). This value can be configured by setting the RECORD_STORE_MAX_THREADS environment variable to a positive integer value.
118
+
115
119
  ----
116
120
 
117
121
  # Development
data/dev.yml CHANGED
@@ -2,7 +2,7 @@
2
2
  name: recordstore
3
3
 
4
4
  up:
5
- - ruby: 2.5.0
5
+ - ruby: 2.6.4
6
6
  - bundler
7
7
 
8
8
  commands:
@@ -52,6 +52,10 @@ module RecordStore
52
52
  false
53
53
  end
54
54
 
55
+ def empty_non_terminal_over_wildcard?
56
+ true
57
+ end
58
+
55
59
  def build_zone(zone_name:, config:)
56
60
  zone = Zone.new(name: zone_name)
57
61
  zone.records = retrieve_current_records(zone: zone_name)
@@ -128,6 +132,37 @@ module RecordStore
128
132
 
129
133
  dns.getresource(zone_name, Resolv::DNS::Resource::IN::SOA).mname.to_s
130
134
  end
135
+
136
+ def retry_on_connection_errors(
137
+ max_timeouts: 5,
138
+ max_conn_resets: 5,
139
+ delay: 1,
140
+ backoff_multiplier: 2,
141
+ max_backoff: 10
142
+ )
143
+ waiter = BackoffWaiter.new(
144
+ "Waiting to retry after a connection reset",
145
+ initial_delay: delay,
146
+ multiplier: backoff_multiplier,
147
+ max_delay: max_backoff,
148
+ )
149
+
150
+ loop do
151
+ begin
152
+ return yield
153
+ rescue Net::OpenTimeout, Errno::ETIMEDOUT
154
+ raise if max_timeouts <= 0
155
+ max_timeouts -= 1
156
+
157
+ $stderr.puts("Retrying after a connection timeout")
158
+ rescue Errno::ECONNRESET
159
+ raise if max_conn_resets <= 0
160
+ max_conn_resets -= 1
161
+
162
+ waiter.wait
163
+ end
164
+ end
165
+ end
131
166
  end
132
167
  end
133
168
  end
@@ -12,21 +12,29 @@ module RecordStore
12
12
  true
13
13
  end
14
14
 
15
+ def empty_non_terminal_over_wildcard?
16
+ false
17
+ end
18
+
15
19
  # returns an array of Record objects that match the records which exist in the provider
16
20
  def retrieve_current_records(zone:, stdout: $stdout)
17
- session.zones.all_records(account_id, zone).data.map do |record|
18
- begin
19
- build_from_api(record, zone)
20
- rescue StandardError
21
- stdout.puts "Cannot build record: #{record}"
22
- raise
23
- end
24
- end.compact
21
+ retry_on_connection_errors do
22
+ session.zones.all_records(account_id, zone).data.map do |record|
23
+ begin
24
+ build_from_api(record, zone)
25
+ rescue StandardError
26
+ stdout.puts "Cannot build record: #{record}"
27
+ raise
28
+ end
29
+ end.compact
30
+ end
25
31
  end
26
32
 
27
33
  # Returns an array of the zones managed by provider as strings
28
34
  def zones
29
- session.zones.all_zones(account_id).data.map(&:name)
35
+ retry_on_connection_errors do
36
+ session.zones.all_zones(account_id).data.map(&:name)
37
+ end
30
38
  end
31
39
 
32
40
  private
@@ -1,15 +1,17 @@
1
+ require_relative '../provider_utils/waiter'
2
+
1
3
  # Patch Dnsimple client method which retrieves headers for API rate limit dynamically
2
4
  module Dnsimple
3
5
  class Client
4
6
  def execute(method, path, data = nil, options = {})
5
7
  response = request(method, path, data, options)
6
- rate_limit_sleep(response.headers["x-ratelimit-reset"].to_i, response.headers["x-ratelimit-remaining"].to_i)
8
+ rate_limit_sleep(response.headers['x-ratelimit-reset'].to_i, response.headers['x-ratelimit-remaining'].to_i)
7
9
 
8
10
  case response.code
9
11
  when 200..299
10
12
  response
11
13
  when 401
12
- raise AuthenticationFailed, response["message"]
14
+ raise AuthenticationFailed, response['message']
13
15
  when 404
14
16
  raise NotFoundError, response
15
17
  else
@@ -22,9 +24,10 @@ module Dnsimple
22
24
  def rate_limit_sleep(rate_limit_reset, rate_limit_remaining)
23
25
  rate_limit_reset_in = [0, rate_limit_reset - Time.now.to_i].max
24
26
  rate_limit_periods = rate_limit_remaining + 1
25
- wait_time = rate_limit_reset_in / rate_limit_periods.to_f
27
+ sleep_time = rate_limit_reset_in / rate_limit_periods.to_f
26
28
 
27
- sleep(wait_time) if wait_time > 0
29
+ rate_limit = RateLimitWaiter.new('DNSimple')
30
+ rate_limit.wait(sleep_time)
28
31
  end
29
32
  end
30
33
  end
@@ -60,14 +60,18 @@ module RecordStore
60
60
 
61
61
  # Returns an array of the zones managed by provider as strings
62
62
  def zones
63
- client.zones.map { |zone| zone['zone'] }
63
+ retry_on_connection_errors do
64
+ client.zones.map { |zone| zone['zone'] }
65
+ end
64
66
  end
65
67
 
66
68
  private
67
69
 
68
70
  # Fetches simplified records for the provided zone
69
71
  def records_for_zone(zone)
70
- client.zone(zone)['records']
72
+ retry_on_connection_errors do
73
+ client.zone(zone)['records']
74
+ end
71
75
  end
72
76
 
73
77
  # Creates a new record to the zone. It is expected this call modifies external state.
@@ -1,11 +1,22 @@
1
1
  require 'net/http'
2
+ require_relative '../provider_utils/waiter'
2
3
 
3
4
  # Patch the method which retrieves headers for API rate limit dynamically
4
5
  module NS1::Transport
5
6
  class NetHttp
7
+ X_RATELIMIT_PERIOD = 'x-ratelimit-period'.freeze
8
+ X_RATELIMIT_REMAINING = 'x-ratelimit-remaining'.freeze
9
+
6
10
  def process_response(response)
7
- sleep(response.to_hash["x-ratelimit-period"].first.to_i /
8
- [1, response.to_hash["x-ratelimit-remaining"].first.to_i].max.to_f)
11
+ response_hash = response.to_hash
12
+
13
+ if response_hash.key?(X_RATELIMIT_PERIOD) && response_hash.key?(X_RATELIMIT_REMAINING)
14
+ sleep_time = response_hash[X_RATELIMIT_PERIOD].first.to_i /
15
+ [1, response_hash[X_RATELIMIT_REMAINING].first.to_i].max.to_f
16
+
17
+ rate_limit = RateLimitWaiter.new('NS1')
18
+ rate_limit.wait(sleep_time)
19
+ end
9
20
 
10
21
  body = JSON.parse(response.body)
11
22
  case response
@@ -0,0 +1,41 @@
1
+ class Waiter
2
+ def initialize(message = nil)
3
+ @message = message || 'Waiting'
4
+ end
5
+
6
+ attr_accessor :message
7
+
8
+ def wait(sleep_time)
9
+ while sleep_time > 0
10
+ wait_time = [10, sleep_time].min
11
+ puts "#{message} (#{sleep_time}s left)" if wait_time > 1
12
+ sleep(wait_time)
13
+ sleep_time -= wait_time
14
+ end
15
+ end
16
+ end
17
+
18
+ class RateLimitWaiter < Waiter
19
+ def initialize(provider)
20
+ super("Waiting on #{provider} rate-limit")
21
+ end
22
+ end
23
+
24
+ class BackoffWaiter < Waiter
25
+ def initialize(message, initial_delay:, multiplier:, max_delay: nil)
26
+ super(message)
27
+
28
+ @initial_delay = @current_delay = initial_delay
29
+ @multiplier = multiplier
30
+ @max_delay = max_delay
31
+ end
32
+
33
+ def reset
34
+ @current_delay = @initial_delay
35
+ end
36
+
37
+ def wait
38
+ super(@current_delay)
39
+ @current_delay = [@current_delay * @multiplier, @max_delay].compact.min
40
+ end
41
+ end
@@ -112,6 +112,10 @@ module RecordStore
112
112
  "[#{type}Record] #{fqdn} #{ttl} IN #{type} #{rdata_txt}"
113
113
  end
114
114
 
115
+ def wildcard?
116
+ fqdn.match?(/^\*\./)
117
+ end
118
+
115
119
  protected
116
120
 
117
121
  def validate_label_length
@@ -1,3 +1,3 @@
1
1
  module RecordStore
2
- VERSION = '6.1.0'.freeze
2
+ VERSION = '6.3.0'.freeze
3
3
  end
@@ -18,6 +18,7 @@ module RecordStore
18
18
  validate :validate_cname_records_dont_point_to_root
19
19
  validate :validate_same_ttl_for_records_sharing_fqdn_and_type
20
20
  validate :validate_provider_can_handle_zone_records
21
+ validate :validate_no_empty_non_terminal
21
22
  validate :validate_can_handle_alias_records
22
23
 
23
24
  class << self
@@ -44,13 +45,18 @@ module RecordStore
44
45
  end
45
46
  end
46
47
 
47
- MAX_PARALLEL_THREADS = 10
48
+ DEFAULT_MAX_PARALLEL_THREADS = 10
49
+
50
+ def max_parallel_threads
51
+ (ENV['RECORD_STORE_MAX_THREADS'] || DEFAULT_MAX_PARALLEL_THREADS).to_i
52
+ end
53
+
48
54
  def modified(verbose: false) # rubocop:disable Lint/UnusedMethodArgument
49
55
  modified_zones = []
50
56
  mutex = Mutex.new
51
57
  zones = all
52
58
 
53
- (1..MAX_PARALLEL_THREADS).map do
59
+ (1..max_parallel_threads).map do
54
60
  Thread.new do
55
61
  current_zone = nil
56
62
  while zones.any?
@@ -258,6 +264,28 @@ module RecordStore
258
264
  end
259
265
  end
260
266
 
267
+ def validate_no_empty_non_terminal
268
+ return unless config.empty_non_terminal_over_wildcard?
269
+
270
+ wildcards = records.select(&:wildcard?).map(&:fqdn).uniq
271
+ wildcards.each do |wildcard|
272
+ suffix = wildcard[1..-1]
273
+
274
+ terminal_records = records.map(&:fqdn)
275
+ .select { |record| record.match?(/^([a-zA-Z0-9\-_]+\.[a-zA-Z0-9\-_])#{Regexp.escape(suffix)}$/) }
276
+ next unless terminal_records.any?
277
+
278
+ intermediate_records = records.map(&:fqdn)
279
+ .select { |record| record.match?(/^([a-zA-Z0-9\-_]+)#{Regexp.escape(suffix)}$/) }
280
+ terminal_records.each do |terminal_record|
281
+ non_terminal = terminal_record.partition('.').last
282
+ errors.add(:records, "found empty non-terminal #{non_terminal} "\
283
+ "(caused by existing records #{wildcard} and #{terminal_record})")\
284
+ unless intermediate_records.include?(non_terminal)
285
+ end
286
+ end
287
+ end
288
+
261
289
  def validate_can_handle_alias_records
262
290
  return unless records.any? { |record| record.is_a?(Record::ALIAS) }
263
291
  return if config.supports_alias?
@@ -24,6 +24,10 @@ module RecordStore
24
24
  end
25
25
  end
26
26
 
27
+ def empty_non_terminal_over_wildcard?
28
+ valid_providers? && providers.any? { |provider| Provider.const_get(provider).empty_non_terminal_over_wildcard? }
29
+ end
30
+
27
31
  def to_hash
28
32
  config_hash = {
29
33
  providers: providers,
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: record_store
3
3
  version: !ruby/object:Gem::Version
4
- version: 6.1.0
4
+ version: 6.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Willem van Bergen
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2020-05-21 00:00:00.000000000 Z
12
+ date: 2020-08-31 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: thor
@@ -362,6 +362,7 @@ files:
362
362
  - lib/record_store/provider/ns1/client.rb
363
363
  - lib/record_store/provider/ns1/patch_api_header.rb
364
364
  - lib/record_store/provider/oracle_cloud_dns.rb
365
+ - lib/record_store/provider/provider_utils/waiter.rb
365
366
  - lib/record_store/record.rb
366
367
  - lib/record_store/record/a.rb
367
368
  - lib/record_store/record/aaaa.rb