elasticgraph-indexer_autoscaler_lambda 0.18.0.5 → 0.19.0.0.rc2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/elasticgraph-indexer_autoscaler_lambda.gemspec +1 -0
  3. data/lib/elastic_graph/indexer_autoscaler_lambda/concurrency_scaler.rb +62 -32
  4. data/lib/elastic_graph/indexer_autoscaler_lambda/details_logger.rb +21 -3
  5. data/lib/elastic_graph/indexer_autoscaler_lambda/lambda_function.rb +4 -1
  6. data/lib/elastic_graph/indexer_autoscaler_lambda.rb +12 -2
  7. metadata +39 -20
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 85978cdb432da1408b0510327c96a5e7966713a39f908665c9931f7917723863
4
- data.tar.gz: 827b3fe4ca249c191a1730e88350e8a5e8090be2b99134e2e6572a997b3d5ccd
3
+ metadata.gz: 61d9c146526099b1791d6ff68b7b6165abc9cb1cf6d067c8ad89ab4a1d1e28e0
4
+ data.tar.gz: b1ffa30773f7a99288e5c30493144d293fa47daeef8395fe53ac039b5f3fda17
5
5
  SHA512:
6
- metadata.gz: fbb12fe2436dc674389a25eb44683d55c7ba231a7cf6e758c7a1d1b812daa98af69bd116c04351c5ab8e400b752aec4a0528591c5772efa0df1aa6f007f84758
7
- data.tar.gz: 43c355bde2c6b412ab260456b1501f345cc647417c9e5e6acb7a8d7d21e42619c37ecd734faf5b9b0778984f9f09e8b2d671efe43c16960991e1d65d09194adc
6
+ metadata.gz: a63056f5178352e0ab4495268778f6da5a7991d782a41902203006fbb91ad0c11ee0fe4370750d7781388a3685f74b78337870140b2a735b1488496751e384fb
7
+ data.tar.gz: b71908de2119a50bda919988cc7b0e8403243f81f7c52d471274d26957527302276e49457079dac4db171d7b9d0d7c0acbc29421fb1b9f566bfa3757492ab5c4
data/elasticgraph-indexer_autoscaler_lambda.gemspec CHANGED
@@ -16,6 +16,7 @@ ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, catego
16
16
 
17
17
  spec.add_dependency "aws-sdk-lambda", "~> 1.125"
18
18
  spec.add_dependency "aws-sdk-sqs", "~> 1.80"
19
+ spec.add_dependency "aws-sdk-cloudwatch", "~> 1.104"
19
20
 
20
21
  # aws-sdk-sqs requires an XML library be available. On Ruby < 3 it'll use rexml from the standard library but on Ruby 3.0+
21
22
  # we have to add an explicit dependency. It supports ox, oga, libxml, nokogiri or rexml, and of those, ox seems to be the
data/lib/elastic_graph/indexer_autoscaler_lambda/concurrency_scaler.rb CHANGED
@@ -12,19 +12,25 @@ module ElasticGraph
12
12
  class IndexerAutoscalerLambda
13
13
  # @private
14
14
  class ConcurrencyScaler
15
- def initialize(datastore_core:, sqs_client:, lambda_client:)
15
+ def initialize(datastore_core:, sqs_client:, lambda_client:, cloudwatch_client:)
16
16
  @logger = datastore_core.logger
17
17
  @datastore_core = datastore_core
18
18
  @sqs_client = sqs_client
19
19
  @lambda_client = lambda_client
20
+ @cloudwatch_client = cloudwatch_client
20
21
  end
21
22
 
22
- # AWS requires the value be in this range:
23
- # https://docs.aws.amazon.com/lambda/latest/api/API_ScalingConfig.html#API_ScalingConfig_Contents
24
- MAXIMUM_CONCURRENCY = 1000
25
23
  MINIMUM_CONCURRENCY = 2
26
24
 
27
- def tune_indexer_concurrency(queue_urls:, min_cpu_target:, max_cpu_target:, event_source_mapping_uuids:)
25
+ def tune_indexer_concurrency(
26
+ queue_urls:,
27
+ min_cpu_target:,
28
+ max_cpu_target:,
29
+ maximum_concurrency:,
30
+ required_free_storage_in_mb:,
31
+ indexer_function_name:,
32
+ cluster_name:
33
+ )
28
34
  queue_attributes = get_queue_attributes(queue_urls)
29
35
  queue_arns = queue_attributes.fetch(:queue_arns)
30
36
  num_messages = queue_attributes.fetch(:total_messages)
@@ -40,16 +46,29 @@ module ElasticGraph
40
46
 
41
47
  new_target_concurrency =
42
48
  if num_messages.positive?
49
+ lowest_node_free_storage_in_mb = get_lowest_node_free_storage_in_mb(cluster_name)
50
+
43
51
  cpu_utilization = get_max_cpu_utilization
44
52
  cpu_midpoint = (max_cpu_target + min_cpu_target) / 2.0
45
53
 
46
- current_concurrency = get_total_concurrency(event_source_mapping_uuids)
54
+ current_concurrency = get_concurrency(indexer_function_name)
47
55
 
48
- if cpu_utilization < min_cpu_target
56
+ if current_concurrency.nil?
57
+ details_logger.log_unset
58
+ nil
59
+ elsif lowest_node_free_storage_in_mb < required_free_storage_in_mb
60
+ details_logger.log_pause(
61
+ lowest_node_free_storage_in_mb: lowest_node_free_storage_in_mb,
62
+ required_free_storage_in_mb: required_free_storage_in_mb
63
+ )
64
+ MINIMUM_CONCURRENCY
65
+ elsif cpu_utilization < min_cpu_target
49
66
  increase_factor = (cpu_midpoint / cpu_utilization).clamp(0.0, 1.5)
50
67
  (current_concurrency * increase_factor).round.tap do |new_concurrency|
51
68
  details_logger.log_increase(
52
69
  cpu_utilization: cpu_utilization,
70
+ lowest_node_free_storage_in_mb: lowest_node_free_storage_in_mb,
71
+ required_free_storage_in_mb: required_free_storage_in_mb,
53
72
  current_concurrency: current_concurrency,
54
73
  new_concurrency: new_concurrency
55
74
  )
@@ -59,6 +78,8 @@ module ElasticGraph
59
78
  (current_concurrency - (current_concurrency * decrease_factor)).round.tap do |new_concurrency|
60
79
  details_logger.log_decrease(
61
80
  cpu_utilization: cpu_utilization,
81
+ lowest_node_free_storage_in_mb: lowest_node_free_storage_in_mb,
82
+ required_free_storage_in_mb: required_free_storage_in_mb,
62
83
  current_concurrency: current_concurrency,
63
84
  new_concurrency: new_concurrency
64
85
  )
@@ -66,19 +87,22 @@ module ElasticGraph
66
87
  else
67
88
  details_logger.log_no_change(
68
89
  cpu_utilization: cpu_utilization,
90
+ lowest_node_free_storage_in_mb: lowest_node_free_storage_in_mb,
91
+ required_free_storage_in_mb: required_free_storage_in_mb,
69
92
  current_concurrency: current_concurrency
70
93
  )
71
94
  current_concurrency
72
95
  end
73
96
  else
74
97
  details_logger.log_reset
75
- 0
98
+ MINIMUM_CONCURRENCY
76
99
  end
77
100
 
78
- if new_target_concurrency != current_concurrency
79
- update_event_source_mapping(
80
- event_source_mapping_uuids: event_source_mapping_uuids,
81
- concurrency: new_target_concurrency
101
+ if new_target_concurrency && new_target_concurrency != current_concurrency
102
+ update_concurrency(
103
+ indexer_function_name: indexer_function_name,
104
+ concurrency: new_target_concurrency,
105
+ maximum_concurrency: maximum_concurrency
82
106
  )
83
107
  end
84
108
  end
@@ -93,6 +117,22 @@ module ElasticGraph
93
117
  end.max.to_f
94
118
  end
95
119
 
120
+ def get_lowest_node_free_storage_in_mb(cluster_name)
121
+ metric_response = @cloudwatch_client.get_metric_data({
122
+ start_time: ::Time.now - 1200, # past 20 minutes
123
+ end_time: ::Time.now,
124
+ metric_data_queries: [
125
+ {
126
+ id: "minFreeStorageAcrossNodes",
127
+ expression: "SEARCH('{AWS/ES,ClientId,DomainName} MetricName=\"FreeStorageSpace\" AND DomainName=\"#{cluster_name}\"', 'Minimum', 60)",
128
+ return_data: true
129
+ }
130
+ ]
131
+ })
132
+
133
+ metric_response.metric_data_results.first.values.first
134
+ end
135
+
96
136
  def get_queue_attributes(queue_urls)
97
137
  attributes_per_queue = queue_urls.map do |queue_url|
98
138
  @sqs_client.get_queue_attributes(
@@ -113,28 +153,18 @@ module ElasticGraph
113
153
  }
114
154
  end
115
155
 
116
- def get_total_concurrency(event_source_mapping_uuids)
117
- event_source_mapping_uuids.map do |event_source_mapping_uuid|
118
- @lambda_client.get_event_source_mapping(
119
- uuid: event_source_mapping_uuid
120
- ).scaling_config.maximum_concurrency
121
- end.sum
156
+ def get_concurrency(indexer_function_name)
157
+ @lambda_client.get_function_concurrency(
158
+ function_name: indexer_function_name
159
+ ).reserved_concurrent_executions
122
160
  end
123
161
 
124
- def update_event_source_mapping(concurrency:, event_source_mapping_uuids:)
125
- concurrency_per_queue = concurrency / event_source_mapping_uuids.length
126
-
127
- target_concurrency =
128
- concurrency_per_queue.clamp(MINIMUM_CONCURRENCY, MAXIMUM_CONCURRENCY)
129
-
130
- event_source_mapping_uuids.map do |event_source_mapping_uuid|
131
- @lambda_client.update_event_source_mapping(
132
- uuid: event_source_mapping_uuid,
133
- scaling_config: {
134
- maximum_concurrency: target_concurrency
135
- }
136
- )
137
- end
162
+ def update_concurrency(indexer_function_name:, concurrency:, maximum_concurrency:)
163
+ target_concurrency = concurrency.clamp(MINIMUM_CONCURRENCY, maximum_concurrency)
164
+ @lambda_client.put_function_concurrency(
165
+ function_name: indexer_function_name,
166
+ reserved_concurrent_executions: target_concurrency
167
+ )
138
168
  end
139
169
  end
140
170
  end
data/lib/elastic_graph/indexer_autoscaler_lambda/details_logger.rb CHANGED
@@ -30,36 +30,54 @@ module ElasticGraph
30
30
  }
31
31
  end
32
32
 
33
- def log_increase(cpu_utilization:, current_concurrency:, new_concurrency:)
33
+ def log_increase(cpu_utilization:, lowest_node_free_storage_in_mb:, required_free_storage_in_mb:, current_concurrency:, new_concurrency:)
34
34
  log_result({
35
35
  "action" => "increase",
36
36
  "cpu_utilization" => cpu_utilization,
37
+ "lowest_node_free_storage_in_mb" => lowest_node_free_storage_in_mb,
38
+ "required_free_storage_in_mb" => required_free_storage_in_mb,
37
39
  "current_concurrency" => current_concurrency,
38
40
  "new_concurrency" => new_concurrency
39
41
  })
40
42
  end
41
43
 
42
- def log_decrease(cpu_utilization:, current_concurrency:, new_concurrency:)
44
+ def log_decrease(cpu_utilization:, lowest_node_free_storage_in_mb:, required_free_storage_in_mb:, current_concurrency:, new_concurrency:)
43
45
  log_result({
44
46
  "action" => "decrease",
45
47
  "cpu_utilization" => cpu_utilization,
48
+ "lowest_node_free_storage_in_mb" => lowest_node_free_storage_in_mb,
49
+ "required_free_storage_in_mb" => required_free_storage_in_mb,
46
50
  "current_concurrency" => current_concurrency,
47
51
  "new_concurrency" => new_concurrency
48
52
  })
49
53
  end
50
54
 
51
- def log_no_change(cpu_utilization:, current_concurrency:)
55
+ def log_no_change(cpu_utilization:, lowest_node_free_storage_in_mb:, required_free_storage_in_mb:, current_concurrency:)
52
56
  log_result({
53
57
  "action" => "no_change",
54
58
  "cpu_utilization" => cpu_utilization,
59
+ "lowest_node_free_storage_in_mb" => lowest_node_free_storage_in_mb,
60
+ "required_free_storage_in_mb" => required_free_storage_in_mb,
55
61
  "current_concurrency" => current_concurrency
56
62
  })
57
63
  end
58
64
 
65
+ def log_pause(lowest_node_free_storage_in_mb:, required_free_storage_in_mb:)
66
+ log_result({
67
+ "action" => "pause",
68
+ "lowest_node_free_storage_in_mb" => lowest_node_free_storage_in_mb,
69
+ "required_free_storage_in_mb" => required_free_storage_in_mb
70
+ })
71
+ end
72
+
59
73
  def log_reset
60
74
  log_result({"action" => "reset"})
61
75
  end
62
76
 
77
+ def log_unset
78
+ log_result({"action" => "unset"})
79
+ end
80
+
63
81
  private
64
82
 
65
83
  def log_result(data)
data/lib/elastic_graph/indexer_autoscaler_lambda/lambda_function.rb CHANGED
@@ -25,7 +25,10 @@ module ElasticGraph
25
25
  queue_urls: event.fetch("queue_urls"),
26
26
  min_cpu_target: event.fetch("min_cpu_target"),
27
27
  max_cpu_target: event.fetch("max_cpu_target"),
28
- event_source_mapping_uuids: event.fetch("event_source_mapping_uuids")
28
+ maximum_concurrency: event.fetch("maximum_concurrency"),
29
+ required_free_storage_in_mb: event.fetch("required_free_storage_in_mb"),
30
+ indexer_function_name: event.fetch("indexer_function_name"),
31
+ cluster_name: event.fetch("cluster_name")
29
32
  )
30
33
  end
31
34
  end
data/lib/elastic_graph/indexer_autoscaler_lambda.rb CHANGED
@@ -32,11 +32,13 @@ module ElasticGraph
32
32
  def initialize(
33
33
  datastore_core:,
34
34
  sqs_client: nil,
35
- lambda_client: nil
35
+ lambda_client: nil,
36
+ cloudwatch_client: nil
36
37
  )
37
38
  @datastore_core = datastore_core
38
39
  @sqs_client = sqs_client
39
40
  @lambda_client = lambda_client
41
+ @cloudwatch_client = cloudwatch_client
40
42
  end
41
43
 
42
44
  def sqs_client
@@ -53,13 +55,21 @@ module ElasticGraph
53
55
  end
54
56
  end
55
57
 
58
+ def cloudwatch_client
59
+ @cloudwatch_client ||= begin
60
+ require "aws-sdk-cloudwatch"
61
+ Aws::CloudWatch::Client.new
62
+ end
63
+ end
64
+
56
65
  def concurrency_scaler
57
66
  @concurrency_scaler ||= begin
58
67
  require "elastic_graph/indexer_autoscaler_lambda/concurrency_scaler"
59
68
  ConcurrencyScaler.new(
60
69
  datastore_core: @datastore_core,
61
70
  sqs_client: sqs_client,
62
- lambda_client: lambda_client
71
+ lambda_client: lambda_client,
72
+ cloudwatch_client: cloudwatch_client
63
73
  )
64
74
  end
65
75
  end
metadata CHANGED
@@ -1,16 +1,16 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: elasticgraph-indexer_autoscaler_lambda
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.18.0.5
4
+ version: 0.19.0.0.rc2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Myron Marston
8
8
  - Ben VandenBos
9
- - Square Engineering
9
+ - Block Engineering
10
10
  autorequire:
11
11
  bindir: exe
12
12
  cert_chain: []
13
- date: 2024-09-20 00:00:00.000000000 Z
13
+ date: 2024-12-03 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: rubocop-factory_bot
@@ -46,42 +46,42 @@ dependencies:
46
46
  requirements:
47
47
  - - "~>"
48
48
  - !ruby/object:Gem::Version
49
- version: '3.0'
49
+ version: '3.1'
50
50
  type: :development
51
51
  prerelease: false
52
52
  version_requirements: !ruby/object:Gem::Requirement
53
53
  requirements:
54
54
  - - "~>"
55
55
  - !ruby/object:Gem::Version
56
- version: '3.0'
56
+ version: '3.1'
57
57
  - !ruby/object:Gem::Dependency
58
58
  name: standard
59
59
  requirement: !ruby/object:Gem::Requirement
60
60
  requirements:
61
61
  - - "~>"
62
62
  - !ruby/object:Gem::Version
63
- version: 1.40.0
63
+ version: 1.41.0
64
64
  type: :development
65
65
  prerelease: false
66
66
  version_requirements: !ruby/object:Gem::Requirement
67
67
  requirements:
68
68
  - - "~>"
69
69
  - !ruby/object:Gem::Version
70
- version: 1.40.0
70
+ version: 1.41.0
71
71
  - !ruby/object:Gem::Dependency
72
72
  name: steep
73
73
  requirement: !ruby/object:Gem::Requirement
74
74
  requirements:
75
75
  - - "~>"
76
76
  - !ruby/object:Gem::Version
77
- version: '1.7'
77
+ version: '1.8'
78
78
  type: :development
79
79
  prerelease: false
80
80
  version_requirements: !ruby/object:Gem::Requirement
81
81
  requirements:
82
82
  - - "~>"
83
83
  - !ruby/object:Gem::Version
84
- version: '1.7'
84
+ version: '1.8'
85
85
  - !ruby/object:Gem::Dependency
86
86
  name: coderay
87
87
  requirement: !ruby/object:Gem::Requirement
@@ -136,14 +136,14 @@ dependencies:
136
136
  requirements:
137
137
  - - "~>"
138
138
  - !ruby/object:Gem::Version
139
- version: '0.12'
139
+ version: '0.13'
140
140
  type: :development
141
141
  prerelease: false
142
142
  version_requirements: !ruby/object:Gem::Requirement
143
143
  requirements:
144
144
  - - "~>"
145
145
  - !ruby/object:Gem::Version
146
- version: '0.12'
146
+ version: '0.13'
147
147
  - !ruby/object:Gem::Dependency
148
148
  name: simplecov
149
149
  requirement: !ruby/object:Gem::Requirement
@@ -192,28 +192,28 @@ dependencies:
192
192
  requirements:
193
193
  - - '='
194
194
  - !ruby/object:Gem::Version
195
- version: 0.18.0.5
195
+ version: 0.19.0.0.rc2
196
196
  type: :runtime
197
197
  prerelease: false
198
198
  version_requirements: !ruby/object:Gem::Requirement
199
199
  requirements:
200
200
  - - '='
201
201
  - !ruby/object:Gem::Version
202
- version: 0.18.0.5
202
+ version: 0.19.0.0.rc2
203
203
  - !ruby/object:Gem::Dependency
204
204
  name: elasticgraph-lambda_support
205
205
  requirement: !ruby/object:Gem::Requirement
206
206
  requirements:
207
207
  - - '='
208
208
  - !ruby/object:Gem::Version
209
- version: 0.18.0.5
209
+ version: 0.19.0.0.rc2
210
210
  type: :runtime
211
211
  prerelease: false
212
212
  version_requirements: !ruby/object:Gem::Requirement
213
213
  requirements:
214
214
  - - '='
215
215
  - !ruby/object:Gem::Version
216
- version: 0.18.0.5
216
+ version: 0.19.0.0.rc2
217
217
  - !ruby/object:Gem::Dependency
218
218
  name: aws-sdk-lambda
219
219
  requirement: !ruby/object:Gem::Requirement
@@ -242,6 +242,20 @@ dependencies:
242
242
  - - "~>"
243
243
  - !ruby/object:Gem::Version
244
244
  version: '1.80'
245
+ - !ruby/object:Gem::Dependency
246
+ name: aws-sdk-cloudwatch
247
+ requirement: !ruby/object:Gem::Requirement
248
+ requirements:
249
+ - - "~>"
250
+ - !ruby/object:Gem::Version
251
+ version: '1.104'
252
+ type: :runtime
253
+ prerelease: false
254
+ version_requirements: !ruby/object:Gem::Requirement
255
+ requirements:
256
+ - - "~>"
257
+ - !ruby/object:Gem::Version
258
+ version: '1.104'
245
259
  - !ruby/object:Gem::Dependency
246
260
  name: ox
247
261
  requirement: !ruby/object:Gem::Requirement
@@ -262,28 +276,28 @@ dependencies:
262
276
  requirements:
263
277
  - - '='
264
278
  - !ruby/object:Gem::Version
265
- version: 0.18.0.5
279
+ version: 0.19.0.0.rc2
266
280
  type: :development
267
281
  prerelease: false
268
282
  version_requirements: !ruby/object:Gem::Requirement
269
283
  requirements:
270
284
  - - '='
271
285
  - !ruby/object:Gem::Version
272
- version: 0.18.0.5
286
+ version: 0.19.0.0.rc2
273
287
  - !ruby/object:Gem::Dependency
274
288
  name: elasticgraph-opensearch
275
289
  requirement: !ruby/object:Gem::Requirement
276
290
  requirements:
277
291
  - - '='
278
292
  - !ruby/object:Gem::Version
279
- version: 0.18.0.5
293
+ version: 0.19.0.0.rc2
280
294
  type: :development
281
295
  prerelease: false
282
296
  version_requirements: !ruby/object:Gem::Requirement
283
297
  requirements:
284
298
  - - '='
285
299
  - !ruby/object:Gem::Version
286
- version: 0.18.0.5
300
+ version: 0.19.0.0.rc2
287
301
  description:
288
302
  email:
289
303
  - myron@squareup.com
@@ -298,10 +312,15 @@ files:
298
312
  - lib/elastic_graph/indexer_autoscaler_lambda/concurrency_scaler.rb
299
313
  - lib/elastic_graph/indexer_autoscaler_lambda/details_logger.rb
300
314
  - lib/elastic_graph/indexer_autoscaler_lambda/lambda_function.rb
301
- homepage:
315
+ homepage: https://block.github.io/elasticgraph/
302
316
  licenses:
303
317
  - MIT
304
318
  metadata:
319
+ bug_tracker_uri: https://github.com/block/elasticgraph/issues
320
+ changelog_uri: https://github.com/block/elasticgraph/releases/tag/v0.19.0.0.rc2
321
+ documentation_uri: https://block.github.io/elasticgraph/docs/main/
322
+ homepage_uri: https://block.github.io/elasticgraph/
323
+ source_code_uri: https://github.com/block/elasticgraph/tree/v0.19.0.0.rc2/elasticgraph-indexer_autoscaler_lambda
305
324
  gem_category: lambda
306
325
  post_install_message:
307
326
  rdoc_options: []