elasticgraph-indexer_autoscaler_lambda 0.18.0.5 → 0.19.0.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/elasticgraph-indexer_autoscaler_lambda.gemspec +1 -0
  3. data/lib/elastic_graph/indexer_autoscaler_lambda/concurrency_scaler.rb +62 -32
  4. data/lib/elastic_graph/indexer_autoscaler_lambda/details_logger.rb +21 -3
  5. data/lib/elastic_graph/indexer_autoscaler_lambda/lambda_function.rb +4 -1
  6. data/lib/elastic_graph/indexer_autoscaler_lambda.rb +12 -2
  7. metadata +39 -20
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 85978cdb432da1408b0510327c96a5e7966713a39f908665c9931f7917723863
4
- data.tar.gz: 827b3fe4ca249c191a1730e88350e8a5e8090be2b99134e2e6572a997b3d5ccd
3
+ metadata.gz: 685736c9754d12dc3f6b15fe111d3ce50b00409a92f5523448df4ea6502e47cd
4
+ data.tar.gz: 32b7cf30ccac9b1dba9dbd1c87c16f385e2f5ee5d9bb92cf2dee80183489bc85
5
5
  SHA512:
6
- metadata.gz: fbb12fe2436dc674389a25eb44683d55c7ba231a7cf6e758c7a1d1b812daa98af69bd116c04351c5ab8e400b752aec4a0528591c5772efa0df1aa6f007f84758
7
- data.tar.gz: 43c355bde2c6b412ab260456b1501f345cc647417c9e5e6acb7a8d7d21e42619c37ecd734faf5b9b0778984f9f09e8b2d671efe43c16960991e1d65d09194adc
6
+ metadata.gz: 5d724c7c83e80723484b66e58d28557c4a658abca00f86eac6722bbf6b8c5c650067599279c0b22d16e3879efb1c855f3b97b7740411ee98d33df8a24d54950c
7
+ data.tar.gz: ce686c78fb6fa9bfdbad29ef8eda03b44b2c36e1ce04119dad55b267402dddd20f8cfffecaefa443a00f4f953f2c0398dc84687d94229a41c4665eefdc884703
data/elasticgraph-indexer_autoscaler_lambda.gemspec CHANGED
@@ -16,6 +16,7 @@ ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, catego
16
16
 
17
17
  spec.add_dependency "aws-sdk-lambda", "~> 1.125"
18
18
  spec.add_dependency "aws-sdk-sqs", "~> 1.80"
19
+ spec.add_dependency "aws-sdk-cloudwatch", "~> 1.104"
19
20
 
20
21
  # aws-sdk-sqs requires an XML library be available. On Ruby < 3 it'll use rexml from the standard library but on Ruby 3.0+
21
22
  # we have to add an explicit dependency. It supports ox, oga, libxml, nokogiri or rexml, and of those, ox seems to be the
data/lib/elastic_graph/indexer_autoscaler_lambda/concurrency_scaler.rb CHANGED
@@ -12,19 +12,25 @@ module ElasticGraph
12
12
  class IndexerAutoscalerLambda
13
13
  # @private
14
14
  class ConcurrencyScaler
15
- def initialize(datastore_core:, sqs_client:, lambda_client:)
15
+ def initialize(datastore_core:, sqs_client:, lambda_client:, cloudwatch_client:)
16
16
  @logger = datastore_core.logger
17
17
  @datastore_core = datastore_core
18
18
  @sqs_client = sqs_client
19
19
  @lambda_client = lambda_client
20
+ @cloudwatch_client = cloudwatch_client
20
21
  end
21
22
 
22
- # AWS requires the value be in this range:
23
- # https://docs.aws.amazon.com/lambda/latest/api/API_ScalingConfig.html#API_ScalingConfig_Contents
24
- MAXIMUM_CONCURRENCY = 1000
25
23
  MINIMUM_CONCURRENCY = 2
26
24
 
27
- def tune_indexer_concurrency(queue_urls:, min_cpu_target:, max_cpu_target:, event_source_mapping_uuids:)
25
+ def tune_indexer_concurrency(
26
+ queue_urls:,
27
+ min_cpu_target:,
28
+ max_cpu_target:,
29
+ maximum_concurrency:,
30
+ required_free_storage_in_mb:,
31
+ indexer_function_name:,
32
+ cluster_name:
33
+ )
28
34
  queue_attributes = get_queue_attributes(queue_urls)
29
35
  queue_arns = queue_attributes.fetch(:queue_arns)
30
36
  num_messages = queue_attributes.fetch(:total_messages)
@@ -40,16 +46,29 @@ module ElasticGraph
40
46
 
41
47
  new_target_concurrency =
42
48
  if num_messages.positive?
49
+ lowest_node_free_storage_in_mb = get_lowest_node_free_storage_in_mb(cluster_name)
50
+
43
51
  cpu_utilization = get_max_cpu_utilization
44
52
  cpu_midpoint = (max_cpu_target + min_cpu_target) / 2.0
45
53
 
46
- current_concurrency = get_total_concurrency(event_source_mapping_uuids)
54
+ current_concurrency = get_concurrency(indexer_function_name)
47
55
 
48
- if cpu_utilization < min_cpu_target
56
+ if current_concurrency.nil?
57
+ details_logger.log_unset
58
+ nil
59
+ elsif lowest_node_free_storage_in_mb < required_free_storage_in_mb
60
+ details_logger.log_pause(
61
+ lowest_node_free_storage_in_mb: lowest_node_free_storage_in_mb,
62
+ required_free_storage_in_mb: required_free_storage_in_mb
63
+ )
64
+ MINIMUM_CONCURRENCY
65
+ elsif cpu_utilization < min_cpu_target
49
66
  increase_factor = (cpu_midpoint / cpu_utilization).clamp(0.0, 1.5)
50
67
  (current_concurrency * increase_factor).round.tap do |new_concurrency|
51
68
  details_logger.log_increase(
52
69
  cpu_utilization: cpu_utilization,
70
+ lowest_node_free_storage_in_mb: lowest_node_free_storage_in_mb,
71
+ required_free_storage_in_mb: required_free_storage_in_mb,
53
72
  current_concurrency: current_concurrency,
54
73
  new_concurrency: new_concurrency
55
74
  )
@@ -59,6 +78,8 @@ module ElasticGraph
59
78
  (current_concurrency - (current_concurrency * decrease_factor)).round.tap do |new_concurrency|
60
79
  details_logger.log_decrease(
61
80
  cpu_utilization: cpu_utilization,
81
+ lowest_node_free_storage_in_mb: lowest_node_free_storage_in_mb,
82
+ required_free_storage_in_mb: required_free_storage_in_mb,
62
83
  current_concurrency: current_concurrency,
63
84
  new_concurrency: new_concurrency
64
85
  )
@@ -66,19 +87,22 @@ module ElasticGraph
66
87
  else
67
88
  details_logger.log_no_change(
68
89
  cpu_utilization: cpu_utilization,
90
+ lowest_node_free_storage_in_mb: lowest_node_free_storage_in_mb,
91
+ required_free_storage_in_mb: required_free_storage_in_mb,
69
92
  current_concurrency: current_concurrency
70
93
  )
71
94
  current_concurrency
72
95
  end
73
96
  else
74
97
  details_logger.log_reset
75
- 0
98
+ MINIMUM_CONCURRENCY
76
99
  end
77
100
 
78
- if new_target_concurrency != current_concurrency
79
- update_event_source_mapping(
80
- event_source_mapping_uuids: event_source_mapping_uuids,
81
- concurrency: new_target_concurrency
101
+ if new_target_concurrency && new_target_concurrency != current_concurrency
102
+ update_concurrency(
103
+ indexer_function_name: indexer_function_name,
104
+ concurrency: new_target_concurrency,
105
+ maximum_concurrency: maximum_concurrency
82
106
  )
83
107
  end
84
108
  end
@@ -93,6 +117,22 @@ module ElasticGraph
93
117
  end.max.to_f
94
118
  end
95
119
 
120
+ def get_lowest_node_free_storage_in_mb(cluster_name)
121
+ metric_response = @cloudwatch_client.get_metric_data({
122
+ start_time: ::Time.now - 1200, # past 20 minutes
123
+ end_time: ::Time.now,
124
+ metric_data_queries: [
125
+ {
126
+ id: "minFreeStorageAcrossNodes",
127
+ expression: "SEARCH('{AWS/ES,ClientId,DomainName} MetricName=\"FreeStorageSpace\" AND DomainName=\"#{cluster_name}\"', 'Minimum', 60)",
128
+ return_data: true
129
+ }
130
+ ]
131
+ })
132
+
133
+ metric_response.metric_data_results.first.values.first
134
+ end
135
+
96
136
  def get_queue_attributes(queue_urls)
97
137
  attributes_per_queue = queue_urls.map do |queue_url|
98
138
  @sqs_client.get_queue_attributes(
@@ -113,28 +153,18 @@ module ElasticGraph
113
153
  }
114
154
  end
115
155
 
116
- def get_total_concurrency(event_source_mapping_uuids)
117
- event_source_mapping_uuids.map do |event_source_mapping_uuid|
118
- @lambda_client.get_event_source_mapping(
119
- uuid: event_source_mapping_uuid
120
- ).scaling_config.maximum_concurrency
121
- end.sum
156
+ def get_concurrency(indexer_function_name)
157
+ @lambda_client.get_function_concurrency(
158
+ function_name: indexer_function_name
159
+ ).reserved_concurrent_executions
122
160
  end
123
161
 
124
- def update_event_source_mapping(concurrency:, event_source_mapping_uuids:)
125
- concurrency_per_queue = concurrency / event_source_mapping_uuids.length
126
-
127
- target_concurrency =
128
- concurrency_per_queue.clamp(MINIMUM_CONCURRENCY, MAXIMUM_CONCURRENCY)
129
-
130
- event_source_mapping_uuids.map do |event_source_mapping_uuid|
131
- @lambda_client.update_event_source_mapping(
132
- uuid: event_source_mapping_uuid,
133
- scaling_config: {
134
- maximum_concurrency: target_concurrency
135
- }
136
- )
137
- end
162
+ def update_concurrency(indexer_function_name:, concurrency:, maximum_concurrency:)
163
+ target_concurrency = concurrency.clamp(MINIMUM_CONCURRENCY, maximum_concurrency)
164
+ @lambda_client.put_function_concurrency(
165
+ function_name: indexer_function_name,
166
+ reserved_concurrent_executions: target_concurrency
167
+ )
138
168
  end
139
169
  end
140
170
  end
data/lib/elastic_graph/indexer_autoscaler_lambda/details_logger.rb CHANGED
@@ -30,36 +30,54 @@ module ElasticGraph
30
30
  }
31
31
  end
32
32
 
33
- def log_increase(cpu_utilization:, current_concurrency:, new_concurrency:)
33
+ def log_increase(cpu_utilization:, lowest_node_free_storage_in_mb:, required_free_storage_in_mb:, current_concurrency:, new_concurrency:)
34
34
  log_result({
35
35
  "action" => "increase",
36
36
  "cpu_utilization" => cpu_utilization,
37
+ "lowest_node_free_storage_in_mb" => lowest_node_free_storage_in_mb,
38
+ "required_free_storage_in_mb" => required_free_storage_in_mb,
37
39
  "current_concurrency" => current_concurrency,
38
40
  "new_concurrency" => new_concurrency
39
41
  })
40
42
  end
41
43
 
42
- def log_decrease(cpu_utilization:, current_concurrency:, new_concurrency:)
44
+ def log_decrease(cpu_utilization:, lowest_node_free_storage_in_mb:, required_free_storage_in_mb:, current_concurrency:, new_concurrency:)
43
45
  log_result({
44
46
  "action" => "decrease",
45
47
  "cpu_utilization" => cpu_utilization,
48
+ "lowest_node_free_storage_in_mb" => lowest_node_free_storage_in_mb,
49
+ "required_free_storage_in_mb" => required_free_storage_in_mb,
46
50
  "current_concurrency" => current_concurrency,
47
51
  "new_concurrency" => new_concurrency
48
52
  })
49
53
  end
50
54
 
51
- def log_no_change(cpu_utilization:, current_concurrency:)
55
+ def log_no_change(cpu_utilization:, lowest_node_free_storage_in_mb:, required_free_storage_in_mb:, current_concurrency:)
52
56
  log_result({
53
57
  "action" => "no_change",
54
58
  "cpu_utilization" => cpu_utilization,
59
+ "lowest_node_free_storage_in_mb" => lowest_node_free_storage_in_mb,
60
+ "required_free_storage_in_mb" => required_free_storage_in_mb,
55
61
  "current_concurrency" => current_concurrency
56
62
  })
57
63
  end
58
64
 
65
+ def log_pause(lowest_node_free_storage_in_mb:, required_free_storage_in_mb:)
66
+ log_result({
67
+ "action" => "pause",
68
+ "lowest_node_free_storage_in_mb" => lowest_node_free_storage_in_mb,
69
+ "required_free_storage_in_mb" => required_free_storage_in_mb
70
+ })
71
+ end
72
+
59
73
  def log_reset
60
74
  log_result({"action" => "reset"})
61
75
  end
62
76
 
77
+ def log_unset
78
+ log_result({"action" => "unset"})
79
+ end
80
+
63
81
  private
64
82
 
65
83
  def log_result(data)
data/lib/elastic_graph/indexer_autoscaler_lambda/lambda_function.rb CHANGED
@@ -25,7 +25,10 @@ module ElasticGraph
25
25
  queue_urls: event.fetch("queue_urls"),
26
26
  min_cpu_target: event.fetch("min_cpu_target"),
27
27
  max_cpu_target: event.fetch("max_cpu_target"),
28
- event_source_mapping_uuids: event.fetch("event_source_mapping_uuids")
28
+ maximum_concurrency: event.fetch("maximum_concurrency"),
29
+ required_free_storage_in_mb: event.fetch("required_free_storage_in_mb"),
30
+ indexer_function_name: event.fetch("indexer_function_name"),
31
+ cluster_name: event.fetch("cluster_name")
29
32
  )
30
33
  end
31
34
  end
data/lib/elastic_graph/indexer_autoscaler_lambda.rb CHANGED
@@ -32,11 +32,13 @@ module ElasticGraph
32
32
  def initialize(
33
33
  datastore_core:,
34
34
  sqs_client: nil,
35
- lambda_client: nil
35
+ lambda_client: nil,
36
+ cloudwatch_client: nil
36
37
  )
37
38
  @datastore_core = datastore_core
38
39
  @sqs_client = sqs_client
39
40
  @lambda_client = lambda_client
41
+ @cloudwatch_client = cloudwatch_client
40
42
  end
41
43
 
42
44
  def sqs_client
@@ -53,13 +55,21 @@ module ElasticGraph
53
55
  end
54
56
  end
55
57
 
58
+ def cloudwatch_client
59
+ @cloudwatch_client ||= begin
60
+ require "aws-sdk-cloudwatch"
61
+ Aws::CloudWatch::Client.new
62
+ end
63
+ end
64
+
56
65
  def concurrency_scaler
57
66
  @concurrency_scaler ||= begin
58
67
  require "elastic_graph/indexer_autoscaler_lambda/concurrency_scaler"
59
68
  ConcurrencyScaler.new(
60
69
  datastore_core: @datastore_core,
61
70
  sqs_client: sqs_client,
62
- lambda_client: lambda_client
71
+ lambda_client: lambda_client,
72
+ cloudwatch_client: cloudwatch_client
63
73
  )
64
74
  end
65
75
  end
metadata CHANGED
@@ -1,16 +1,16 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: elasticgraph-indexer_autoscaler_lambda
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.18.0.5
4
+ version: 0.19.0.0.rc1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Myron Marston
8
8
  - Ben VandenBos
9
- - Square Engineering
9
+ - Block Engineering
10
10
  autorequire:
11
11
  bindir: exe
12
12
  cert_chain: []
13
- date: 2024-09-20 00:00:00.000000000 Z
13
+ date: 2024-12-03 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: rubocop-factory_bot
@@ -46,42 +46,42 @@ dependencies:
46
46
  requirements:
47
47
  - - "~>"
48
48
  - !ruby/object:Gem::Version
49
- version: '3.0'
49
+ version: '3.1'
50
50
  type: :development
51
51
  prerelease: false
52
52
  version_requirements: !ruby/object:Gem::Requirement
53
53
  requirements:
54
54
  - - "~>"
55
55
  - !ruby/object:Gem::Version
56
- version: '3.0'
56
+ version: '3.1'
57
57
  - !ruby/object:Gem::Dependency
58
58
  name: standard
59
59
  requirement: !ruby/object:Gem::Requirement
60
60
  requirements:
61
61
  - - "~>"
62
62
  - !ruby/object:Gem::Version
63
- version: 1.40.0
63
+ version: 1.41.0
64
64
  type: :development
65
65
  prerelease: false
66
66
  version_requirements: !ruby/object:Gem::Requirement
67
67
  requirements:
68
68
  - - "~>"
69
69
  - !ruby/object:Gem::Version
70
- version: 1.40.0
70
+ version: 1.41.0
71
71
  - !ruby/object:Gem::Dependency
72
72
  name: steep
73
73
  requirement: !ruby/object:Gem::Requirement
74
74
  requirements:
75
75
  - - "~>"
76
76
  - !ruby/object:Gem::Version
77
- version: '1.7'
77
+ version: '1.8'
78
78
  type: :development
79
79
  prerelease: false
80
80
  version_requirements: !ruby/object:Gem::Requirement
81
81
  requirements:
82
82
  - - "~>"
83
83
  - !ruby/object:Gem::Version
84
- version: '1.7'
84
+ version: '1.8'
85
85
  - !ruby/object:Gem::Dependency
86
86
  name: coderay
87
87
  requirement: !ruby/object:Gem::Requirement
@@ -136,14 +136,14 @@ dependencies:
136
136
  requirements:
137
137
  - - "~>"
138
138
  - !ruby/object:Gem::Version
139
- version: '0.12'
139
+ version: '0.13'
140
140
  type: :development
141
141
  prerelease: false
142
142
  version_requirements: !ruby/object:Gem::Requirement
143
143
  requirements:
144
144
  - - "~>"
145
145
  - !ruby/object:Gem::Version
146
- version: '0.12'
146
+ version: '0.13'
147
147
  - !ruby/object:Gem::Dependency
148
148
  name: simplecov
149
149
  requirement: !ruby/object:Gem::Requirement
@@ -192,28 +192,28 @@ dependencies:
192
192
  requirements:
193
193
  - - '='
194
194
  - !ruby/object:Gem::Version
195
- version: 0.18.0.5
195
+ version: 0.19.0.0.rc1
196
196
  type: :runtime
197
197
  prerelease: false
198
198
  version_requirements: !ruby/object:Gem::Requirement
199
199
  requirements:
200
200
  - - '='
201
201
  - !ruby/object:Gem::Version
202
- version: 0.18.0.5
202
+ version: 0.19.0.0.rc1
203
203
  - !ruby/object:Gem::Dependency
204
204
  name: elasticgraph-lambda_support
205
205
  requirement: !ruby/object:Gem::Requirement
206
206
  requirements:
207
207
  - - '='
208
208
  - !ruby/object:Gem::Version
209
- version: 0.18.0.5
209
+ version: 0.19.0.0.rc1
210
210
  type: :runtime
211
211
  prerelease: false
212
212
  version_requirements: !ruby/object:Gem::Requirement
213
213
  requirements:
214
214
  - - '='
215
215
  - !ruby/object:Gem::Version
216
- version: 0.18.0.5
216
+ version: 0.19.0.0.rc1
217
217
  - !ruby/object:Gem::Dependency
218
218
  name: aws-sdk-lambda
219
219
  requirement: !ruby/object:Gem::Requirement
@@ -242,6 +242,20 @@ dependencies:
242
242
  - - "~>"
243
243
  - !ruby/object:Gem::Version
244
244
  version: '1.80'
245
+ - !ruby/object:Gem::Dependency
246
+ name: aws-sdk-cloudwatch
247
+ requirement: !ruby/object:Gem::Requirement
248
+ requirements:
249
+ - - "~>"
250
+ - !ruby/object:Gem::Version
251
+ version: '1.104'
252
+ type: :runtime
253
+ prerelease: false
254
+ version_requirements: !ruby/object:Gem::Requirement
255
+ requirements:
256
+ - - "~>"
257
+ - !ruby/object:Gem::Version
258
+ version: '1.104'
245
259
  - !ruby/object:Gem::Dependency
246
260
  name: ox
247
261
  requirement: !ruby/object:Gem::Requirement
@@ -262,28 +276,28 @@ dependencies:
262
276
  requirements:
263
277
  - - '='
264
278
  - !ruby/object:Gem::Version
265
- version: 0.18.0.5
279
+ version: 0.19.0.0.rc1
266
280
  type: :development
267
281
  prerelease: false
268
282
  version_requirements: !ruby/object:Gem::Requirement
269
283
  requirements:
270
284
  - - '='
271
285
  - !ruby/object:Gem::Version
272
- version: 0.18.0.5
286
+ version: 0.19.0.0.rc1
273
287
  - !ruby/object:Gem::Dependency
274
288
  name: elasticgraph-opensearch
275
289
  requirement: !ruby/object:Gem::Requirement
276
290
  requirements:
277
291
  - - '='
278
292
  - !ruby/object:Gem::Version
279
- version: 0.18.0.5
293
+ version: 0.19.0.0.rc1
280
294
  type: :development
281
295
  prerelease: false
282
296
  version_requirements: !ruby/object:Gem::Requirement
283
297
  requirements:
284
298
  - - '='
285
299
  - !ruby/object:Gem::Version
286
- version: 0.18.0.5
300
+ version: 0.19.0.0.rc1
287
301
  description:
288
302
  email:
289
303
  - myron@squareup.com
@@ -298,10 +312,15 @@ files:
298
312
  - lib/elastic_graph/indexer_autoscaler_lambda/concurrency_scaler.rb
299
313
  - lib/elastic_graph/indexer_autoscaler_lambda/details_logger.rb
300
314
  - lib/elastic_graph/indexer_autoscaler_lambda/lambda_function.rb
301
- homepage:
315
+ homepage: https://block.github.io/elasticgraph/
302
316
  licenses:
303
317
  - MIT
304
318
  metadata:
319
+ bug_tracker_uri: https://github.com/block/elasticgraph/issues
320
+ changelog_uri: https://github.com/block/elasticgraph/releases/tag/v0.19.0.0.rc1
321
+ documentation_uri: https://block.github.io/elasticgraph/docs/main/
322
+ homepage_uri: https://block.github.io/elasticgraph/
323
+ source_code_uri: https://github.com/block/elasticgraph/tree/v0.19.0.0.rc1/elasticgraph-indexer_autoscaler_lambda
305
324
  gem_category: lambda
306
325
  post_install_message:
307
326
  rdoc_options: []