elasticgraph-indexer_autoscaler_lambda 0.18.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 5ed020864bd849464587075ad6bd0ed677b65c55967f68a5a4db2609dfd2cc5d
+  data.tar.gz: 8bd8b246b28ddfb7bd78a0fd913b6d22a955f4005559ab126bc88c14399ddbc6
+SHA512:
+  metadata.gz: 2b06675425650e6ce74bd224eaa32de5b52eef9883b286babe94143be0859af67629588af59e4c5824be0e421d7ae286cbd498163c416b8474b96ece186d8c2d
+  data.tar.gz: 83e1a9ce30d8e2428ed29ce3d6246b461ea17e511e98c8b75d2959f889dae2c9414fc22c9866db0d2920baeabf9dddca235696bf72166531792e64de0df13d33

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Block, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1 @@
+# ElasticGraph::IndexerAutoscalerLambda

data/elasticgraph-indexer_autoscaler_lambda.gemspec

@@ -0,0 +1,27 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require_relative "../gemspec_helper"
+
+ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, category: :lambda) do |spec, eg_version|
+  spec.summary = "ElasticGraph gem that monitors OpenSearch CPU utilization to autoscale indexer lambda concurrency."
+
+  spec.add_dependency "elasticgraph-datastore_core", eg_version
+  spec.add_dependency "elasticgraph-lambda_support", eg_version
+
+  spec.add_dependency "aws-sdk-lambda", "~> 1.125"
+  spec.add_dependency "aws-sdk-sqs", "~> 1.80"
+
+  # aws-sdk-sqs requires an XML library be available. On Ruby < 3 it'll use rexml from the standard library but on Ruby 3.0+
+  # we have to add an explicit dependency. It supports ox, oga, libxml, nokogiri or rexml, and of those, ox seems to be the
+  # best choice: it leads benchmarks, is well-maintained, has no dependencies, and is MIT-licensed.
+  spec.add_dependency "ox", "~> 2.14"
+
+  spec.add_development_dependency "elasticgraph-elasticsearch", eg_version
+  spec.add_development_dependency "elasticgraph-opensearch", eg_version
+end

data/lib/elastic_graph/indexer_autoscaler_lambda/concurrency_scaler.rb

@@ -0,0 +1,140 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/indexer_autoscaler_lambda/details_logger"
+
+module ElasticGraph
+  class IndexerAutoscalerLambda
+    class ConcurrencyScaler
+      def initialize(datastore_core:, sqs_client:, lambda_client:)
+        @logger = datastore_core.logger
+        @datastore_core = datastore_core
+        @sqs_client = sqs_client
+        @lambda_client = lambda_client
+      end
+
+      # AWS requires the value be in this range:
+      # https://docs.aws.amazon.com/lambda/latest/api/API_ScalingConfig.html#API_ScalingConfig_Contents
+      MAXIMUM_CONCURRENCY = 1000
+      MINIMUM_CONCURRENCY = 2
+
+      def tune_indexer_concurrency(queue_urls:, min_cpu_target:, max_cpu_target:, event_source_mapping_uuids:)
+        queue_attributes = get_queue_attributes(queue_urls)
+        queue_arns = queue_attributes.fetch(:queue_arns)
+        num_messages = queue_attributes.fetch(:total_messages)
+
+        details_logger = DetailsLogger.new(
+          logger: @logger,
+          queue_arns: queue_arns,
+          queue_urls: queue_urls,
+          min_cpu_target: min_cpu_target,
+          max_cpu_target: max_cpu_target,
+          num_messages: num_messages
+        )
+
+        new_target_concurrency =
+          if num_messages.positive?
+            cpu_utilization = get_max_cpu_utilization
+            cpu_midpoint = (max_cpu_target + min_cpu_target) / 2.0
+
+            current_concurrency = get_total_concurrency(event_source_mapping_uuids)
+
+            if cpu_utilization < min_cpu_target
+              increase_factor = (cpu_midpoint / cpu_utilization).clamp(0.0, 1.5)
+              (current_concurrency * increase_factor).round.tap do |new_concurrency|
+                details_logger.log_increase(
+                  cpu_utilization: cpu_utilization,
+                  current_concurrency: current_concurrency,
+                  new_concurrency: new_concurrency
+                )
+              end
+            elsif cpu_utilization > max_cpu_target
+              decrease_factor = cpu_utilization / cpu_midpoint - 1
+              (current_concurrency - (current_concurrency * decrease_factor)).round.tap do |new_concurrency|
+                details_logger.log_decrease(
+                  cpu_utilization: cpu_utilization,
+                  current_concurrency: current_concurrency,
+                  new_concurrency: new_concurrency
+                )
+              end
+            else
+              details_logger.log_no_change(
+                cpu_utilization: cpu_utilization,
+                current_concurrency: current_concurrency
+              )
+              current_concurrency
+            end
+          else
+            details_logger.log_reset
+            0
+          end
+
+        if new_target_concurrency != current_concurrency
+          update_event_source_mapping(
+            event_source_mapping_uuids: event_source_mapping_uuids,
+            concurrency: new_target_concurrency
+          )
+        end
+      end
+
+      private
+
+      def get_max_cpu_utilization
+        @datastore_core.clients_by_name.values.flat_map do |client|
+          client.get_node_os_stats.fetch("nodes").values.map do |node|
+            node.dig("os", "cpu", "percent")
+          end
+        end.max.to_f
+      end
+
+      def get_queue_attributes(queue_urls)
+        attributes_per_queue = queue_urls.map do |queue_url|
+          @sqs_client.get_queue_attributes(
+            queue_url: queue_url,
+            attribute_names: ["QueueArn", "ApproximateNumberOfMessages"]
+          ).attributes
+        end
+
+        total_messages = attributes_per_queue
+          .map { |attr| Integer(attr.fetch("ApproximateNumberOfMessages")) }
+          .sum
+
+        queue_arns = attributes_per_queue.map { |attr| attr.fetch("QueueArn") }
+
+        {
+          total_messages: total_messages,
+          queue_arns: queue_arns
+        }
+      end
+
+      def get_total_concurrency(event_source_mapping_uuids)
+        event_source_mapping_uuids.map do |event_source_mapping_uuid|
+          @lambda_client.get_event_source_mapping(
+            uuid: event_source_mapping_uuid
+          ).scaling_config.maximum_concurrency
+        end.sum
+      end
+
+      def update_event_source_mapping(concurrency:, event_source_mapping_uuids:)
+        concurrency_per_queue = concurrency / event_source_mapping_uuids.length
+
+        target_concurrency =
+          concurrency_per_queue.clamp(MINIMUM_CONCURRENCY, MAXIMUM_CONCURRENCY)
+
+        event_source_mapping_uuids.map do |event_source_mapping_uuid|
+          @lambda_client.update_event_source_mapping(
+            uuid: event_source_mapping_uuid,
+            scaling_config: {
+              maximum_concurrency: target_concurrency
+            }
+          )
+        end
+      end
+    end
+  end
+end

data/lib/elastic_graph/indexer_autoscaler_lambda/details_logger.rb

@@ -0,0 +1,69 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+module ElasticGraph
+  class IndexerAutoscalerLambda
+    class DetailsLogger
+      def initialize(
+        logger:,
+        queue_arns:,
+        queue_urls:,
+        min_cpu_target:,
+        max_cpu_target:,
+        num_messages:
+      )
+        @logger = logger
+
+        @log_data = {
+          "message_type" => "ConcurrencyScalerResult",
+          "queue_arns" => queue_arns,
+          "queue_urls" => queue_urls,
+          "min_cpu_target" => min_cpu_target,
+          "max_cpu_target" => max_cpu_target,
+          "num_messages" => num_messages
+        }
+      end
+
+      def log_increase(cpu_utilization:, current_concurrency:, new_concurrency:)
+        log_result({
+          "action" => "increase",
+          "cpu_utilization" => cpu_utilization,
+          "current_concurrency" => current_concurrency,
+          "new_concurrency" => new_concurrency
+        })
+      end
+
+      def log_decrease(cpu_utilization:, current_concurrency:, new_concurrency:)
+        log_result({
+          "action" => "decrease",
+          "cpu_utilization" => cpu_utilization,
+          "current_concurrency" => current_concurrency,
+          "new_concurrency" => new_concurrency
+        })
+      end
+
+      def log_no_change(cpu_utilization:, current_concurrency:)
+        log_result({
+          "action" => "no_change",
+          "cpu_utilization" => cpu_utilization,
+          "current_concurrency" => current_concurrency
+        })
+      end
+
+      def log_reset
+        log_result({"action" => "reset"})
+      end
+
+      private
+
+      def log_result(data)
+        @logger.info(@log_data.merge(data))
+      end
+    end
+  end
+end

data/lib/elastic_graph/indexer_autoscaler_lambda/lambda_function.rb

@@ -0,0 +1,35 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/lambda_support/lambda_function"
+
+module ElasticGraph
+  class IndexerAutoscalerLambda
+    class LambdaFunction
+      prepend LambdaSupport::LambdaFunction
+
+      def initialize
+        require "elastic_graph/indexer_autoscaler_lambda"
+
+        @concurrency_scaler = ElasticGraph::IndexerAutoscalerLambda.from_env.concurrency_scaler
+      end
+
+      def handle_request(event:, context:)
+        @concurrency_scaler.tune_indexer_concurrency(
+          queue_urls: event.fetch("queue_urls"),
+          min_cpu_target: event.fetch("min_cpu_target"),
+          max_cpu_target: event.fetch("max_cpu_target"),
+          event_source_mapping_uuids: event.fetch("event_source_mapping_uuids")
+        )
+      end
+    end
+  end
+end
+
+# This is the constant used by the AWS Lambda function.
+AutoscaleIndexer = ElasticGraph::IndexerAutoscalerLambda::LambdaFunction.new

data/lib/elastic_graph/indexer_autoscaler_lambda.rb

@@ -0,0 +1,66 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/datastore_core"
+require "elastic_graph/lambda_support"
+require "elastic_graph/support/from_yaml_file"
+
+module ElasticGraph
+  class IndexerAutoscalerLambda
+    extend Support::FromYamlFile
+
+    # Builds an `ElasticGraph::IndexerAutoscalerLambda` instance from our lambda ENV vars.
+    def self.from_env
+      LambdaSupport.build_from_env(self)
+    end
+
+    # A factory method that builds a IndexerAutoscalerLambda instance from the given parsed YAML config.
+    # `from_yaml_file(file_name, &block)` is also available (via `Support::FromYamlFile`).
+    def self.from_parsed_yaml(parsed_yaml, &datastore_client_customization_block)
+      new(datastore_core: DatastoreCore.from_parsed_yaml(parsed_yaml, for_context: :indexer_autoscaler_lambda, &datastore_client_customization_block))
+    end
+
+    # @dynamic datastore_core
+    attr_reader :datastore_core
+
+    def initialize(
+      datastore_core:,
+      sqs_client: nil,
+      lambda_client: nil
+    )
+      @datastore_core = datastore_core
+      @sqs_client = sqs_client
+      @lambda_client = lambda_client
+    end
+
+    def sqs_client
+      @sqs_client ||= begin
+        require "aws-sdk-sqs"
+        Aws::SQS::Client.new
+      end
+    end
+
+    def lambda_client
+      @lambda_client ||= begin
+        require "aws-sdk-lambda"
+        Aws::Lambda::Client.new
+      end
+    end
+
+    def concurrency_scaler
+      @concurrency_scaler ||= begin
+        require "elastic_graph/indexer_autoscaler_lambda/concurrency_scaler"
+        ConcurrencyScaler.new(
+          datastore_core: @datastore_core,
+          sqs_client: sqs_client,
+          lambda_client: lambda_client
+        )
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,330 @@
+--- !ruby/object:Gem::Specification
+name: elasticgraph-indexer_autoscaler_lambda
+version: !ruby/object:Gem::Version
+  version: 0.18.0.0
+platform: ruby
+authors:
+- Myron Marston
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2024-08-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rubocop-factory_bot
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.26'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.26'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: standard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.39.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.39.0
+- !ruby/object:Gem::Dependency
+  name: steep
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: coderay
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: flatware-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.3.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.3.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.13'
+- !ruby/object:Gem::Dependency
+  name: super_diff
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.1
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.22'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.22'
+- !ruby/object:Gem::Dependency
+  name: simplecov-console
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: aws_lambda_ric
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  name: elasticgraph-datastore_core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+- !ruby/object:Gem::Dependency
+  name: elasticgraph-lambda_support
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-lambda
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.125'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.125'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-sqs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.80'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.80'
+- !ruby/object:Gem::Dependency
+  name: ox
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.14'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.14'
+- !ruby/object:Gem::Dependency
+  name: elasticgraph-elasticsearch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+- !ruby/object:Gem::Dependency
+  name: elasticgraph-opensearch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+description: 
+email:
+- myron@squareup.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- elasticgraph-indexer_autoscaler_lambda.gemspec
+- lib/elastic_graph/indexer_autoscaler_lambda.rb
+- lib/elastic_graph/indexer_autoscaler_lambda/concurrency_scaler.rb
+- lib/elastic_graph/indexer_autoscaler_lambda/details_logger.rb
+- lib/elastic_graph/indexer_autoscaler_lambda/lambda_function.rb
+homepage: 
+licenses:
+- MIT
+metadata:
+  gem_category: lambda
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: '3.2'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.9
+signing_key: 
+specification_version: 4
+summary: ElasticGraph gem that monitors OpenSearch CPU utilization to autoscale indexer
+  lambda concurrency.
+test_files: []