ECSD 0.9.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c069cbb5aaee9472442db19e06ac7985d348888fe4792e64f507b1e99d385505
+  data.tar.gz: 20ee8759eb119d57eda9f092b601218b0e4653bc5baf38db83fd4fd97f65926a
+SHA512:
+  metadata.gz: c6983cfa3baaeb1218206de55ee79aa347ffb8066ad93c2c39e7ae343183413780fcef89c4e394eff1d2a20af940a896988965081cae4628293defe818c30791
+  data.tar.gz: c4a1d62a25eb836623cd27c824b5f17e094e90dcc1c4b279e55a7b35a8292242d1f417ca9d40785f43ef719b1269b1306a160c44c4d6b89d6fae40458a073ecd

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+# 0.9.0
+
+See commit log.

data/README.md

@@ -0,0 +1,61 @@
+# ECSD
+
+Simple service for generate ec2_sd config files (one file per cluster) 
+to allow ECS cluster to be discovered by prometheus monitoring service.
+
+Warning! Tested only with EC2 cluster model. 
+Working with fargate cluster model is not guaranteed
+
+## Getting started
+
+First create AWS IAM user with roles for describe and list.
+
+Then add gem to your service:
+```ruby
+gem 'ecsd'
+```
+Or install directly:
+```
+$ gem install ecsd
+```
+
+To start discover ECS cluster define config:
+```ruby
+require 'ecsd'
+
+ECSD.config do |c|
+  c.clusters = %w[cluster_name]
+  c.region = 'region'
+  c.credentials = { AWS_ACCESS_KEY_ID: 'ID', AWS_SECRET_ACCESS_KEY: 'SECRET' }
+  c.options = { 
+    export_path: '/dir/path/to/export/folder',
+    timeout: 101
+  }
+  c.logger = Logger.new($stdout)
+end
+```
+
+- `clusters` - array of cluster names to discover. Same name as ECS cluster
+- `region` - AWS region name where ECS cluster deployed
+- `credentials` - `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` from your IAM user
+- `options`
+    - `export_path` - path to export config folder. Default is `#{service root}/export`
+    - `timeout` - time in seconds between cluster discovery cycles.
+- `logger` - define your logger or default logger will be set
+
+Then start discover:
+```ruby
+ECSD.start
+```
+By default ECSD continuously discovering defined clusters and save templates into
+corresponded files.
+
+Also it's possible to start only ony cycle discover:
+```ruby
+ECSD.start(cycle: false)
+```
+
+## Contributing
+
+[Fork the project](https://github.com/Ad1n/ECSD) and send pull
+requests.

data/lib/ecsd/config.rb

@@ -0,0 +1,59 @@
+# @example
+#   ECSD.config do |c|
+#     c.clusters = %w[cluster0 cluster1 cluster2]
+#     c.region = "aws_region_name"
+#     c.credentials = { AWS_ACCESS_KEY_ID: "ID", AWS_SECRET_ACCESS_KEY: "SECRET" }
+#     c.options = {
+#       export_path: 'dir/path/to/export_config/folder',
+#       timeout: 101
+#     }
+#     c.logger = Logger.new($stdout)
+#   end
+module ECSD
+  unless defined?(CoreConfig)
+    class CoreConfig < Struct.new(:credentials, :region, :clusters, :options, :logger)
+      # @param payload [Hash] { AWS_ACCESS_KEY_ID: 'XXX', AWS_SECRET_ACCESS_KEY: 'XXX' }
+      def credentials=(payload)
+        validate_credentials!(payload)
+        super(::Aws::Credentials.new(
+          payload[:AWS_ACCESS_KEY_ID],
+          payload[:AWS_SECRET_ACCESS_KEY]
+        ))
+      end
+
+      # @param payload [Hash] { export_path: 'path/to/folder', timeout: 0 }
+      def options=(payload)
+        payload[:export_path] ||= ECSD::DEFAULT_EXPORT_PATH
+        payload[:timeout] ||= 0
+        super(payload)
+      end
+
+      private
+
+      # Removing the secret credentials from the default inspect string.
+      # @api private
+      def inspect
+        "#<#{self.class.name} clusters=#{clusters.inspect} region=#{region.inspect} options=#{options.inspect} logger=#{logger.inspect}>"
+      end
+
+      def validate_credentials!(p)
+        raise Config::ValidationError, 'Please, provide AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY' if p.values.size != 2
+        raise Config::ValidationError, 'Invalid AWS credentials' unless p.values.all? String
+      end
+    end
+  end
+
+  def config
+    yield @config if block_given?
+
+    @config
+  end
+  module_function :config
+
+  def logger
+    @config.logger || Logger.new($stdout)
+  end
+  module_function :logger
+
+  @config ||= CoreConfig.new
+end

data/lib/ecsd/constants.rb

@@ -0,0 +1,10 @@
+module ECSD
+  module Constants
+    DEFAULT_METRICS_PATH = '/metrics'.freeze
+    DEFAULT_EXPORT_PATH = (File.expand_path('../..', __dir__) << '/export').freeze
+    LAUNCH_TYPE = 'EC2'.freeze
+    CONNECTIVITY = {
+      connected: 'CONNECTED'
+    }.freeze
+  end
+end

data/lib/ecsd/core.rb

@@ -0,0 +1,60 @@
+module ECSD
+  class Core
+    include Helpers::EC2
+    include Helpers::ECS
+    include Helpers::Common
+
+    attr_reader :ecs, :ec2, :cluster
+
+    def initialize(cluster_name)
+      @ecs = ::Aws::ECS::Client.new(
+        credentials: ECSD.config.credentials,
+        region: ECSD.config.region
+      )
+      @ec2 = ::Aws::EC2::Client.new(
+        credentials: ECSD.config.credentials,
+        region: ECSD.config.region
+      )
+      @cluster = cluster_name
+    end
+
+    def handle!
+      task_arns = task_arns(cluster)
+      tasks = connected_tasks(cluster, task_arns)
+      handle_tasks(tasks)
+    end
+
+    def handle_tasks(tasks)
+      ECSD.logger.info <<~TEXT
+
+        Handle cluster: [#{cluster}]
+        Started at: #{Time.now}
+      TEXT
+      tasks.map do |t|
+        instance_id = instance_id(cluster, t)
+        task_definition_name, task_revision = task_definition_data(t)
+        task_container = task_container(t)
+
+        TEMPLATE.call(
+          ip_addr: instance_ip_addr(instance_id),
+          dyn_port: dyn_port(task_container),
+          task_id: task_id(t),
+          task_definition_name: task_definition_name,
+          task_revision: task_revision,
+          cluster_name: cluster_name(t),
+          container_name: task_container.name,
+          instance_id: instance_id
+        )
+      end.then do |r|
+        write_to_file(cluster, r)
+
+        ECSD.logger.info <<~TEXT
+
+          Handled cluster: [#{cluster}]
+          Tasks in cluster: #{r.count}
+          Finished at: #{Time.now}
+        TEXT
+      end
+    end
+  end
+end

data/lib/ecsd/errors.rb

@@ -0,0 +1,7 @@
+module ECSD
+  class BaseError < RuntimeError; end
+
+  class Config
+    class ValidationError < BaseError; end
+  end
+end

data/lib/ecsd/helpers/common.rb

@@ -0,0 +1,40 @@
+module ECSD
+  module Helpers
+    module Common
+      def write_to_file(cluster, data, path = ECSD.config.options[:export_path] || DEFAULT_EXPORT_PATH)
+        File.open("#{path}/#{cluster.downcase}.yml", 'w') do |f|
+          f.write(data.join)
+        end
+      end
+
+      def task_id(task)
+        task.task_arn
+            .split('/')
+            .last
+      end
+
+      def task_definition_data(task)
+        task.task_definition_arn
+            .split('/')
+            .last
+            .split(':')
+      end
+
+      def cluster_name(task)
+        task.cluster_arn
+            .split('/')
+            .last
+      end
+
+      def task_container(task)
+        task.containers.find { |c| c.task_arn == task.task_arn }
+      end
+
+      def dyn_port(task_container)
+        task_container.network_bindings
+                      .first
+                      .host_port
+      end
+    end
+  end
+end

data/lib/ecsd/helpers/ec2.rb

@@ -0,0 +1,17 @@
+module ECSD
+  module Helpers
+    module EC2
+      # @param instance_id [String]
+      # @return [String] private ip
+      def instance_ip_addr(instance_id)
+        ec2.describe_instances(instance_ids: Array(instance_id))
+           .data
+           .reservations
+           .first
+           .instances
+           .first
+           .private_ip_address
+      end
+    end
+  end
+end

data/lib/ecsd/helpers/ecs.rb

@@ -0,0 +1,38 @@
+module ECSD
+  module Helpers
+    module ECS
+      # @param cluster [String] cluster name
+      # @return [Array<Array<String>>] batched task_arns by 100
+      def task_arns(cluster)
+        ecs.list_tasks(cluster: cluster).each.with_object([]) do |response, task_arns|
+          task_arns << response[:task_arns]
+        end
+      end
+
+      # @param cluster [String] cluster name
+      # @param task_arns [Array<Array<String>>] batched task_arns by 100
+      # @return [Array<Array<Aws::ECS::Types:Task>>] connected tasks
+      def connected_tasks(cluster, task_arns)
+        task_arns.each.with_object([]) do |pt_batch, connected_tasks|
+          connected_tasks << ecs.describe_tasks(cluster: cluster, tasks: pt_batch)
+                                .data
+                                .tasks
+                                .select { |t| t.connectivity == ECSD::Constants::CONNECTIVITY[:connected] }
+        end.flatten
+      end
+
+      # @param cluster [String] cluster name
+      # @param task [Aws::ECS::Types::Task]
+      # @return [String]
+      def instance_id(cluster, task)
+        ecs.describe_container_instances(
+          cluster: cluster,
+          container_instances: Array(task.container_instance_arn)
+        ).data
+           .container_instances
+           .first
+           .ec2_instance_id
+      end
+    end
+  end
+end

data/lib/ecsd/runner.rb

@@ -0,0 +1,35 @@
+module ECSD
+  # @info Start ECS clusters discovery
+  def start(cycle: :infinite)
+    case cycle
+    when :infinite
+      loop { invoke!; wait }
+    else
+      invoke!
+    end
+  end
+  module_function :start
+
+  private
+
+  # @info one discovery cycle invocation
+  def invoke!
+    config.clusters.each do |cluster|
+      core(cluster).handle!
+    end
+  end
+  module_function :invoke!
+
+  # @info Initialize core component
+  # @param cluster [String] cluster name
+  def core(cluster)
+    ECSD::Core.new(cluster)
+  end
+  module_function :core
+
+  # @info Time in seconds between discovery iterations
+  def wait
+    sleep ECSD.config.options[:timeout]
+  end
+  module_function :wait
+end

data/lib/ecsd/template.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module ECSD
+  TEMPLATE = lambda do |ip_addr:,
+                        dyn_port:,
+                        task_id:,
+                        task_definition_name:,
+                        task_revision:,
+                        cluster_name:,
+                        container_name:,
+                        instance_id:,
+                        metrics_path: DEFAULT_METRICS_PATH|
+    <<~TEXT
+      - targets:
+        - #{ip_addr}:#{dyn_port}
+        labels:
+          task_id: #{task_id}
+          task_definition_name: #{task_definition_name}
+          task_revision: #{task_revision}
+          cluster_name: #{cluster_name}
+          container_name: #{container_name}
+          instance_id: #{instance_id}
+          __metrics_path__: #{metrics_path}
+    TEXT
+  end
+end

data/lib/ecsd/version.rb

@@ -0,0 +1,3 @@
+module ECSD
+  VERSION = '0.9.0'.freeze
+end

data/lib/ecsd.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-ecs'
+require 'aws-sdk-ec2'
+
+require_relative 'ecsd/version'
+require_relative 'ecsd/errors'
+require_relative 'ecsd/constants'
+require_relative 'ecsd/template'
+require_relative 'ecsd/config'
+require_relative 'ecsd/helpers/ec2'
+require_relative 'ecsd/helpers/ecs'
+require_relative 'ecsd/helpers/common'
+require_relative 'ecsd/runner'
+require_relative 'ecsd/core'
+
+#:nodoc
+module ECSD
+  include Constants
+end

metadata

@@ -0,0 +1,84 @@
+--- !ruby/object:Gem::Specification
+name: ECSD
+version: !ruby/object:Gem::Version
+  version: 0.9.0
+platform: ruby
+authors:
+- Anton Shevtsov
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2022-02-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-ec2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.298'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.298'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-ecs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.95'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.95'
+description: Generate ec2 yaml config files to allow ECS cluster to be discovered
+  by prometheus monitoring service
+email: shevtsovav@bk.ru
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- README.md
+- lib/ecsd.rb
+- lib/ecsd/config.rb
+- lib/ecsd/constants.rb
+- lib/ecsd/core.rb
+- lib/ecsd/errors.rb
+- lib/ecsd/helpers/common.rb
+- lib/ecsd/helpers/ec2.rb
+- lib/ecsd/helpers/ecs.rb
+- lib/ecsd/runner.rb
+- lib/ecsd/template.rb
+- lib/ecsd/version.rb
+homepage:
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.4.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.4
+signing_key:
+specification_version: 4
+summary: ECS(EC2 model) discovery
+test_files: []