aho 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 68395a8675b36a6ef2e9f76ac6c42a5ef69fa08a27bb7da3f1ffca0ee6b3c0dc
-  data.tar.gz: 65a69f4031fea00f45ccb12e500430b2aada64404934405383c8fc5be7243331
+  metadata.gz: f8c2812dff97476d7b3067cef5d3d083dd4c01914d3d2aa13b5ef0d51bdc6268
+  data.tar.gz: d6c0a70dd6833fd1c22bc3c3070cab05d159507dfa1b83bb19d3b16f63b01faf
 SHA512:
-  metadata.gz: f22afb4971fb121cf7f9c0a9deb72e032c8867feb053519f75b51a2a2f6ac2b3cfb9f083b0af60ac36744412929ec81502c6955785d86a3c95b9d866d7ac7627
-  data.tar.gz: 0f9ce4932226b38aa32d3ae0419352d4839f939ff81ebf6de20757d32613f566b9e2e082e5e98ae67eac6738554cb3d9d32012722c14d01b3d537463cf420f32
+  metadata.gz: ecdae714f1c0b98f736c7dc90cfe7c10ea7b4540a8b6f45973dd227109a533f5d61863eff5e4f4ea172441429a779d0a21e5884475dd3e09584f6e39a7dfffed
+  data.tar.gz: 014c55ce1ce59493da340c7b880f019b990f11cbe883c21a8a519deb8a9366f9799663ca58841953a48d0a7c6a83c86d4bebc7e2ba5a22870e9e48bea216c498

data/.rubocop.yml

@@ -1,13 +1,40 @@
+require:
+  - rubocop-rspec
+  - rubocop-performance
+  - rubocop-rake
+
 AllCops:
-  TargetRubyVersion: 3.0
+  TargetRubyVersion: 3.3.0
+  NewCops: enable
+
+Metrics/MethodLength:
+  Max: 15
 
-Style/StringLiterals:
-  Enabled: true
-  EnforcedStyle: double_quotes
+Metrics/CyclomaticComplexity:
+  Max: 21
 
-Style/StringLiteralsInInterpolation:
-  Enabled: true
-  EnforcedStyle: double_quotes
+Metrics/PerceivedComplexity:
+  Max: 21
+
+Metrics/AbcSize:
+  Max: 100
 
 Layout/LineLength:
   Max: 120
+
+Style/Documentation:
+  Enabled: false
+
+RSpec/MultipleExpectations:
+  Enabled: false
+
+RSpec/ExampleLength:
+  Exclude:
+    - 'spec/requests/**/*'
+
+RSpec/MultipleMemoizedHelpers:
+  Exclude:
+    - 'spec/requests/**/*'
+
+Naming/VariableNumber:
+  EnforcedStyle: snake_case

data/.ruby-version

@@ -1 +1 @@
-3.2.2
+3.3.0

data/README.md

@@ -4,12 +4,12 @@ Aho — агент для сбора данных с ресурсов для б
 
 ## Установка
 
-Для установки агента Aho выполните команду:
+Установите агента командой:
 
     $ gem install aho
 
 ## Использование
 
-Для проверки установки выполните `aho`, в ответ должен быть ответ «Aho!»
+Для запуска выполните `aho`, в ответ запустится Puma сервер:
 
     $ aho

data/Rakefile

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 
-require "rubocop/rake_task"
+require 'rubocop/rake_task'
 
 RuboCop::RakeTask.new
 

data/aho.gemspec

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require_relative 'lib/aho/version'
+
+Gem::Specification.new do |spec|
+  spec.name = 'aho'
+  spec.version = Aho::VERSION
+  spec.authors = ['Panasenkov A.']
+  spec.email = ['aapanasenkov@gmail.com']
+
+  spec.summary = 'Aho agent for billing, monitoring and audit data from resources.'
+  spec.description = 'Aho agent is responsible for collecting resource usage data of VM instances on \
+                     individual Compute nodes within an Datamart Platform Studio deployment.'
+  spec.homepage = 'https://datamart.ru/aho'
+  spec.required_ruby_version = '>= 3.3.0'
+
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = 'https://gitlab.datamart.ru/datamart/aho'
+  spec.metadata['changelog_uri'] = 'https://gitlab.datamart.ru/datamart/aho/CHANGELOG.md'
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (File.expand_path(f) == __FILE__) ||
+        f.start_with?(*%w[bin/ test/ spec/ features/ .git appveyor Gemfile])
+    end
+  end
+  spec.bindir = 'exe'
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  # For more information and examples about making a new gem, check out our
+  # guide at: https://bundler.io/guides/creating_gem.html
+  spec.metadata['rubygems_mfa_required'] = 'true'
+
+  spec.add_dependency 'puma', '~> 6.4', '>= 6.4.3'
+  spec.add_dependency 'rack', '~> 3.1', '>= 3.1.8'
+  spec.add_dependency 'rackup', '~> 2.1'
+  spec.add_dependency 'rake', '~> 13.2', '>= 13.2.1'
+end

data/app/controllers/dsp/api/v1/agent_controller.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Dsp
+  module Api
+    module V1
+      class AgentController < BaseController
+        def restart
+          [200, {}, ['restart']]
+        end
+
+        def destroy
+          [200, {}, ['destroy']]
+        end
+      end
+    end
+  end
+end

data/app/controllers/dsp/api/v1/base_controller.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Dsp
+  module Api
+    module V1
+      class BaseController
+        attr_reader :env, :request, :path, :method
+
+        def initialize(env)
+          @env = env
+          @request = Rack::Request.new(env)
+          @path = request.path
+          @method = request.request_method
+        end
+
+        private
+
+        def respond(response, status: 200)
+          [status, { 'Content-Type' => 'application/json' }, [response.to_json]]
+        end
+
+        def schema
+          Aho.router.routes.dig(method.downcase.to_sym, path)[:schema]
+        end
+
+        def validate_schema!(...)
+          Aho::Schema::Validator.new(...).validate!
+        end
+      end
+    end
+  end
+end

data/app/controllers/dsp/api/v1/config_controller.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Dsp
+  module Api
+    module V1
+      class ConfigController < BaseController
+        def index
+          respond(Aho.config.settings)
+        end
+
+        def create
+          body = JSON.parse(request.body.read)
+          validate_schema!(body, schema)
+          Aho.config.update(body)
+          respond({ status: 'OK', message: 'Configuration applied successfully' })
+        end
+      end
+    end
+  end
+end

data/app/controllers/dsp/api/v1/healthcheck_controller.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Dsp
+  module Api
+    module V1
+      class HealthcheckController < BaseController
+        def index
+          render_up
+        rescue StandardError
+          render_down
+        end
+
+        private
+
+        def render_up
+          [200, { 'Content-Type' => 'text/html' }, [html_status(color: 'green')]]
+        end
+
+        def render_down
+          [500, { 'Content-Type' => 'text/html' }, [html_status(color: 'red')]]
+        end
+
+        def html_status(color:)
+          %(<!DOCTYPE html><html><body style="background-color: #{color}"></body></html>)
+        end
+      end
+    end
+  end
+end

data/app/controllers/dsp/api/v1/metrics_controller.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Dsp
+  module Api
+    module V1
+      class MetricsController < BaseController
+        def index
+          [200, {}, ['metrics']]
+        end
+      end
+    end
+  end
+end

data/config/default.aho.yml

@@ -0,0 +1,36 @@
+server_id: STUDIO_SERVER_ID
+ignore_metrics: -1
+life_jwt_time: 3600
+scrape_interval_time: 60
+enable_agent: -1
+
+upload:
+  limit_time: 3600
+  limit_lines: 100
+  limit_size: 1
+
+studio_settings:
+  url: http://localhost:8088/dsp/api/v1/vmagent
+
+agent_settings:
+  port: 16888
+
+storage:
+  limit_size: 0
+  limit_time: 86400
+  directory: /opt/dtms-billing-agent/
+
+file:
+  limit_size: 1
+  limit_time: 86400
+
+archive:
+  files_on: 0
+  size_on: 0
+  files_limit: 10
+  compression_level: -1
+
+aggregation:
+  files_on: 0
+  size_on: 0
+  level: 1

data/config/puma.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require_relative '../lib/aho'
+
+port Aho.config.settings.agent_settings.port

data/config/routes.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+Aho.router.instance_eval do
+  get '/dsp/api/v1/metrics', to: 'dsp/api/v1/metrics#index'
+  get '/dsp/api/v1/config', to: 'dsp/api/v1/config#index'
+  post '/dsp/api/v1/config', to: 'dsp/api/v1/config#create'
+  post '/dsp/api/v1/restart', to: 'dsp/api/v1/agent#restart'
+  delete '/dsp/api/v1/delete', to: 'dsp/api/v1/agent#destroy'
+  get '/dsp/api/v1/healthcheck', to: 'dsp/api/v1/healthcheck#index'
+end

data/config.ru

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require_relative 'config/routes'
+
+run Aho::WebServer::Application.new

data/exe/aho

@@ -1,7 +1,10 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-require "bundler/setup"
-require "aho"
+require 'bundler/setup'
+require 'aho'
+require 'rack/handler/puma'
+require File.join(Aho.root_path, 'config', 'routes')
 
-system "echo Aho!"
+app = Aho::WebServer::Application.new
+Rack::Handler::Puma.run(app)

data/lib/aho/configuration.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Aho
+  class Configuration
+    DEFAULT_CONFIG_FILE_PATH = File.expand_path('../../config/default.aho.yml', __dir__)
+    CONFIG_FILE_PATH = File.expand_path('../../config/aho.yml', __dir__)
+
+    attr_reader :settings
+
+    def initialize
+      reset unless File.exist?(CONFIG_FILE_PATH)
+      raw_settings = YAML.load_file(CONFIG_FILE_PATH)
+      rich_settings = process_env_variables(raw_settings)
+      @settings = DotHash.new(rich_settings)
+    end
+
+    def update(new_settings)
+      @settings.deep_merge!(new_settings.stringify_keys.slice(*default_settings.keys))
+      File.write(CONFIG_FILE_PATH, @settings.to_h.to_yaml)
+    end
+
+    def reset
+      File.write(CONFIG_FILE_PATH, File.read(DEFAULT_CONFIG_FILE_PATH))
+      @settings = DotHash.new(default_settings)
+    end
+
+    private
+
+    def process_env_variables(settings)
+      settings.deep_transform_values do |value|
+        if value.is_a?(String) && ENV.key?(value)
+          ENV.fetch(value, nil)
+        else
+          value
+        end
+      end
+    end
+
+    def default_settings
+      @default_settings ||= YAML.load_file(DEFAULT_CONFIG_FILE_PATH)
+    end
+  end
+end

data/lib/aho/errors.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Aho
+  class Error < StandardError; end
+
+  module Api
+    class MissingFieldError < Error
+      def initialize(field_name)
+        super("Missing required field '#{field_name}'")
+      end
+    end
+
+    class InvalidTypeError < Error
+      def initialize(field_name, expected_type, value)
+        super("Invalid type for '#{field_name}'. Expected #{expected_type}, got #{value.class}")
+      end
+    end
+  end
+end

data/lib/aho/metrics/active.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require_relative '../metrics_builder'
+
+module Aho
+  module Metrics
+    class Active < MetricsBuilder
+      def metrics
+        {
+          timestamp: run('date'),
+          server_id: 'TODO',
+          event_type: 'TODO',
+          description: 'TODO',
+          crashtime: 'TODO',
+          uptime: run('uptime'),
+          cpu_core: run('nproc'),
+          ram_total: run("awk '/MemTotal/ {print $2}' /proc/meminfo"),
+          disk_total: run("lsblk -b -o SIZE | awk 'NR>1 {total+=$1} END {print total}'"),
+          cpu_seconds_total:,
+          ram_used: run("free | awk '/^Mem/ {print $3}'"),
+          disk_used: run("df --block-size=1 | awk 'NR>1 {used+=$3} END {print used}'"),
+          network_load:
+        }
+      end
+
+      private
+
+      def cpu_seconds_total
+        File.readlines('/proc/stat').each_with_object([]) do |line, stats|
+          next unless line.start_with?('cpu') && !line.start_with?('cpu ')
+
+          parts = line.split
+          stats << {
+            cpu: parts[0],
+            user: parts[1].to_i,
+            system: parts[3].to_i,
+            idle: parts[4].to_i
+          }
+        end
+      rescue Errno::ENOENT
+        nil
+      end
+
+      def network_load
+        File.readlines('/proc/net/dev').drop(2).each_with_object([]) do |line, stats|
+          parts = line.split
+          stats << {
+            device: parts[0].delete_suffix(':'),
+            received: parts[1].to_i,
+            sent: parts[9].to_i
+          }
+        end
+      rescue Errno::ENOENT
+        nil
+      end
+    end
+  end
+end

data/lib/aho/metrics/ignore.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require_relative '../metrics_builder'
+
+module Aho
+  module Metrics
+    class Ignore < MetricsBuilder
+      def metrics
+        {
+          loadaverage_01: run("cat /proc/loadavg | awk '{print $1}'"),
+          loadaverage_05: run("cat /proc/loadavg | awk '{print $2}'"),
+          loadaverage_15: run("cat /proc/loadavg | awk '{print $3}'"),
+          tasks_total: run("top -bn1 | grep Tasks | awk '{print $2}'"),
+          tasks_running: run("top -bn1 | grep Tasks | awk '{print $4}'"),
+          disk_reads_completed_total: run("awk '{sum += $4} END {print sum}' /proc/diskstats"),
+          disk_writes_completed_total: run("awk '{sum += $8} END {print sum}' /proc/diskstats"),
+          swap_total_bytes: run("free --bytes | awk '/^Swap:/ {print $2}'"),
+          swap_used_bytes: run("free --bytes | awk '/^Swap:/ {print $3}'")
+        }
+      end
+    end
+  end
+end

data/lib/aho/metrics_aggregator.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require_relative 'metrics/active'
+require_relative 'metrics/ignore'
+
+module Aho
+  class MetricsAggregator
+    attr_reader :metrics
+
+    def initialize
+      @metrics = active_metrics
+      @metrics.merge!(ignore_metrics) if collect_ignore_metrics?
+    end
+
+    private
+
+    def active_metrics
+      Aho::Metrics::Active.new.metrics
+    end
+
+    def ignore_metrics
+      Aho::Metrics::Ignore.new.metrics
+    end
+
+    def collect_ignore_metrics?
+      Aho.config.settings.ignore_metrics == -1
+    end
+  end
+end

data/lib/aho/metrics_builder.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require 'open3'
+
+module Aho
+  class MetricsBuilder
+    def run(command)
+      stdout, _stderr, status = Open3.capture3(command)
+      stdout.strip if status.success?
+    rescue Errno::ENOENT => _e
+      nil
+    end
+  end
+end

data/lib/aho/metrics_formatter.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class MetricsFormatter
+  def call(_severity, _time, _progname, msg)
+    "#{msg.to_json}\n"
+  end
+end

data/lib/aho/metrics_log_device.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class MetricsLogDevice < Logger::LogDevice
+  private
+
+  # Override to prevent the default header from being written
+  def add_log_header(file)
+    # Leave this method empty to avoid adding any header
+  end
+end

data/lib/aho/metrics_logger/period.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class Logger
+  module Period
+    module_function
+
+    def next_rotate_time(now, shift_age)
+      now + shift_age.to_i
+    end
+
+    def previous_period_end(now, shift_age)
+      # case shift_age
+      # when 'daily', :daily
+      #   t = Time.mktime(now.year, now.month, now.mday) - SiD / 2
+      # when 'weekly', :weekly
+      #   t = Time.mktime(now.year, now.month, now.mday) - (SiD * now.wday + SiD / 2)
+      # when 'monthly', :monthly
+      #   t = Time.mktime(now.year, now.month, 1) - SiD / 2
+      # when 'now', 'everytime', :now, :everytime
+      #   return now
+      # else
+      #   raise ArgumentError, "invalid :shift_age #{shift_age.inspect}, should be daily, weekly, monthly, or everytime"
+      # end
+      # Time.mktime(t.year, t.month, t.mday, 23, 59, 59)
+    end
+  end
+end

data/lib/aho/metrics_logger.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require_relative 'metrics_log_device'
+
+module Aho
+  class MetricsLogger < Logger
+    TIMESTAMP_FORMAT = '%Y-%m-%d_%H-%M-%S'
+
+    def initialize(...)
+      super(
+        MetricsLogDevice.new(log_file_path, ...),
+        formatter: MetricsFormatter.new,
+        shift_period_suffix: TIMESTAMP_FORMAT
+      )
+    end
+
+    private
+
+    def log_file_path
+      timestamp = Time.now.strftime(TIMESTAMP_FORMAT)
+      "#{Aho.logs_path}/#{timestamp}_metrics.jsonl"
+    end
+  end
+end

data/lib/aho/schema/parser.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Aho
+  module Schema
+    class Parser
+      OPENAPI_FILE = 'openapi.yml'
+
+      attr_reader :method, :path, :destination, :openapi_paths
+
+      def initialize(method, path, destination)
+        @method = method
+        @path = path
+        @destination = destination
+        @openapi_paths = YAML.load_file(OPENAPI_FILE)['paths']
+      end
+
+      def controller
+        "#{destination.split('#').first.split('/').map(&:capitalize).join('::')}Controller"
+      end
+
+      def action
+        destination.split('#').last
+      end
+
+      def schema
+        remove_descriptions(openapi_paths.dig(path, method, 'requestBody', 'content', 'application/json', 'schema'))
+      end
+
+      private
+
+      def remove_descriptions(hash)
+        return unless hash.is_a?(Hash)
+
+        hash.each_value do |value|
+          if value.is_a?(Hash)
+            value.delete('description')
+            remove_descriptions(value)
+          elsif value.is_a?(Array)
+            value.each { |v| remove_descriptions(v) if v.is_a?(Hash) }
+          end
+        end
+      end
+    end
+  end
+end