fluent-plugin-http-cwm 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0fd32dfa032f161178f7cdbced98e810a58150161c8549505225951197c8b335
-  data.tar.gz: 18a3d8dcde2bd06e01bcefbc405ed7457f6dffae68c84d43e0e69b67b3e0708e
+  metadata.gz: ee799e3ac189ac3437e101b229bf0372b46b853312f45c36c89252f7928e3682
+  data.tar.gz: 2ec2f5a36742eb61f7365436d1e19d6ede930b20ad9c08f5cd2612a9af42ee1f
 SHA512:
-  metadata.gz: 69211562baa2f6e837823bd92bf58b6aa3be9d55657cc0b2a44aef1e718bbe248c44097df3cc91ef510fa851e9b109122e732e57cfa2743cc83f46a60613c5b0
-  data.tar.gz: bc0b0cee0e7b24b254f6d432804530399bdcfffb86e1b98bafc52d3f25f74a3ee8c0dcff004b322ea1ce791db356ebf8ae66d2ba43d65618127b37e05f870888
+  metadata.gz: f98f46b4cd8e5864bbcd0ea2c1801bda9a18de1116904feaba035a3ece3f2c710cdc525d9f202bd15599c39b978c02e137e9a9c9b42f5b07c5bce529b7f8d17e
+  data.tar.gz: 90b110133ed9377e746b81f784fad358830c15660961f2f7cc142a6d5817da2e8d2f8d70018a1614b10e373a49c96c32c23f6f37145c51e9e61a076eeab573ae

data/.github/workflows/publish.yml

@@ -11,7 +11,7 @@ on:
 
 jobs:
   publish:
-    if: github.event.workflow_run.conclusion == 'success' #&& startsWith(github.ref, 'refs/tags/v')
+    if: github.event.workflow_run.conclusion == 'success' && startsWith(github.ref, 'refs/tags/v')
 
     name: Publish Gem
     runs-on: ubuntu-18.04

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-http-cwm (0.2.0)
+    fluent-plugin-http-cwm (0.3.0)
       fluentd (>= 0.14.10, < 2)
       json (~> 2.3, >= 2.3.0)
       redis (~> 4.2, >= 4.2.5)
@@ -12,7 +12,7 @@ GEM
     concurrent-ruby (1.1.8)
     cool.io (1.7.1)
     docile (1.3.5)
-    fluentd (1.12.2)
+    fluentd (1.12.3)
       bundler
       cool.io (>= 1.4.5, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)

data/README.md

@@ -10,6 +10,17 @@
 ![GitHub code size in bytes](https://img.shields.io/github/languages/code-size/iamAzeem/fluent-plugin-http-cwm?style=flat-square)
 ![GitHub repo size](https://img.shields.io/github/repo-size/iamAzeem/fluent-plugin-http-cwm?style=flat-square)
 
+- [Overview](#overview)
+- [Installation](#installation)
+  - [RubyGems](#rubygems)
+  - [Bundler](#bundler)
+- [Configuration](#configuration)
+  - [`<redis>` section (optional) (single)](#redis-section-optional-single)
+  - [Sample Configuration](#sample-configuration)
+- [Contribute](#contribute)
+- [Publish the gem](#publish-the-gem)
+- [License](#license)
+
 ## Overview
 
 [Fluentd](https://fluentd.org/) HTTP input plugin for
@@ -100,7 +111,7 @@ bundle
   - Default value: `6379`.
 - `db` (integer) (optional): The db to use.
   - Default value: `0`.
-- `grace_period` (time) (optional): The grace period for last update.
+- `grace_period` (time) (optional): The grace period for last action update.
   - Default value: `300s`.
 - `flush_interval` (time) (optional): The flush interval to send metrics.
   - Default value: `300s`.
@@ -164,10 +175,21 @@ Example:
 - Check out the latest `main` branch.
 - Create a feature or bugfix branch from `main`.
 - Commit and push your changes.
-- Make sure to add tests.
+- Make sure to add and run tests locally: `bundle exec rake test`.
 - Run Rubocop locally and fix all the lint warnings.
+- Make sure to update [Gemfile.lock](Gemfile.lock): `sudo bundle update`.
 - Submit the PR.
 
+## Publish the gem
+
+The gem is published via the [publish.yml](.github/workflows/publish.yml)
+Workflow on tagging. The tag must be of the format `v0.3.0`. This workflow
+depends on the successful completion of the [ci.yml](.github/workflows/ci.yml)
+workflow and then it looks for the tag. So, make sure that all the CI issues are
+resolved before creating a new tag. If there are issues while publishing the gem
+i.e. publish workflow doesn't work properly, you can delete and then recreate
+the tag to retrigger this workflow.
+
 ## License
 
 [Apache 2.0](./LICENSE)

data/fluent-plugin-http-cwm.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name           = 'fluent-plugin-http-cwm'
-  spec.version        = '0.2.0'
+  spec.version        = '0.3.0'
   spec.authors        = ['Azeem Sajid']
   spec.email          = ['azeem.sajid@gmail.com']
 

data/lib/fluent/plugin/in_http_cwm.rb

@@ -53,7 +53,7 @@ module Fluent
         desc 'The db to use.'
         config_param :db, :integer, default: 0
 
-        desc 'The grace period for last update.'
+        desc 'The grace period for last action update.'
         config_param :grace_period, :time, default: '300s'
 
         desc 'The flush interval to send metrics.'
@@ -71,6 +71,9 @@ module Fluent
 
         @redis = nil
         @deployment_api_metrics = default_api_metrics_hash
+
+        @last_action_queue = Queue.new
+        @last_action_entry = []
       end
 
       def configure(conf)
@@ -82,18 +85,23 @@ module Fluent
       def start
         super
 
-        # set up timer for flush interval
-        timer_execute(:metrics_flush_timer, @redis_config.flush_interval) do
+        # start interval timer to flush api metrics
+        timer_execute(:api_metrics_flush_timer, @redis_config.flush_interval) do
           flush_api_metrics
         end
 
+        # start interval timer to flush last action entry
+        timer_execute(:last_action_flush_timer, '1s') do
+          flush_last_action
+        end
+
         log.info("Starting HTTP server [#{@host}:#{@port}]...")
         http_server_create_http_server(:http_server, addr: @host, port: @port, logger: log) do |server|
           server.post("/#{tag}") do |req|
             data = parse_data(req.body)
             route(data) if update_deployment_metrics(data)
 
-            # return HTTP 200 OK response to MinIO
+            # return HTTP 200 OK response with emtpy body
             [200, { 'Content-Type' => 'text/plain' }, nil]
           end
         end
@@ -141,40 +149,62 @@ module Fluent
         router.emit(@tag, time, record)
       end
 
-      def days_to_seconds(days)
-        days * 24 * 60 * 60
+      def flush_last_action
+        if @last_action_entry.empty?
+          @last_action_entry = @last_action_queue.deq.split('|')
+          log.debug("Dequed last action entry. #{@last_action_entry}")
+        else
+          deploymentid, last_action = @last_action_entry
+          @last_action_entry = [] if update_deployment_last_action(deploymentid, last_action)
+        end
       end
 
-      def update_deployment_last_action(deploymentid)
-        log.debug('Updating deployment last action')
+      def datetime_diff_in_secs(dt_begin, dt_end)
+        seconds = ((dt_end - dt_begin) * 24 * 60 * 60)
+        seconds.to_i
+      end
 
+      def update_deployment_last_action(deploymentid, last_action)
         key = "#{@redis_config.last_update_prefix}:#{deploymentid}"
-        curdt = DateTime.now
-
-        begin
-          log.debug("checking existing last action entry [key: #{key}]")
-          lastval = @redis.get(key)
-
-          is_grace_period_expired = false
-          if lastval
-            lastdt = DateTime.parse(lastval, FMT_DATETIME)
-            dt_diff_secs = days_to_seconds((curdt - lastdt).to_f)
-            if dt_diff_secs > @redis_config.grace_period
-              is_grace_period_expired = true
-              log.debug("grace period [#{@redis_config.grace_period}] expired [#{lastdt} => #{curdt}]")
-            end
-          else
-            log.debug('last action entry not found. going to be set for the first time.')
+        log.debug("Checking existing last action entry [key: #{key}]")
+        lastval = @redis.get(key)
+
+        is_grace_period_expired = false
+        if lastval
+          curdt = DateTime.now
+          lastdt = DateTime.parse(lastval, FMT_DATETIME)
+          dt_diff_secs = datetime_diff_in_secs(lastdt, curdt)
+          log.debug("Current  Data/Time: #{curdt}")
+          log.debug("Previous Date/Time: #{lastdt}")
+          log.debug("Date/Time diff (s): #{dt_diff_secs}")
+
+          if dt_diff_secs >= @redis_config.grace_period
+            is_grace_period_expired = true
+            log.debug("Grace period expired for last action update. [#{@redis_config.grace_period}]")
           end
+        else
+          log.debug('Last action entry does not exist. It will be set for the first time.')
+        end
 
-          if lastdt.nil? || is_grace_period_expired
-            last_action = curdt.strftime(FMT_DATETIME)
-            @redis.set(key, last_action)
-            log.debug("Updated last action entry [#{key} => #{last_action}]")
-          end
-        rescue StandardError => e
-          log.error("Unable to update last action! ERROR: '#{e}'.")
+        if lastdt.nil? || is_grace_period_expired
+          log.debug('Updating deployment last action')
+          last_action = DateTime.parse(last_action, FMT_DATETIME)
+          @redis.set(key, last_action)
+          log.debug("Updated last action entry [#{key} => #{last_action}]")
+          true
+        else
+          false
         end
+      rescue StandardError => e
+        log.error("Unable to update last action! ERROR: '#{e}'.")
+        false
+      end
+
+      def enque_last_action_entry(deploymentid)
+        last_action = DateTime.now
+        entry = "#{deploymentid}|#{last_action}"
+        @last_action_queue.enq(entry)
+        log.debug("Enqued last action entry. [#{entry}]")
       end
 
       def validate_and_get_value(data_hash, key)
@@ -191,7 +221,7 @@ module Fluent
         deploymentid = validate_and_get_value(data, 'deploymentid')
         return false unless deploymentid
 
-        update_deployment_last_action(deploymentid)
+        enque_last_action_entry(deploymentid)
 
         api_data = validate_and_get_value(data, 'api')
         return false unless api_data

data/test/plugin/test_in_http_cwm.rb

@@ -8,9 +8,26 @@ class CwmHttpInputTest < Test::Unit::TestCase
   setup do
     Fluent::Test.setup
 
-    @default_conf = %(
-      tag       test
-    )
+    @default_conf = config_element('ROOT', '', { 'tag' => 'test' })
+    @custom_conf = config_element('ROOT', '', { 'tag' => 'test' }, [
+                                    config_element('redis', '', {
+                                                     'grace_period' => '1s',
+                                                     'flush_interval' => '1s'
+                                                   })
+                                  ])
+
+    @expected_redis_output = {
+      'deploymentid:last_action:docker-compose-http' => '',
+      'deploymentid:last_action:docker-compose-https' => '',
+      'deploymentid:minio-metrics:docker-compose-http:bytes_in' => '61852',
+      'deploymentid:minio-metrics:docker-compose-https:bytes_in' => '14875',
+      'deploymentid:minio-metrics:docker-compose-http:bytes_out' => '112012',
+      'deploymentid:minio-metrics:docker-compose-https:bytes_out' => '2755',
+      'deploymentid:minio-metrics:docker-compose-http:num_requests_out' => '7',
+      'deploymentid:minio-metrics:docker-compose-http:num_requests_in' => '10',
+      'deploymentid:minio-metrics:docker-compose-https:num_requests_in' => '5',
+      'deploymentid:minio-metrics:docker-compose-http:num_requests_misc' => '5'
+    }.freeze
   end
 
   def create_driver(conf)
@@ -27,13 +44,7 @@ class CwmHttpInputTest < Test::Unit::TestCase
     end
 
     test 'redis default configuration test' do
-      conf = %(
-        tag     test
-        <redis>
-        </redis>
-      )
-
-      driver = create_driver(conf)
+      driver = create_driver(@default_conf)
       plugin = driver.instance
       redis = plugin.redis_config
       assert_equal 'localhost', redis.host
@@ -47,7 +58,9 @@ class CwmHttpInputTest < Test::Unit::TestCase
 
   sub_test_case 'route#emit' do
     test 'emit test' do
-      driver = create_driver(@default_conf)
+      driver = create_driver(@custom_conf)
+      plugin = driver.instance
+      redis = plugin.redis_config
 
       res_codes = []
       lines = 0
@@ -63,6 +76,25 @@ class CwmHttpInputTest < Test::Unit::TestCase
       assert_equal lines, res_codes.size
       assert_equal '200', res_codes[0]
       assert_equal 1, res_codes.uniq.size
+
+      # run and test private flushing methods
+      `redis-cli FLUSHALL`
+      sleep(redis.grace_period)
+      driver.events.each do
+        plugin.send(:flush_api_metrics)
+        plugin.send(:flush_last_action)
+      end
+
+      # verify from Redis server
+      @expected_redis_output.each do |key, expected_value|
+        if key.include? 'last_action'
+          exists = `redis-cli EXISTS #{key}`.chomp
+          assert_equal '1', exists
+        else
+          actual_value = `redis-cli GET #{key}`.chomp
+          assert_equal expected_value, actual_value
+        end
+      end
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-http-cwm
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Azeem Sajid
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-04-07 00:00:00.000000000 Z
+date: 2021-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -108,22 +108,22 @@ dependencies:
   name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.3.0
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.3.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.3.0
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.3.0
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
@@ -185,7 +185,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3.1
+rubyforge_project: 
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: fluentd HTTP Input Plugin for CloudWebManage Logging Component