fluent-plugin-coralogix 1.0.3 → 1.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0464583b2b032b742f4f575419787d97d9517cad54969ba205445c587fcb9f84'
-  data.tar.gz: 1a3ecf5ce8ce9da3d4198093c9799c9fd42c824d2c54dade646479229bcdee48
+  metadata.gz: 3dae7df93ce63ae60ba2beffe047731d826b2e016982a32c409bad5fdd50d57e
+  data.tar.gz: 3044ea4a5c4bd4062d9fa8bb5ba50a2cf41defbefab2f2ddfdd65b124fc33b15
 SHA512:
-  metadata.gz: abfd5e328bc35e658b351ac878faed181b4a3e8c1de71d657b8c3d131a00569db9b2a43eb77201875d7d0c238b1f92af7ad997c0da21e068fbc8cdd60851f2b5
-  data.tar.gz: d8bed1800f225ae9cb04a34ada5321f25baf37d8c397216bcab10fad5bd79aa7d5427b91ce7308eafad484e235f780995f315b66ccc963a6afc6b993e9e49c0a
+  metadata.gz: 1d1c799e6636b76cf379f6cccb6eca910de452ed62c0853540cc270112536f1d74080601d963f6f6e2bc43aad22b70a2d45ac39c5c74e3666ad221dff58faf71
+  data.tar.gz: f04beb89599c9de420bff3d0d025c8667b57cc6dca65be6821c7d864c0ecc8561d62b7292f0457d0de3dcdc2af4d305dc5f61583690fa96b4b45860e7c8e8006

data/lib/fluent/plugin/out_coralogix.rb

@@ -13,35 +13,53 @@ module Fluent
     # First, register the plugin. NAME is the name of this plugin
     # and identifies the plugin in the configuration file.
     Fluent::Plugin.register_output('coralogix', self)
-    config_param :config, :hash, :default => {}
-    config_param :log_key_name, :string, :default => nil
+    config_param :endpoint, :string, :default => nil
     config_param :privatekey, :string, :default => nil
     config_param :appname, :string, :default => ""
     config_param :subsystemname, :string, :default => ""
-    config_param :is_json, :bool, :default => false
+    config_param :log_key_name, :string, :default => nil
     config_param :timestamp_key_name, :default => nil
+    config_param :is_json, :bool, :default => false
     config_param :force_compression, :bool, :default => false
-    config_param :debug, :bool, :default => false
-    @configured = false
-
+    config_param :number_of_workers, :integer, :default => 1
+    config_section :proxy, :param_name => "proxy", required: false, multi: false do
+      config_param :host, :string
+      config_param :port, :integer
+      config_param :user, :string, :default =>  nil
+      config_param :password, :string, :default =>  nil, secret: true
+    end
 
     # This method is called before starting.
     def configure(conf)
       super
       begin
-        @loggers = {}
-       
-        #If config parameters doesn't start with $ then we can configure Coralogix logger now.
-        if !appname.start_with?("$") && !subsystemname.start_with?("$")
-          app_name = config.fetch("APP_NAME", appname)
-          sub_name = config.fetch("SUB_SYSTEM", subsystemname)
+        @currentWorker = 0;
+        @workers = {}
+        @app_name_from_config = DEFAULT_appname;
+        @sub_name_from_config = DEFAULT_subsystemname;
+        i = 0
+        log.info "configure #{number_of_workers} workers"
+        until i == number_of_workers do
+          @workers[i.to_s] = {}
+          log.info "init worker ##{i}"
           private_key = get_private_key
-          
-          @logger = CoralogixLogger.new private_key, app_name, sub_name, debug, "FluentD (#{version?})", force_compression
-          @configured = true
+          if !appname.start_with?("$") && !subsystemname.start_with?("$")
+            @app_name_from_config = config.fetch("APP_NAME", appname)
+            @sub_name_from_config = config.fetch("SUB_SYSTEM", subsystemname)
+            @workers[i.to_s]["#{@app_name_from_config}.#{@sub_name_from_config}"] = CoralogixLogger.new private_key, @app_name_from_config, @sub_name_from_config, false, "FluentD (#{version?})", force_compression, proxy ? proxy.to_h.map { |k,v| [k.to_s,v] }.to_h : Hash.new, log
+          else
+            @workers[i.to_s] = {}
+          end
+          i+=1
+        end
+
+        unless endpoint.nil?
+          ENV["CORALOGIX_LOG_URL"] = "https://#{endpoint}/api/v1/logs"
+          ENV["CORALOGIX_TIME_DELTA_URL"] = "https://#{endpoint}/sdk/v1/time"
         end
+
       rescue Exception => e
-        $log.error "Failed to configure: #{e}"
+        log.error "Failed to configure: #{e}"
       end
     end
 
@@ -51,9 +69,9 @@ module Fluent
 
     def version?
       begin
-          Gem.loaded_specs['fluent-plugin-coralogix'].version.to_s
-      rescue Exception => e  
-          return '0.0.0'
+        Gem.loaded_specs['fluent-plugin-coralogix'].version.to_s
+      rescue Exception => e
+        return '0.0.0'
       end
     end
 
@@ -62,12 +80,12 @@ module Fluent
         res = record
         return key unless key.start_with?("$")
         key[1..-1].split(".").each do |k|
-          res = res.fetch(k,nil)
+          res = res.fetch(k, nil)
           return default if res == nil
         end
         return res
       rescue Exception => e
-        $log.error "Failed to extract #{key}: #{e}"
+        log.error "Failed to extract #{key}: #{e}"
         return default
       end
     end
@@ -80,17 +98,26 @@ module Fluent
     end
 
     def get_logger(record)
-
-      return @logger if @configured
-      
       private_key = get_private_key
-      app_name, sub_name = get_app_sub_name(record)
+      if !appname.start_with?("$") && !subsystemname.start_with?("$")
+        app_name = @app_name_from_config
+        sub_name = @sub_name_from_config
+      else
+        app_name, sub_name = get_app_sub_name(record)
+      end
       
-      if !@loggers.key?("#{app_name}.#{sub_name}")
-        @loggers["#{app_name}.#{sub_name}"] = CoralogixLogger.new private_key, app_name, sub_name
+      # YK@2020-11-26T10:56 - We had encountered a case in which this value reached above 7K and the value of worker became null
+      if @currentWorker >= number_of_workers
+        @currentWorker = 0
+      end
+      worker = @workers[@currentWorker.to_s]
+      @currentWorker+=1;
+      if !worker.key?("#{app_name}.#{sub_name}")
+        worker["#{app_name}.#{sub_name}"] = CoralogixLogger.new private_key, app_name, sub_name, false, "FluentD (#{version?})", force_compression, proxy ? proxy.to_h.map { |k, v| [k.to_s, v] }.to_h : Hash.new, log
       end
 
-      return @loggers["#{app_name}.#{sub_name}"]
+      return worker["#{app_name}.#{sub_name}"]
+      
     end
 
 
@@ -113,7 +140,7 @@ module Fluent
     # NOTE! This method is called by Fluentd's main thread so you should not write slow routine here. It causes Fluentd's performance degression.
     def emit(tag, es, chain)
       chain.next
-      es.each {|time,record|
+      es.each { |time, record|
         logger = get_logger(record)
 
         log_record = log_key_name != nil ? record.fetch(log_key_name, record) : record
@@ -121,17 +148,17 @@ module Fluent
         log_record = log_record.to_s.empty? ? record : log_record
 
         timestamp = record.fetch(timestamp_key_name, nil)
-        if(timestamp.nil?)
+        if (timestamp.nil?)
           logger.debug log_record
         else
           begin
             float_timestamp = DateTime.parse(timestamp.to_s).to_time.to_f * 1000
-            logger.debug log_record, nil, timestamp:float_timestamp
-          rescue Exception => e  
+            logger.debug log_record, nil, timestamp: float_timestamp
+          rescue Exception => e
             logger.debug log_record
-          end          
+          end
         end
       }
     end
   end
-end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-coralogix
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.0.8
 platform: ruby
 authors:
 - Royee Goldberg
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-05-22 00:00:00.000000000 Z
+date: 2021-03-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: centralized_ruby_logger
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.0.15
+        version: 0.0.16
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.0.15
+        version: 0.0.16
 description: Coralogix Fluentd plugin to send logs to Coralogix server.
 email: royee@coralogix.com
 executables: []
@@ -35,7 +35,7 @@ homepage: http://www.coralogix.com
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -50,9 +50,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
+rubyforge_project:
 rubygems_version: 2.7.3
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Coralogix Fluentd out plugin
 test_files: []