fluent-plugin-site24x7 0.1.4 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eaa27dafc08120c093de5e57ba15b129209637db8759e68a20c2053513cb55e9
-  data.tar.gz: 6fc10de6e63e332c816935a656cfb234e733710c7849f7578ebaa035eec7a8d4
+  metadata.gz: fc1338accf3055ea6b93742a4280cdbdb8ec4b5e0ab97ea4ffc79e77855f4f28
+  data.tar.gz: ce48c6512000a972e2f3209b4b142d6ef4cffcec7268720f0a56fed15e75553c
 SHA512:
-  metadata.gz: 58d44f268199174dabf93bc3b0fc3ceb0add74b2718001aaab95c3c380bf4f2b18bec949162a47a778a7d78d80a067302cc509506ceac6f8a2f21d5dedcc60ba
-  data.tar.gz: bd0dc479e64382935a9154006b281b46cad6d93c883b18d5ddb1cbd026ae49d427a35c0e4836ee6cf7eb1996d6d3557cd5d3bb4f862d9149af5655be862ad1d5
+  metadata.gz: a48c2d71823a1756d55ae9343017331f1a6b0579c42f9fcec1c61070062bb51071adfb614b71cd162015943fd368fa179d4f932662d2b8a6e54678ce7e167539
+  data.tar.gz: d95117ea90350c380ee4dcacc1dd2238bac3d67e8338fa973c8231a81f2f701ead6d8f02d306b82789afb396094e250507a915ba974f64509ef480339597c881

data/fluent-plugin-site24x7.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-site24x7"
-  spec.version = "0.1.4"
+  spec.version = "0.1.6"
   spec.authors = ["Magesh Rajan"]
   spec.email   = ["magesh.rajan@zohocorp.com"]
 

data/lib/fluent/plugin/out_site24x7.rb

@@ -21,8 +21,10 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
   helpers :compat_parameters
 
   config_param :log_type_config, :string
-  config_param :max_retry, :integer, :default => 3
+  config_param :max_retry, :integer, :default => -1
   config_param :retry_interval, :integer, :default => 2
+  config_param :maxretry_interval, :integer, :default => 60 #1 minutes 
+  config_param :retry_timeout, :integer, :default => 24 * 3600  # 24 hours
   config_param :http_idle_timeout, :integer, default: 5
   config_param :http_read_timeout, :integer, default: 30
   config_param :http_proxy, :string, :default => nil
@@ -143,9 +145,13 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
       for field,rules in @logtype_config['filterConfig'] do
         temp = []
         for value in @logtype_config['filterConfig'][field]['values'] do
-          temp.push(Regexp.compile(value))
+          if @logtype_config['filterConfig'][field]['exact']
+            temp.push("\\A"+value+"\\Z")
+          else
+            temp.push(value)
+          end
         end
-        @logtype_config['filterConfig'][field]['values'] = temp.join('|') 
+        @logtype_config['filterConfig'][field]['values'] = Regexp.compile(temp.join('|')) 
       end
     end  
     
@@ -165,13 +171,13 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
       if !@is_timezone_present && @logtype_config.has_key?('timezone')
   @s247_datetime_format_string += '%z'
 	tz_value = @logtype_config['timezone']
-	if tz_value.start_with?('+')
-	    @s247_tz['hrs'] = Integer('-' + tz_value[1..4])
-	    @s247_tz['mins'] = Integer('-' + tz_value[3..6])
-	elsif tz_value.start_with?('-')
-	    @s247_tz['hrs'] = Integer('+' + tz_value[1..4])
-	    @s247_tz['mins'] = Integer('+' + tz_value[3..6])
-	end
+  if tz_value.start_with?('+')
+    @s247_tz['hrs'] = Integer('-' + tz_value[1..2])
+    @s247_tz['mins'] = Integer('-' + tz_value[3..4])
+  elsif tz_value.start_with?('-')
+    @s247_tz['hrs'] = Integer('+' + tz_value[1..2])
+    @s247_tz['mins'] = Integer('+' + tz_value[3..4])
+  end
       end
     end
     Thread.new { timer_task() }
@@ -208,18 +214,30 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
             time_zone = String(@s247_tz['hrs'])+':'+String(@s247_tz['mins'])
             datetime_string += if time_zone.start_with?('-') then time_zone else '+'+time_zone end
         end
+        #log.info "datetime_string : (#{datetime_string}), s247_datetime_format_string: (#{@s247_datetime_format_string})"
         datetime_data = DateTime.strptime(datetime_string, @s247_datetime_format_string)
         return Integer(datetime_data.strftime('%Q'))
     rescue Exception => e
-      @logger.error "Exception in parsing date: #{e.backtrace}"
-        return 0
+      log.error "Exception in parsing date: #{e.message}"
+      return 0
     end
   end
 
-  def log_line_filter()
-    applyMasking()
-    applyHashing()
-    getDerivedFields()
+  def data_collector()
+    if @formatted_line.has_key?('_zl_timestamp')
+      applyMasking()
+      applyHashing()
+      getDerivedFields()
+      if !is_filters_matched()
+        @formatted_line = {}
+        return
+      end
+      remove_ignored_fields()
+      log_size_calculation()
+    else
+      @formatted_line = {}
+      return
+    end
   end 
 
   def get_last_group_inregex(s247_custom_regex)
@@ -228,7 +246,9 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
 
   def remove_ignored_fields()
     @s247_ignored_fields.each do |field_name|
-      @log_size -= if @log_fields.has_key?field_name then @log_fields.delete(field_name).bytesize else 0 end
+      if @formatted_line.has_key?field_name
+        @formatted_line.delete(field_name)
+      end
     end
   end
 
@@ -245,10 +265,7 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
           match = line.match(@s247_custom_regex)
           if match
             @formatted_line.update(@old_formatted_line)
-            @log_size += @old_log_size
-            @old_log_size = line.bytesize
             @log_fields = match&.named_captures
-            remove_ignored_fields()
             add_message_metadata()
             @old_formatted_line = @log_fields
             @last_line_matched = true
@@ -262,14 +279,13 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
                 @old_log_size += line.bytesize
                 @trace_started = true
                 @last_line_matched = false
-              end 
+              end
             end
-          end   
-          if @formatted_line.has_key?('_zl_timestamp')
-            log_line_filter()
+          end
+          data_collector()
+          if @formatted_line.length >0
             parsed_lines.push(@formatted_line)
-            @formatted_line = {}
-          end    
+          end     
         rescue Exception => e
           log.error "Exception in parse_line #{e.backtrace}"
           @formatted_line = {}
@@ -283,12 +299,14 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
     begin
       if @logtype_config.has_key?'filterConfig'
         @logtype_config['filterConfig'].each do |config,value|
-          if @formatted_line[config].scan(Regexp.new(@logtype_config['filterConfig'][config]['values'])).length > 0
-            val = true 
-          else
-            val = false
+          if (@formatted_line.has_key?config)
+            if @logtype_config['filterConfig'][config]['values'].match(@formatted_line[config])
+              val = true 
+            else
+              val = false
+            end
           end
-          if (@formatted_line.has_key?config) && (@logtype_config['filterConfig'][config]['match'] ^ (val))
+          if (@logtype_config['filterConfig'][config]['match'] ^ (val))
             return false
           end
         end
@@ -342,20 +360,18 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
         matcher = regex.match(@log_fields.delete(key))
         if matcher
           @log_fields.update(matcher.named_captures)
-          remove_ignored_fields()
           @formatted_line.update(@log_fields)
         end 
       end
     end
-    if !(is_filters_matched())
-      return false
-    else
-      add_message_metadata()
-      @formatted_line.update(@log_fields)
-      log_line_filter()
-      @log_size += json_log_size
+    add_message_metadata()
+    @formatted_line.update(@log_fields)
+    data_collector()
+    if @formatted_line.length >0
       return true
-     end
+    else
+      return false
+    end 
   end
 
   def json_log_parser(lines)
@@ -367,7 +383,7 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
           if line[0] == '{' && @json_data[-1] == '}'
             if json_log_applier(@json_data)
               parsed_lines.push(@formatted_line)
-              end
+            end
             @json_data=''
           end
           @json_data += line
@@ -380,20 +396,14 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
   end
 
   def ml_regex_applier(ml_trace, ml_data)
-    begin    
-      @log_size += @ml_trace.bytesize
+    begin
       matcher = @s247_custom_regex.match(@ml_trace)  
       @log_fields = matcher.named_captures
       @log_fields.update(@ml_data)
-      if @s247_ignored_fields
-        remove_ignored_fields()
-      end
       add_message_metadata()
       @formatted_line.update(@log_fields)
-      log_line_filter()
     rescue Exception => e
       log.error "Exception occurred in ml_parser : #{e.backtrace}"
-      @formatted_line = {}
     end
   end
   
@@ -413,14 +423,15 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
                   ml_regex_applier(@ml_trace, @ml_data)
                   if @ml_trace_buffer && @formatted_line
                       @formatted_line[@message_key] = @formatted_line[@message_key] + @ml_trace_buffer
-                      @log_size += @ml_trace_buffer.bytesize
                   end
-                  parsed_lines.push(@formatted_line)
+                  data_collector()
+                  if @formatted_line.length >0
+                    parsed_lines.push(@formatted_line)
+                  end    
                   @ml_trace = ''
                   @ml_trace_buffer = ''
                   if @ml_found
                     @ml_data = ml_start_matcher.named_captures
-                    @log_size += line.bytesize
                   else
                       @ml_data = {}
                   end
@@ -429,7 +440,6 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
                 log.error "Exception occurred in ml_parser : #{e.backtrace}"
                 end
             elsif @ml_found
-              @log_size += line.bytesize
               @ml_data = ml_start_matcher.named_captures
             end
           elsif @ml_found
@@ -463,26 +473,27 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
   end     
 
   def write(chunk)
+    current_chunk_id = "#{chunk.dump_unique_id_hex(chunk.unique_id)}"
     begin
-	events = Array.new
-	chunk.msgpack_each do |record|
-	  next if record.empty?
-	  events.push record[0]
-	end
-	process_http_events(events)
+      events = Array.new
+      chunk.msgpack_each do |record|
+        next if record.empty?
+        events.push record[0]
+      end
+      process_http_events(events, current_chunk_id)
     rescue Exception => e
       log.error "Exception #{e.backtrace}"
     end
   end
 
-  def process_http_events(events)
+  def process_http_events(events, current_chunk_id)
     @before_time = Time.now
     batches = batch_http_events(events)
     batches.each do |batched_event|
       formatted_events, @log_size = format_http_event_batch(batched_event)
       if (formatted_events.length>0)
         formatted_events = gzip_compress(formatted_events)
-        send_logs_to_s247(formatted_events, @log_size)
+        send_logs_to_s247(formatted_events, @log_size, current_chunk_id)
       end
     end 
   end
@@ -540,49 +551,73 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
     gz.string
   end
 
-  def send_logs_to_s247(gzipped_parsed_lines, log_size)
+  def send_logs_to_s247(gzipped_parsed_lines, log_size, current_chunk_id)
       request = Net::HTTP::Post.new @uri.request_uri
       request.body = gzipped_parsed_lines
-      @s247_http_client.override_headers["Log-Size"] = @log_size
+
       sleep_interval = @retry_interval
      begin
-        @max_retry.times do |counter|
+        retries = 0
+        first_upload_time =  Time.now
+        while true
           need_retry = false
           begin
+            @s247_http_client.override_headers["Log-Size"] = @log_size
+            @s247_http_client.override_headers["upload-id"] = current_chunk_id
             response = @s247_http_client.request @uri, request
             resp_headers = response.each_header.to_h
-
+            
             if response.code == '200'
-              if resp_headers.has_key?'LOG_LICENSE_EXCEEDS' && resp_headers['LOG_LICENSE_EXCEEDS'] == 'True'
-                log.error "Log license limit exceeds so not able to send logs"
+              if resp_headers['x-uploadid'] == nil
+                log.error "[#{current_chunk_id}]:upload id is empty hence retry flag enabled #{resp_headers}"
+                need_retry = true
+              else
+		            @log_upload_allowed = true
+                log.debug "[#{current_chunk_id}]:Successfully sent logs with size #{gzipped_parsed_lines.size} / #{@log_size} to site24x7. Upload Id : #{resp_headers['x-uploadid']}"
+              end
+            elsif response.code == '400'
+              if resp_headers.has_key?'log_license_exceeds' and resp_headers['log_license_exceeds'] == 'True'
+                log.error "[#{current_chunk_id}]:Log license limit exceeds so not able to send logs"
                 @log_upload_allowed = false
 		            @log_upload_stopped_time =Time.now.to_i
-              elsif resp_headers.has_key?'BLOCKED_LOGTYPE' && resp_headers['BLOCKED_LOGTYPE'] == 'True'
-                log.error "Max upload limit reached for log type"
+              elsif resp_headers.has_key?'blocked_logtype' and resp_headers['blocked_logtype'] == 'True'
+                log.error "[#{current_chunk_id}]:Max upload limit reached for log type"
                 @log_upload_allowed = false
 		            @log_upload_stopped_time =Time.now.to_i
-              elsif resp_headers.has_key?'INVALID_LOGTYPE' && resp_headers['INVALID_LOGTYPE'] == 'True'
-                log.error "Log type not present in this account so stopping log collection"
+              elsif resp_headers.has_key?'api_upload_enabled' and resp_headers['api_upload_enabled'] == 'False'
+                log.error "[#{current_chunk_id}]:API upload not enabled for log type : "
+                @log_upload_allowed = false
+		            @log_upload_stopped_time =Time.now.to_i
+              elsif resp_headers.has_key?'invalid_logtype' and resp_headers['invalid_logtype'] == 'True'
+                log.error "[#{current_chunk_id}]:Log type not present in this account so stopping log collection"
+		            @valid_logtype = false
+              elsif resp_headers.has_key?'invalid_account' and resp_headers['invalid_account'] == 'True'
+                log.error "[#{current_chunk_id}]: Invalid account so stopping log collection"
 		            @valid_logtype = false
               else
-		            @log_upload_allowed = true
-                log.debug "Successfully sent logs with size #{gzipped_parsed_lines.size} / #{@log_size} to site24x7. Upload Id : #{resp_headers['x-uploadid']}"
+                log.error "[#{current_chunk_id}]: Upload failed for reason : #{response.message}"
               end
             else
-              log.error "Response Code #{resp_headers} from Site24x7, so retrying (#{counter + 1}/#{@max_retry})"
+              log.error "[#{current_chunk_id}]:Response Code #{response.code} from Site24x7, so retrying (#{retries + 1}/#{@max_retry})"
               need_retry = true
             end
           rescue StandardError => e
-            log.error "Error connecting to Site24x7. exception: #{e} (#{counter + 1}/#{@max_retry})"
+            log.error "[#{current_chunk_id}]:Error connecting to Site24x7. exception: #{e} (#{retries + 1}/#{@max_retry})"
+            need_retry = true
           end
 
           if need_retry
-            if counter == @max_retry - 1
-              log.error "Could not send your logs after #{max_retry} tries"
+            retries += 1 
+            if (retries >= @max_retry && @max_retry > 0) || (Time.now > first_upload_time + @retry_timeout && @retry_timeout > 0)
+              log.error "[#{current_chunk_id}]: Internal max retries(#{max_retry}) or retry_timeout : #{first_upload_time + @retry_timeout} reached"
               break
             end
+            log.info "[#{current_chunk_id}]:Going to retry the upload at #{Time.now + sleep_interval}"
             sleep(sleep_interval)
             sleep_interval *= 2
+            if sleep_interval > @maxretry_interval
+                sleep_interval = @maxretry_interval
+            end
           else
             return
           end
@@ -599,8 +634,7 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
         ml_regex_applier(@ml_trace, @ml_data)
         if @ml_trace_buffer
           if !(@formatted_line.empty?)
-            @formatted_line[@message_key] = @formatted_line[@message_key] + @ml_trace_buffer
-            @log_size += @ml_trace_buffer.bytesize  
+            @formatted_line[@message_key] = @formatted_line[@message_key] + @ml_trace_buffer 
           else
             @ml_trace += @ml_trace_buffer.gsub('\n', '<NewLine>')
             ml_regex_applier(@ml_trace, @ml_data)
@@ -615,10 +649,8 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
         @json_data = ''
       elsif @old_formatted_line
         @formatted_line.update(@old_formatted_line)
-        log_line_filter()
-        @log_size += @old_log_size
+        data_collector()
         @old_formatted_line = {}
-        @old_log_size = 0
       end
       @logged = true
       if @format_record
@@ -654,9 +686,7 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
                 end
               end
               @formatted_line[key] = field_value
-              @log_size -= adjust_length
             else
-              @log_size -= (@formatted_line[key].bytesize - @masking_config[key]['string'].bytesize)
               @formatted_line[key] = @masking_config[key]['string']
             end
           end
@@ -693,11 +723,6 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
                   end
                 end
               end
-            end 
-            if adjust_length
-              @log_size -= adjust_length
-            else
-              @log_size -= (@formatted_line[key].bytesize - field_value.bytesize)
             end
             @formatted_line[key] = field_value
           end
@@ -716,10 +741,7 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
               if @formatted_line.has_key?key
                 match_derived = each.match(@formatted_line[key])
                 if match_derived
-                  @formatted_line.update(match_derived.named_captures)
-                  for field_name,value in match_derived.named_captures do
-                    @log_size += @formatted_line[field_name].bytesize
-                  end  
+                  @formatted_line.update(match_derived.named_captures) 
                 end
                 break
               end
@@ -727,10 +749,19 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
         end
       rescue Exception => e
         log.error "Exception occurred in derived fields : #{e.backtrace}"
-      end  
+      end
     end
   end
   
+  def log_size_calculation()
+    data_exclusion = ["_zl", "s247", "inode"]
+    @formatted_line.each do |field, value|
+      unless data_exclusion.any? { |exclusion| field.start_with?(exclusion) }
+        @log_size += value.to_s.bytesize
+      end
+    end
+  end
+
   def timer_task()
     while true
       @after_time = Time.now
@@ -741,7 +772,7 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
           if out != nil
             out = Yajl.dump([out])
             out = gzip_compress(out)
-            send_logs_to_s247(out, @log_size)
+            send_logs_to_s247(out, @log_size, 'holded_line')
           end
         end
       end
@@ -749,4 +780,4 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
     end
   end
 
-end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-site24x7
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.6
 platform: ruby
 authors:
 - Magesh Rajan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-12-05 00:00:00.000000000 Z
+date: 2023-08-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -152,7 +152,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Site24x7 output plugin for Fluent event collector.