fluent-plugin-site24x7 0.1.3 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a33475317d8f77918035c64d70be3fbeb32ea214554da51753f83b51904c7f68
-  data.tar.gz: 31284ae4e2d074f1545c7bb6641886bc12b3a4ca2ed6fd5de2a12e319409a2bc
+  metadata.gz: f625b690765ab8a387bfef2ce93ce69a8dbcd42130248fe72913e6947395e204
+  data.tar.gz: a47d573e473f76802523ac3e88951c86e058f7577aff44b4fdf1ef221e4826c9
 SHA512:
-  metadata.gz: c90315b127043a5ce466c5af1aada7b26fa15614b4e498fd802ef3b88bf145d56dd8fe6e8f038b564750bc5b9d40d3be5f2c83de6c2dfcfd3ee09f5e199b7d6b
-  data.tar.gz: 75c9d7cc52685aef20717a37e544dd7adc4b31951fa984449ec9ee7f23f58366ebf85bde6f54716008d88c2923d6013fb6ad836d76807a9550d1b2a2e9037298
+  metadata.gz: 450f8c60b8cf1a193d5a2ed8265b2cd9b223368f1df22c37459cdc8d5ca0d21590b6241749432c9b15b5595ae069dfb59b4810bf39e7b40f9bb43193f56eb1bd
+  data.tar.gz: d06c06451ed65bdb99b5e61cb7445d22554bb78d6777609ba0faace2a860135f51478fe003d56a023202584d0974eb2d7f8e952a4133e6137cc96bfe01e86b86

data/fluent-plugin-site24x7.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-site24x7"
-  spec.version = "0.1.3"
+  spec.version = "0.1.5"
   spec.authors = ["Magesh Rajan"]
   spec.email   = ["magesh.rajan@zohocorp.com"]
 

data/lib/fluent/plugin/out_site24x7.rb

@@ -5,6 +5,8 @@ require "yajl"
 require "zlib"
 require "date"
 require "fluent/plugin/output"
+require 'digest'
+require 'json'
 
 class Fluent::Site24x7Output < Fluent::Plugin::Output
   
@@ -73,6 +75,84 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
     @logtype_config = Yajl::Parser.parse(base64_url_decode(@log_type_config))
     @s247_custom_regex = if @logtype_config.has_key? 'regex' then Regexp.compile(@logtype_config['regex'].gsub('?P<','?<')) else nil end
     @s247_ignored_fields = if @logtype_config.has_key? 'ignored_fields' then @logtype_config['ignored_fields'] else [] end
+    @datetime_regex = if @logtype_config.has_key?'dateRegex' then Regexp.compile(@logtype_config['dateRegex'].gsub('?P<','?<')) else nil end
+
+    @ml_regex = if @logtype_config.has_key? 'ml_regex' then Regexp.compile(@logtype_config['ml_regex'].gsub('?P<','?<')) else nil end
+    @ml_end_regex = if @logtype_config.has_key? 'ml_end_regex' then Regexp.compile(@logtype_config['ml_end_regex'].gsub('?P<','?<')) else nil end
+    @max_ml_count = if @logtype_config.has_key? 'ml_regex' then @s247_custom_regex.inspect.scan('\<NewLine\>').length else nil end
+    @max_trace_line = 100
+    @ml_trace = ''
+    @ml_trace_buffer = ''
+    @ml_found = false
+    @ml_end_line_found = false
+    @ml_data = nil
+    @ml_count = 0
+    
+    @json_data = ''
+    @sub_pattern = {}
+
+    if !(@logtype_config.has_key?('jsonPath'))
+      @message_key = get_last_group_inregex(@s247_custom_regex)
+    end
+    
+    if @logtype_config.has_key?('jsonPath')
+      @logtype_config['jsonPath'].each_with_index do | key, index |
+        if key.has_key?('pattern')
+          begin
+            if Regexp.new(key['pattern'].gsub('?P<','?<'))
+              @sub_pattern[key['name']] = Regexp.compile(key['pattern'].gsub('?P<','?<'))
+            end
+          rescue Exception => e
+            log.error "Invalid subpattern regex #{e.backtrace}"
+          end
+        end
+      end 
+    end
+
+    @old_formatted_line = {}
+    @formatted_line = {}
+
+    @masking_config = if @logtype_config.has_key? 'maskingConfig' then  @logtype_config['maskingConfig']  else nil end
+    @hashing_config = if @logtype_config.has_key? 'hashingConfig' then  @logtype_config['hashingConfig']  else nil end
+    @derived_config = if @logtype_config.has_key? 'derivedConfig' then @logtype_config['derivedConfig'] else nil end
+    @general_regex = Regexp.compile("(.*)")
+    
+    if @derived_config
+      @derived_fields = {}
+      for key,value in @derived_config do
+        @derived_fields[key] = []
+        for values in @derived_config[key] do
+          @derived_fields[key].push(Regexp.compile(values.gsub('\\\\', '\\')))
+        end
+      end
+    end
+    
+    if @masking_config
+      for key,value in @masking_config do
+        @masking_config[key]["regex"] = Regexp.compile(@masking_config[key]["regex"])
+      end
+    end
+    
+    if @hashing_config
+      for key,value in @hashing_config do
+        @hashing_config[key]["regex"] = Regexp.compile(@hashing_config[key]["regex"])
+      end
+    end
+
+    if @logtype_config.has_key?'filterConfig'
+      for field,rules in @logtype_config['filterConfig'] do
+        temp = []
+        for value in @logtype_config['filterConfig'][field]['values'] do
+          if @logtype_config['filterConfig'][field]['exact']
+            temp.push("\\A"+value+"\\Z")
+          else
+            temp.push(value)
+          end
+        end
+        @logtype_config['filterConfig'][field]['values'] = Regexp.compile(temp.join('|')) 
+      end
+    end  
+    
     @s247_tz = {'hrs': 0, 'mins': 0} #UTC
     @log_source = Socket.gethostname
     @valid_logtype = true
@@ -87,16 +167,18 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
       end   
       @is_timezone_present = if @s247_datetime_format_string.include? '%z' then true else false end
       if !@is_timezone_present && @logtype_config.has_key?('timezone')
+  @s247_datetime_format_string += '%z'
 	tz_value = @logtype_config['timezone']
-	if tz_value.start_with?('+')
-	    @s247_tz['hrs'] = Integer('-' + tz_value[1..4])
-	    @s247_tz['mins'] = Integer('-' + tz_value[3..6])
-	elsif tz_value.start_with?('-')
-	    @s247_tz['hrs'] = Integer('+' + tz_value[1..4])
-	    @s247_tz['mins'] = Integer('+' + tz_value[3..6])
-	end
+  if tz_value.start_with?('+')
+    @s247_tz['hrs'] = Integer('-' + tz_value[1..2])
+    @s247_tz['mins'] = Integer('-' + tz_value[3..4])
+  elsif tz_value.start_with?('-')
+    @s247_tz['hrs'] = Integer('+' + tz_value[1..2])
+    @s247_tz['mins'] = Integer('+' + tz_value[3..4])
+  end
       end
     end
+    Thread.new { timer_task() }
   end
 
   def init_http_client(logtype_config)
@@ -116,7 +198,7 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
     @s247_http_client.override_headers["User-Agent"] = 'Fluentd'
     if !@s247_http_client.proxy_uri.nil?
         log.info "Using HTTP proxy #{@s247_http_client.proxy_uri.scheme}://#{@s247_http_client.proxy_uri.host}:#{@s247_http_client.proxy_uri.port} username: #{@s247_http_client.proxy_uri.user ? "configured" : "not configured"}, password: #{@s247_http_client.proxy_uri.password ? "configured" : "not configured"}"
-    end
+    end    
   end
 
   def get_timestamp(datetime_string)
@@ -127,113 +209,265 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
         end
         datetime_string += if !@is_year_present then ' '+String(Time.new.year) else '' end
         if !@is_timezone_present && @logtype_config.has_key?('timezone')
-            @s247_datetime_format_string += '%z'
             time_zone = String(@s247_tz['hrs'])+':'+String(@s247_tz['mins'])
             datetime_string += if time_zone.start_with?('-') then time_zone else '+'+time_zone end
         end
         datetime_data = DateTime.strptime(datetime_string, @s247_datetime_format_string)
         return Integer(datetime_data.strftime('%Q'))
-    rescue
+    rescue Exception => e
+      @logger.error "Exception in parsing date: #{e.backtrace}"
         return 0
     end
   end
 
+  def data_collector()
+    if @formatted_line.has_key?('_zl_timestamp')
+      applyMasking()
+      applyHashing()
+      getDerivedFields()
+      if !is_filters_matched()
+        @formatted_line = {}
+        return
+      end
+      remove_ignored_fields()
+      log_size_calculation()
+    else
+      @formatted_line = {}
+      return
+    end
+  end 
+
+  def get_last_group_inregex(s247_custom_regex)
+    return @s247_custom_regex.names[-1]
+  end
+
+  def remove_ignored_fields()
+    @s247_ignored_fields.each do |field_name|
+      if @formatted_line.has_key?field_name
+        @formatted_line.delete(field_name)
+      end
+    end
+  end
+
+  def add_message_metadata()
+    @log_fields.update({'_zl_timestamp' => get_timestamp(@log_fields[@logtype_config['dateField']]), 's247agentuid' => @log_source})
+  end
+
   def parse_lines(lines)
     parsed_lines = []
-    log_size = 0
     lines.each do |line|
-       if !line.empty?
-	    begin
-		if match = line.match(@s247_custom_regex)
-                    log_size += line.bytesize
-		    log_fields = match&.named_captures
-		    removed_log_size=0
-		    @s247_ignored_fields.each do |field_name|
-		        removed_log_size += if log_fields.has_key?field_name then log_fields.delete(field_name).bytesize else 0 end
-		    end
-		    formatted_line = {'_zl_timestamp' => get_timestamp(log_fields[@logtype_config['dateField']]), 's247agentuid' => @log_source}
-		    formatted_line.merge!(log_fields)
-                    parsed_lines.push(formatted_line)
-		    log_size -= removed_log_size
-                else
-                    log.debug "pattern not matched regex : #{@s247_custom_regex} and received line : #{line}"
-		end
-	    rescue Exception => e
-		log.error "Exception in parse_line #{e.backtrace}"
-	    end
-       end
+      if !line.empty?
+        begin
+          @logged = false
+          match = line.match(@s247_custom_regex)
+          if match
+            @formatted_line.update(@old_formatted_line)
+            @log_fields = match&.named_captures
+            add_message_metadata()
+            @old_formatted_line = @log_fields
+            @last_line_matched = true
+            @trace_started = false             
+          elsif @last_line_matched || @trace_started
+            is_date_present = !(line.scan(@datetime_regex).empty?)
+            @trace_started = !(is_date_present)
+            if !(is_date_present) && @old_formatted_line
+              if @old_formatted_line.has_key?(@message_key)
+                @old_formatted_line[@message_key] += '\n' + line
+                @old_log_size += line.bytesize
+                @trace_started = true
+                @last_line_matched = false
+              end
+            end
+          end
+          data_collector()
+          if @formatted_line.length >0
+            parsed_lines.push(@formatted_line)
+          end     
+        rescue Exception => e
+          log.error "Exception in parse_line #{e.backtrace}"
+          @formatted_line = {}
+        end
+      end
     end
-    return parsed_lines, log_size
+    return parsed_lines
   end
 
-  def is_filters_matched(formatted_line)
-    if @logtype_config.has_key?'filterConfig'
-        @logtype_config['filterConfig'].each do |config|
-            if formatted_line.has_key?config && (filter_config[config]['match'] ^ (filter_config[config]['values'].include?formatted_line[config]))
-                return false
-	    end
+  def is_filters_matched()
+    begin
+      if @logtype_config.has_key?'filterConfig'
+        @logtype_config['filterConfig'].each do |config,value|
+          if (@formatted_line.has_key?config)
+            if @logtype_config['filterConfig'][config]['values'].match(@formatted_line[config])
+              val = true 
+            else
+              val = false
+            end
+          end
+          if (@logtype_config['filterConfig'][config]['match'] ^ (val))
+            return false
+          end
         end
+      end
+    rescue Exception => e
+      log.error "Exception occurred in filter: #{e.backtrace}" 
     end
     return true
   end
 
   def get_json_value(obj, key, datatype=nil)
     if obj != nil && (obj.has_key?key)
-       if datatype and datatype == 'json-object'
-	  arr_json = []
-          child_obj = obj[key]
-          if child_obj.class == String
-             child_obj = Yajl::Parser.parse(child_obj.gsub('\\','\\\\'))
-          end             
-          child_obj.each do |key, value|
-             arr_json.push({'key' => key, 'value' => String(value)})
-          end
-          return arr_json
-       else
-         return (if obj.has_key?key then obj[key] else obj[key.downcase] end)
-       end
-    elsif key.include?'.'
-	parent_key = key[0..key.index('.')-1]
-	child_key = key[key.index('.')+1..-1]
-        child_obj = obj[if obj.has_key?parent_key then parent_key else parent_key.capitalize() end]
-	if child_obj.class == String
-            child_obj = Yajl::Parser.parse(child_obj.replace('\\','\\\\'))
+      if datatype and datatype == 'json-object'
+        arr_json = []
+        child_obj = obj[key]
+        if child_obj.class == String
+          child_obj = Yajl::Parser.parse(child_obj.gsub('\\','\\\\'))
+        end             
+        child_obj.each do |key, value|
+          arr_json.push({'key' => key, 'value' => String(value)})
         end
-        return get_json_value(child_obj, child_key)
+        return arr_json
+      else
+        return (if obj.has_key?key then obj[key] else obj[key.downcase] end)
+      end
+    elsif key.include?'.'
+      parent_key = key[0..key.index('.')-1]
+      child_key = key[key.index('.')+1..-1]
+      child_obj = obj[if obj.has_key?parent_key then parent_key else parent_key.capitalize() end]
+      if child_obj.class == String
+        child_obj = Yajl::Parser.parse(child_obj.replace('\\','\\\\'))
+      end
+      return get_json_value(child_obj, child_key,datatype)
     end
   end
 
-  def json_log_parser(lines_read)
-    log_size = 0
+  def json_log_applier(line)
+    json_log_size=0
+    @formatted_line = {}
+    @log_fields = {}
+    event_obj = if line.is_a?(String) then Yajl::Parser.parse(line) else line end
+    @logtype_config['jsonPath'].each do |path_obj|
+      value = get_json_value(event_obj, path_obj[if path_obj.has_key?'key' then 'key' else 'name' end], path_obj['type'])
+      if value
+        @log_fields[path_obj['name']] = value 
+        json_log_size+= String(value).bytesize - (if value.class == Array then value.size*20 else 0 end)
+      end
+    end
+    for key,regex in @sub_pattern do
+      if @log_fields.has_key?(key)
+        matcher = regex.match(@log_fields.delete(key))
+        if matcher
+          @log_fields.update(matcher.named_captures)
+          @formatted_line.update(@log_fields)
+        end 
+      end
+    end
+    add_message_metadata()
+    @formatted_line.update(@log_fields)
+    data_collector()
+    if @formatted_line.length >0
+      return true
+    else
+      return false
+    end 
+  end
+
+  def json_log_parser(lines)
     parsed_lines = []
-    lines_read.each do |line|
+    lines.each do |line|
+      begin
+        @logged = false
         if !line.empty?
-          current_log_size = 0
-	  formatted_line = {}
-          event_obj = if line.is_a?(String) then Yajl::Parser.parse(line) else line end
-	  @logtype_config['jsonPath'].each do |path_obj|
-	    value = get_json_value(event_obj, path_obj[if path_obj.has_key?'key' then 'key' else 'name' end], path_obj['type'])
-            if value
-	      formatted_line[path_obj['name']] = value 
-	      current_log_size+= String(value).size - (if value.class == Array then value.size*20 else 0 end)
+          if line[0] == '{' && @json_data[-1] == '}'
+            if json_log_applier(@json_data)
+              parsed_lines.push(@formatted_line)
             end
+            @json_data=''
           end
-	  if is_filters_matched(formatted_line)
-	    formatted_line['_zl_timestamp'] = get_timestamp(formatted_line[@logtype_config['dateField']])
-	    formatted_line['s247agentuid'] = @log_source
-	    parsed_lines.push(formatted_line)
-            log_size+=current_log_size
+          @json_data += line
+        end
+      rescue Exception => e
+        log.error "Exception in parse_line #{e.backtrace}"
+      end
+    end
+    return parsed_lines
+  end
+
+  def ml_regex_applier(ml_trace, ml_data)
+    begin
+      matcher = @s247_custom_regex.match(@ml_trace)  
+      @log_fields = matcher.named_captures
+      @log_fields.update(@ml_data)
+      add_message_metadata()
+      @formatted_line.update(@log_fields)
+    rescue Exception => e
+      log.error "Exception occurred in ml_parser : #{e.backtrace}"
+    end
+  end
+  
+  def ml_log_parser(lines)
+    parsed_lines = []
+    lines.each do |line|
+      if !line.empty?
+        begin
+          @logged = false
+          ml_start_matcher = @ml_regex.match(line)
+          if ml_start_matcher || @ml_end_line_found
+            @ml_found = ml_start_matcher
+            @ml_end_line_found = false
+            @formatted_line = {}
+            if @ml_trace.length > 0 
+              begin
+                  ml_regex_applier(@ml_trace, @ml_data)
+                  if @ml_trace_buffer && @formatted_line
+                      @formatted_line[@message_key] = @formatted_line[@message_key] + @ml_trace_buffer
+                  end
+                  data_collector()
+                  if @formatted_line.length >0
+                    parsed_lines.push(@formatted_line)
+                  end    
+                  @ml_trace = ''
+                  @ml_trace_buffer = ''
+                  if @ml_found
+                    @ml_data = ml_start_matcher.named_captures
+                  else
+                      @ml_data = {}
+                  end
+                  @ml_count = 0
+              rescue Exception => e
+                log.error "Exception occurred in ml_parser : #{e.backtrace}"
+                end
+            elsif @ml_found
+              @ml_data = ml_start_matcher.named_captures
+            end
+          elsif @ml_found
+            if @ml_count < @max_ml_count
+              @ml_trace += '<NewLine>' + line
+            elsif @ml_end_regex && @ml_end_regex.match(line)
+              @ml_end_line_found = True
+            elsif (@ml_count - @max_ml_count) < @max_trace_line
+              @ml_trace_buffer += "\n" + line
+            end
+            @ml_count += 1
           end
-	end
+        rescue Exception => e
+          log.error "Exception occurred in ml_parser : #{e.backtrace}"
+        end
+      end
     end
-    return parsed_lines, log_size
+    return parsed_lines
   end
 
   def format(tag, time, record)
     if @valid_logtype && (@log_upload_allowed || (time.to_i - @log_upload_stopped_time > S247_LOG_UPLOAD_CHECK_INTERVAL))
-      if !@logtype_config.has_key?'jsonPath' then [record['message']].to_msgpack  else [record].to_msgpack end
+      if (record.size == 1)
+        if record.has_key?'message'
+          [record['message']].to_msgpack
+        end
+      else
+        [record.to_json].to_msgpack
+      end
     end
-  end
+  end     
 
   def write(chunk)
     begin
@@ -249,12 +483,15 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
   end
 
   def process_http_events(events)
+    @before_time = Time.now
     batches = batch_http_events(events)
     batches.each do |batched_event|
-      formatted_events, log_size = format_http_event_batch(batched_event)
-      formatted_events = gzip_compress(formatted_events)
-      send_logs_to_s247(formatted_events, log_size)
-    end
+      formatted_events, @log_size = format_http_event_batch(batched_event)
+      if (formatted_events.length>0)
+        formatted_events = gzip_compress(formatted_events)
+        send_logs_to_s247(formatted_events, @log_size)
+      end
+    end 
   end
 
   def batch_http_events(encoded_events)
@@ -283,13 +520,19 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
 
   def format_http_event_batch(events)
     parsed_lines = []
-    log_size = 0
+    @log_size = 0
+    @old_log_size=0
     if @logtype_config.has_key?'jsonPath'
-      parsed_lines, log_size = json_log_parser(events)
+      parsed_lines = json_log_parser(events)
+    elsif @logtype_config.has_key?'ml_regex'
+      parsed_lines = ml_log_parser(events)
     else
-       parsed_lines, log_size = parse_lines(events)
+      parsed_lines = parse_lines(events)
     end
-    return Yajl.dump(parsed_lines), log_size
+    if (parsed_lines.length > 0)
+      return Yajl.dump(parsed_lines), @log_size
+    end
+    return [],0
   end
 
   def gzip_compress(payload)
@@ -307,7 +550,7 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
   def send_logs_to_s247(gzipped_parsed_lines, log_size)
       request = Net::HTTP::Post.new @uri.request_uri
       request.body = gzipped_parsed_lines
-      @s247_http_client.override_headers["Log-Size"] = log_size
+      @s247_http_client.override_headers["Log-Size"] = @log_size
       sleep_interval = @retry_interval
      begin
         @max_retry.times do |counter|
@@ -315,22 +558,25 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
           begin
             response = @s247_http_client.request @uri, request
             resp_headers = response.each_header.to_h
-
+            
             if response.code == '200'
               if resp_headers.has_key?'LOG_LICENSE_EXCEEDS' && resp_headers['LOG_LICENSE_EXCEEDS'] == 'True'
                 log.error "Log license limit exceeds so not able to send logs"
                 @log_upload_allowed = false
-		@log_upload_stopped_time =Time.now.to_i
+		            @log_upload_stopped_time =Time.now.to_i
               elsif resp_headers.has_key?'BLOCKED_LOGTYPE' && resp_headers['BLOCKED_LOGTYPE'] == 'True'
                 log.error "Max upload limit reached for log type"
                 @log_upload_allowed = false
-		@log_upload_stopped_time =Time.now.to_i
+		            @log_upload_stopped_time =Time.now.to_i
               elsif resp_headers.has_key?'INVALID_LOGTYPE' && resp_headers['INVALID_LOGTYPE'] == 'True'
                 log.error "Log type not present in this account so stopping log collection"
-		@valid_logtype = false
+		            @valid_logtype = false
+              elsif resp_headers['x-uploadid'] == nil
+                log.error "upload id is empty hence retry flag enabled #{gzipped_parsed_lines.size} / #{@log_size}"
+                need_retry = true
               else
-		@log_upload_allowed = true
-                log.debug "Successfully sent logs with size #{gzipped_parsed_lines.size} / #{log_size} to site24x7. Upload Id : #{resp_headers['x-uploadid']}"
+		            @log_upload_allowed = true
+                log.debug "Successfully sent logs with size #{gzipped_parsed_lines.size} / #{@log_size} to site24x7. Upload Id : #{resp_headers['x-uploadid']}"
               end
             else
               log.error "Response Code #{resp_headers} from Site24x7, so retrying (#{counter + 1}/#{@max_retry})"
@@ -356,4 +602,157 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
       end
   end
 
-end
+  def log_the_holded_line()
+    @log_size = 0
+    if @logged == false
+      if (@ml_trace.length>0)
+        ml_regex_applier(@ml_trace, @ml_data)
+        if @ml_trace_buffer
+          if !(@formatted_line.empty?)
+            @formatted_line[@message_key] = @formatted_line[@message_key] + @ml_trace_buffer 
+          else
+            @ml_trace += @ml_trace_buffer.gsub('\n', '<NewLine>')
+            ml_regex_applier(@ml_trace, @ml_data)
+          end
+          @ml_trace_buffer = ''          
+        end    
+        @ml_trace = ''
+      elsif (@json_data.length>0)
+        if !(json_log_applier(@json_data))
+          @formatted_line={}
+        end
+        @json_data = ''
+      elsif @old_formatted_line
+        @formatted_line.update(@old_formatted_line)
+        data_collector()
+        @old_formatted_line = {}
+      end
+      @logged = true
+      if @format_record
+        @custom_parser.format_record()
+      end
+      if !(@formatted_line.empty?)
+        return @formatted_line
+      end
+    end
+    return nil
+  end
+
+  def applyMasking()
+    if @masking_config
+      begin
+        for key,value in @masking_config do
+          adjust_length = 0
+          mask_regex = @masking_config[key]["regex"]
+          if @formatted_line.has_key?key
+            field_value = @formatted_line[key]
+            if !(mask_regex.eql?(@general_regex))
+              matcher = field_value.to_enum(:scan, mask_regex).map { Regexp.last_match }
+              if matcher
+                (0..(matcher.length)-1).map do |index| 
+                  start = matcher[index].offset(1)[0]  
+                  _end = matcher[index].offset(1)[1]
+                  if ((start >= 0) && (_end > 0))
+                    start = start - adjust_length
+                    _end = _end - adjust_length
+                    adjust_length += (_end - start) - @masking_config[key]['string'].bytesize
+                    field_value = field_value[0..(start-1)] + @masking_config[key]['string'] + field_value[_end..field_value.bytesize]
+                  end
+                end
+              end
+              @formatted_line[key] = field_value
+            else
+              @formatted_line[key] = @masking_config[key]['string']
+            end
+          end
+        end
+      rescue Exception => e
+        log.error "Exception occurred in masking : #{e.backtrace}"
+      end        
+    end
+  end
+  
+  def applyHashing()
+    if @hashing_config
+      begin
+        for key,value in @hashing_config do
+          hash_regex = @hashing_config[key]["regex"]
+          if @formatted_line.has_key?key 
+            field_value = @formatted_line[key]
+            if (hash_regex.eql?(@general_regex))
+              hash_string =  Digest::SHA256.hexdigest(field_value)
+              field_value = hash_string
+            else 
+              adjust_length = 0 
+              matcher = field_value.to_enum(:scan, hash_regex).map { Regexp.last_match }
+              if matcher
+                (0..(matcher.length)-1).map do |index| 
+                  start = matcher[index].offset(1)[0] 
+                  _end = matcher[index].offset(1)[1] 
+                  if ((start >= 0) && (_end > 0))
+                    start = start - adjust_length
+                    _end = _end - adjust_length
+                    hash_string =  Digest::SHA256.hexdigest(field_value[start..(_end-1)])
+                    adjust_length += (_end - start) - hash_string.bytesize
+                    field_value = field_value[0..(start-1)] + hash_string + field_value[_end..field_value.bytesize]
+                  end
+                end
+              end
+            end
+            @formatted_line[key] = field_value
+          end
+        end
+      rescue Exception => e
+        log.error "Exception occurred in hashing : #{e.backtrace}"
+      end
+    end
+  end
+  
+  def getDerivedFields()
+    if @derived_config
+      begin
+        for key,value in @derived_fields do
+          for each in @derived_fields[key] do
+              if @formatted_line.has_key?key
+                match_derived = each.match(@formatted_line[key])
+                if match_derived
+                  @formatted_line.update(match_derived.named_captures) 
+                end
+                break
+              end
+          end
+        end
+      rescue Exception => e
+        log.error "Exception occurred in derived fields : #{e.backtrace}"
+      end
+    end
+  end
+  
+  def log_size_calculation()
+    data_exclusion = ["_zl", "s247", "inode"]
+    @formatted_line.each do |field, value|
+      unless data_exclusion.any? { |exclusion| field.start_with?(exclusion) }
+        @log_size += value.to_s.bytesize
+      end
+    end
+  end
+
+  def timer_task()
+    while true
+      @after_time = Time.now
+      if @before_time
+        diff = @after_time-@before_time
+        if diff.to_i > 29
+          out = log_the_holded_line()
+          if out != nil
+            out = Yajl.dump([out])
+            out = gzip_compress(out)
+            send_logs_to_s247(out, @log_size)
+          end
+        end
+      end
+      sleep(30)
+    end
+  end
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-site24x7
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.5
 platform: ruby
 authors:
 - Magesh Rajan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-10-12 00:00:00.000000000 Z
+date: 2023-06-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler