fluent-plugin-site24x7 0.1.2 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c2391265977dd8a33dde6dab1876296406fae59c7633aa0ec0045821bb5688f1
-  data.tar.gz: 6bd7a51fca919c3c38c4f12cf226f403772c473d12f8c2a716f72ccb844f5ec7
+  metadata.gz: eaa27dafc08120c093de5e57ba15b129209637db8759e68a20c2053513cb55e9
+  data.tar.gz: 6fc10de6e63e332c816935a656cfb234e733710c7849f7578ebaa035eec7a8d4
 SHA512:
-  metadata.gz: aab17017da06b62df8c7ebc8b2ee5e045e9833687945fd1f262890bd32bc9478f58208e1e07675e4295dd5c05fc6691bad50a19c35ea635c50d627438bb6dd01
-  data.tar.gz: de638056920dcc7cce4a00c1bddad3a4628ce5342e3b837e430d599f5f0db949ffcd81ff18efa55eca2e2d07b8dec761f3bd673367a076af3c69e8dcee6dd7ef
+  metadata.gz: 58d44f268199174dabf93bc3b0fc3ceb0add74b2718001aaab95c3c380bf4f2b18bec949162a47a778a7d78d80a067302cc509506ceac6f8a2f21d5dedcc60ba
+  data.tar.gz: bd0dc479e64382935a9154006b281b46cad6d93c883b18d5ddb1cbd026ae49d427a35c0e4836ee6cf7eb1996d6d3557cd5d3bb4f862d9149af5655be862ad1d5

data/README.md

@@ -1,6 +1,6 @@
 # Fluentd output plugin for Site24x7
 
-This output plugin allows parse and sending logs directly from Fluentd to Site2x7 - so you don't have to use a separate log shipper.
+With Site24x7 plugin for Fluentd, you can parse and send logs directly from Fluentd, without having to use a separate log shipper.
 
 
 ## Installation
@@ -10,7 +10,7 @@ To add the plugin to your fluentd agent, use the following command:
 ```
 $ gem install fluent-plugin-site24x7
 ```
-If you installed the td-agent instead
+If you have installed the td-agent instead
 
 ```
 $ /usr/sbin/td-agent-gem install fluent-plugin-site24x7
@@ -20,7 +20,7 @@ $ /usr/sbin/td-agent-gem install fluent-plugin-site24x7
 
 **Configure the output plugin**
 
-To match events and send them to Site24x7, simply add the following code to your configuration file.
+To match events and send them to Site24x7, add the following code to your configuration file.
 
 ```cfg
 # Match events tagged with "site24x7.**" and send them to Site24x7
@@ -49,21 +49,22 @@ To match events and send them to Site24x7, simply add the following code to your
 
 </match>
 ```
-After a restart of FluentD, any events tagged with site24x7 are shipped to site24x7 platform.
+After restarting FluentD, any events tagged with site24x7 are shipped to Site24x7 platform.
 
 ## Parameters
-As fluent-plugin-site24x7 is an output_buffer, you can set all output_buffer properties like it's describe in the [fluentd documentation](http://docs.fluentd.org/articles/output-plugin-overview#buffered-output-parameters).
+As fluent-plugin-site24x7 is an output_buffer, you can set all the output_buffer properties like it's described in the [fluentd documentation](http://docs.fluentd.org/articles/output-plugin-overview#buffered-output-parameters).
 
 Property | Description | Default Value
 ------------ | -------------|------------
-log_type_config | log_type_config for your configured log type in site24x7 | nil
-max_retry | How many times to resend failed uploads | 3
-retry_interval |  How long to sleep initially between retries, exponential step-off | 2 seconds
-http_idle_timeout | Timeout in seconds that the http persistent connection will stay open without traffic | 5 seconds
-http_read_timeout | Timeout in seconds when the socket connects until the connection breaks | 30 secods
+log_type_config | log_type_config of your configured log type in site24x7 | nil
+max_retry | Number of times to resend failed uploads | 3
+retry_interval |  Time interval to sleep initially between retries, exponential step-off | 2 seconds
+http_idle_timeout | Timeout (in seconds) for which http persistent connection will stay open without traffic | 5 seconds
+http_read_timeout | Timeout (in seconds) when the socket connects until the connection breaks | 30 secods
 http_proxy | Your proxy uri | nil
 
 ## Release Notes
-
+* 0.1.2 - Timezone parsing issue fixed
+* 0.1.1 - Minor changes
 * 0.1.0 - Initial Release
 

data/fluent-plugin-site24x7.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-site24x7"
-  spec.version = "0.1.2"
+  spec.version = "0.1.4"
   spec.authors = ["Magesh Rajan"]
   spec.email   = ["magesh.rajan@zohocorp.com"]
 

data/lib/fluent/plugin/out_site24x7.rb

@@ -5,6 +5,8 @@ require "yajl"
 require "zlib"
 require "date"
 require "fluent/plugin/output"
+require 'digest'
+require 'json'
 
 class Fluent::Site24x7Output < Fluent::Plugin::Output
   
@@ -73,6 +75,80 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
     @logtype_config = Yajl::Parser.parse(base64_url_decode(@log_type_config))
     @s247_custom_regex = if @logtype_config.has_key? 'regex' then Regexp.compile(@logtype_config['regex'].gsub('?P<','?<')) else nil end
     @s247_ignored_fields = if @logtype_config.has_key? 'ignored_fields' then @logtype_config['ignored_fields'] else [] end
+    @datetime_regex = if @logtype_config.has_key?'dateRegex' then Regexp.compile(@logtype_config['dateRegex'].gsub('?P<','?<')) else nil end
+
+    @ml_regex = if @logtype_config.has_key? 'ml_regex' then Regexp.compile(@logtype_config['ml_regex'].gsub('?P<','?<')) else nil end
+    @ml_end_regex = if @logtype_config.has_key? 'ml_end_regex' then Regexp.compile(@logtype_config['ml_end_regex'].gsub('?P<','?<')) else nil end
+    @max_ml_count = if @logtype_config.has_key? 'ml_regex' then @s247_custom_regex.inspect.scan('\<NewLine\>').length else nil end
+    @max_trace_line = 100
+    @ml_trace = ''
+    @ml_trace_buffer = ''
+    @ml_found = false
+    @ml_end_line_found = false
+    @ml_data = nil
+    @ml_count = 0
+    
+    @json_data = ''
+    @sub_pattern = {}
+
+    if !(@logtype_config.has_key?('jsonPath'))
+      @message_key = get_last_group_inregex(@s247_custom_regex)
+    end
+    
+    if @logtype_config.has_key?('jsonPath')
+      @logtype_config['jsonPath'].each_with_index do | key, index |
+        if key.has_key?('pattern')
+          begin
+            if Regexp.new(key['pattern'].gsub('?P<','?<'))
+              @sub_pattern[key['name']] = Regexp.compile(key['pattern'].gsub('?P<','?<'))
+            end
+          rescue Exception => e
+            log.error "Invalid subpattern regex #{e.backtrace}"
+          end
+        end
+      end 
+    end
+
+    @old_formatted_line = {}
+    @formatted_line = {}
+
+    @masking_config = if @logtype_config.has_key? 'maskingConfig' then  @logtype_config['maskingConfig']  else nil end
+    @hashing_config = if @logtype_config.has_key? 'hashingConfig' then  @logtype_config['hashingConfig']  else nil end
+    @derived_config = if @logtype_config.has_key? 'derivedConfig' then @logtype_config['derivedConfig'] else nil end
+    @general_regex = Regexp.compile("(.*)")
+    
+    if @derived_config
+      @derived_fields = {}
+      for key,value in @derived_config do
+        @derived_fields[key] = []
+        for values in @derived_config[key] do
+          @derived_fields[key].push(Regexp.compile(values.gsub('\\\\', '\\')))
+        end
+      end
+    end
+    
+    if @masking_config
+      for key,value in @masking_config do
+        @masking_config[key]["regex"] = Regexp.compile(@masking_config[key]["regex"])
+      end
+    end
+    
+    if @hashing_config
+      for key,value in @hashing_config do
+        @hashing_config[key]["regex"] = Regexp.compile(@hashing_config[key]["regex"])
+      end
+    end
+
+    if @logtype_config.has_key?'filterConfig'
+      for field,rules in @logtype_config['filterConfig'] do
+        temp = []
+        for value in @logtype_config['filterConfig'][field]['values'] do
+          temp.push(Regexp.compile(value))
+        end
+        @logtype_config['filterConfig'][field]['values'] = temp.join('|') 
+      end
+    end  
+    
     @s247_tz = {'hrs': 0, 'mins': 0} #UTC
     @log_source = Socket.gethostname
     @valid_logtype = true
@@ -87,6 +163,7 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
       end   
       @is_timezone_present = if @s247_datetime_format_string.include? '%z' then true else false end
       if !@is_timezone_present && @logtype_config.has_key?('timezone')
+  @s247_datetime_format_string += '%z'
 	tz_value = @logtype_config['timezone']
 	if tz_value.start_with?('+')
 	    @s247_tz['hrs'] = Integer('-' + tz_value[1..4])
@@ -97,6 +174,7 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
 	end
       end
     end
+    Thread.new { timer_task() }
   end
 
   def init_http_client(logtype_config)
@@ -116,7 +194,7 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
     @s247_http_client.override_headers["User-Agent"] = 'Fluentd'
     if !@s247_http_client.proxy_uri.nil?
         log.info "Using HTTP proxy #{@s247_http_client.proxy_uri.scheme}://#{@s247_http_client.proxy_uri.host}:#{@s247_http_client.proxy_uri.port} username: #{@s247_http_client.proxy_uri.user ? "configured" : "not configured"}, password: #{@s247_http_client.proxy_uri.password ? "configured" : "not configured"}"
-    end
+    end    
   end
 
   def get_timestamp(datetime_string)
@@ -127,113 +205,262 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
         end
         datetime_string += if !@is_year_present then ' '+String(Time.new.year) else '' end
         if !@is_timezone_present && @logtype_config.has_key?('timezone')
-            @s247_datetime_format_string += '%z'
             time_zone = String(@s247_tz['hrs'])+':'+String(@s247_tz['mins'])
             datetime_string += if time_zone.start_with?('-') then time_zone else '+'+time_zone end
         end
         datetime_data = DateTime.strptime(datetime_string, @s247_datetime_format_string)
         return Integer(datetime_data.strftime('%Q'))
-    rescue
+    rescue Exception => e
+      @logger.error "Exception in parsing date: #{e.backtrace}"
         return 0
     end
   end
 
+  def log_line_filter()
+    applyMasking()
+    applyHashing()
+    getDerivedFields()
+  end 
+
+  def get_last_group_inregex(s247_custom_regex)
+    return @s247_custom_regex.names[-1]
+  end
+
+  def remove_ignored_fields()
+    @s247_ignored_fields.each do |field_name|
+      @log_size -= if @log_fields.has_key?field_name then @log_fields.delete(field_name).bytesize else 0 end
+    end
+  end
+
+  def add_message_metadata()
+    @log_fields.update({'_zl_timestamp' => get_timestamp(@log_fields[@logtype_config['dateField']]), 's247agentuid' => @log_source})
+  end
+
   def parse_lines(lines)
     parsed_lines = []
-    log_size = 0
     lines.each do |line|
-       if !line.empty?
-	    begin
-		if match = line.match(@s247_custom_regex)
-                    log_size += line.bytesize
-		    log_fields = match&.named_captures
-		    removed_log_size=0
-		    @s247_ignored_fields.each do |field_name|
-		        removed_log_size += if log_fields.has_key?field_name then log_fields.delete(field_name).bytesize else 0 end
-		    end
-		    formatted_line = {'_zl_timestamp' => get_timestamp(log_fields[@logtype_config['dateField']]), 's247agentuid' => @log_source}
-		    formatted_line.merge!(log_fields)
-                    parsed_lines.push(formatted_line)
-		    log_size -= removed_log_size
-                else
-                    log.debug "pattern not matched regex : #{@s247_custom_regex} and received line : #{line}"
-		end
-	    rescue Exception => e
-		log.error "Exception in parse_line #{e.backtrace}"
-	    end
-       end
+      if !line.empty?
+        begin
+          @logged = false
+          match = line.match(@s247_custom_regex)
+          if match
+            @formatted_line.update(@old_formatted_line)
+            @log_size += @old_log_size
+            @old_log_size = line.bytesize
+            @log_fields = match&.named_captures
+            remove_ignored_fields()
+            add_message_metadata()
+            @old_formatted_line = @log_fields
+            @last_line_matched = true
+            @trace_started = false             
+          elsif @last_line_matched || @trace_started
+            is_date_present = !(line.scan(@datetime_regex).empty?)
+            @trace_started = !(is_date_present)
+            if !(is_date_present) && @old_formatted_line
+              if @old_formatted_line.has_key?(@message_key)
+                @old_formatted_line[@message_key] += '\n' + line
+                @old_log_size += line.bytesize
+                @trace_started = true
+                @last_line_matched = false
+              end 
+            end
+          end   
+          if @formatted_line.has_key?('_zl_timestamp')
+            log_line_filter()
+            parsed_lines.push(@formatted_line)
+            @formatted_line = {}
+          end    
+        rescue Exception => e
+          log.error "Exception in parse_line #{e.backtrace}"
+          @formatted_line = {}
+        end
+      end
     end
-    return parsed_lines, log_size
+    return parsed_lines
   end
 
-  def is_filters_matched(formatted_line)
-    if @logtype_config.has_key?'filterConfig'
-        @logtype_config['filterConfig'].each do |config|
-            if formatted_line.has_key?config && (filter_config[config]['match'] ^ (filter_config[config]['values'].include?formatted_line[config]))
-                return false
-	    end
+  def is_filters_matched()
+    begin
+      if @logtype_config.has_key?'filterConfig'
+        @logtype_config['filterConfig'].each do |config,value|
+          if @formatted_line[config].scan(Regexp.new(@logtype_config['filterConfig'][config]['values'])).length > 0
+            val = true 
+          else
+            val = false
+          end
+          if (@formatted_line.has_key?config) && (@logtype_config['filterConfig'][config]['match'] ^ (val))
+            return false
+          end
         end
+      end
+    rescue Exception => e
+      log.error "Exception occurred in filter: #{e.backtrace}" 
     end
     return true
   end
 
   def get_json_value(obj, key, datatype=nil)
     if obj != nil && (obj.has_key?key)
-       if datatype and datatype == 'json-object'
-	  arr_json = []
-          child_obj = obj[key]
-          if child_obj.class == String
-             child_obj = Yajl::Parser.parse(child_obj.gsub('\\','\\\\'))
-          end             
-          child_obj.each do |key, value|
-             arr_json.push({'key' => key, 'value' => String(value)})
-          end
-          return arr_json
-       else
-         return (if obj.has_key?key then obj[key] else obj[key.downcase] end)
-       end
-    elsif key.include?'.'
-	parent_key = key[0..key.index('.')-1]
-	child_key = key[key.index('.')+1..-1]
-        child_obj = obj[if obj.has_key?parent_key then parent_key else parent_key.capitalize() end]
-	if child_obj.class == String
-            child_obj = Yajl::Parser.parse(child_obj.replace('\\','\\\\'))
+      if datatype and datatype == 'json-object'
+        arr_json = []
+        child_obj = obj[key]
+        if child_obj.class == String
+          child_obj = Yajl::Parser.parse(child_obj.gsub('\\','\\\\'))
+        end             
+        child_obj.each do |key, value|
+          arr_json.push({'key' => key, 'value' => String(value)})
         end
-        return get_json_value(child_obj, child_key)
+        return arr_json
+      else
+        return (if obj.has_key?key then obj[key] else obj[key.downcase] end)
+      end
+    elsif key.include?'.'
+      parent_key = key[0..key.index('.')-1]
+      child_key = key[key.index('.')+1..-1]
+      child_obj = obj[if obj.has_key?parent_key then parent_key else parent_key.capitalize() end]
+      if child_obj.class == String
+        child_obj = Yajl::Parser.parse(child_obj.replace('\\','\\\\'))
+      end
+      return get_json_value(child_obj, child_key,datatype)
     end
   end
 
-  def json_log_parser(lines_read)
-    log_size = 0
+  def json_log_applier(line)
+    json_log_size=0
+    @formatted_line = {}
+    @log_fields = {}
+    event_obj = if line.is_a?(String) then Yajl::Parser.parse(line) else line end
+    @logtype_config['jsonPath'].each do |path_obj|
+      value = get_json_value(event_obj, path_obj[if path_obj.has_key?'key' then 'key' else 'name' end], path_obj['type'])
+      if value
+        @log_fields[path_obj['name']] = value 
+        json_log_size+= String(value).bytesize - (if value.class == Array then value.size*20 else 0 end)
+      end
+    end
+    for key,regex in @sub_pattern do
+      if @log_fields.has_key?(key)
+        matcher = regex.match(@log_fields.delete(key))
+        if matcher
+          @log_fields.update(matcher.named_captures)
+          remove_ignored_fields()
+          @formatted_line.update(@log_fields)
+        end 
+      end
+    end
+    if !(is_filters_matched())
+      return false
+    else
+      add_message_metadata()
+      @formatted_line.update(@log_fields)
+      log_line_filter()
+      @log_size += json_log_size
+      return true
+     end
+  end
+
+  def json_log_parser(lines)
     parsed_lines = []
-    lines_read.each do |line|
+    lines.each do |line|
+      begin
+        @logged = false
         if !line.empty?
-          current_log_size = 0
-	  formatted_line = {}
-	  event_obj = Yajl::Parser.parse(line)
-	  @logtype_config['jsonPath'].each do |path_obj|
-	    value = get_json_value(event_obj, path_obj[if path_obj.has_key?'key' then 'key' else 'name' end], path_obj['type'])
-            if value
-	      formatted_line[path_obj['name']] = value 
-	      current_log_size+= String(value).size - (if value.class == Array then value.size*20 else 0 end)
-            end
+          if line[0] == '{' && @json_data[-1] == '}'
+            if json_log_applier(@json_data)
+              parsed_lines.push(@formatted_line)
+              end
+            @json_data=''
           end
-	  if is_filters_matched(formatted_line)
-	    formatted_line['_zl_timestamp'] = get_timestamp(formatted_line[@logtype_config['dateField']])
-	    formatted_line['s247agentuid'] = @log_source
-	    parsed_lines.push(formatted_line)
-            log_size+=current_log_size
+          @json_data += line
+        end
+      rescue Exception => e
+        log.error "Exception in parse_line #{e.backtrace}"
+      end
+    end
+    return parsed_lines
+  end
+
+  def ml_regex_applier(ml_trace, ml_data)
+    begin    
+      @log_size += @ml_trace.bytesize
+      matcher = @s247_custom_regex.match(@ml_trace)  
+      @log_fields = matcher.named_captures
+      @log_fields.update(@ml_data)
+      if @s247_ignored_fields
+        remove_ignored_fields()
+      end
+      add_message_metadata()
+      @formatted_line.update(@log_fields)
+      log_line_filter()
+    rescue Exception => e
+      log.error "Exception occurred in ml_parser : #{e.backtrace}"
+      @formatted_line = {}
+    end
+  end
+  
+  def ml_log_parser(lines)
+    parsed_lines = []
+    lines.each do |line|
+      if !line.empty?
+        begin
+          @logged = false
+          ml_start_matcher = @ml_regex.match(line)
+          if ml_start_matcher || @ml_end_line_found
+            @ml_found = ml_start_matcher
+            @ml_end_line_found = false
+            @formatted_line = {}
+            if @ml_trace.length > 0 
+              begin
+                  ml_regex_applier(@ml_trace, @ml_data)
+                  if @ml_trace_buffer && @formatted_line
+                      @formatted_line[@message_key] = @formatted_line[@message_key] + @ml_trace_buffer
+                      @log_size += @ml_trace_buffer.bytesize
+                  end
+                  parsed_lines.push(@formatted_line)
+                  @ml_trace = ''
+                  @ml_trace_buffer = ''
+                  if @ml_found
+                    @ml_data = ml_start_matcher.named_captures
+                    @log_size += line.bytesize
+                  else
+                      @ml_data = {}
+                  end
+                  @ml_count = 0
+              rescue Exception => e
+                log.error "Exception occurred in ml_parser : #{e.backtrace}"
+                end
+            elsif @ml_found
+              @log_size += line.bytesize
+              @ml_data = ml_start_matcher.named_captures
+            end
+          elsif @ml_found
+            if @ml_count < @max_ml_count
+              @ml_trace += '<NewLine>' + line
+            elsif @ml_end_regex && @ml_end_regex.match(line)
+              @ml_end_line_found = True
+            elsif (@ml_count - @max_ml_count) < @max_trace_line
+              @ml_trace_buffer += "\n" + line
+            end
+            @ml_count += 1
           end
-	end
+        rescue Exception => e
+          log.error "Exception occurred in ml_parser : #{e.backtrace}"
+        end
+      end
     end
-    return parsed_lines, log_size
+    return parsed_lines
   end
 
   def format(tag, time, record)
     if @valid_logtype && (@log_upload_allowed || (time.to_i - @log_upload_stopped_time > S247_LOG_UPLOAD_CHECK_INTERVAL))
-       [record['message']].to_msgpack
+      if (record.size == 1)
+        if record.has_key?'message'
+          [record['message']].to_msgpack
+        end
+      else
+        [record.to_json].to_msgpack
+      end
     end
-  end
+  end     
 
   def write(chunk)
     begin
@@ -249,12 +476,15 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
   end
 
   def process_http_events(events)
+    @before_time = Time.now
     batches = batch_http_events(events)
     batches.each do |batched_event|
-      formatted_events, log_size = format_http_event_batch(batched_event)
-      formatted_events = gzip_compress(formatted_events)
-      send_logs_to_s247(formatted_events, log_size)
-    end
+      formatted_events, @log_size = format_http_event_batch(batched_event)
+      if (formatted_events.length>0)
+        formatted_events = gzip_compress(formatted_events)
+        send_logs_to_s247(formatted_events, @log_size)
+      end
+    end 
   end
 
   def batch_http_events(encoded_events)
@@ -262,7 +492,7 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
     current_batch = []
     current_batch_size = 0
     encoded_events.each_with_index do |encoded_event, i|
-      current_event_size = encoded_event.bytesize
+      current_event_size = String(encoded_event).bytesize
       if current_event_size > S247_MAX_RECORD_SIZE
         encoded_event = encoded_event[0..(S247_MAX_RECORD_SIZE-DD_TRUNCATION_SUFFIX.length)]+DD_TRUNCATION_SUFFIX
         current_event_size = encoded_event.bytesize
@@ -283,13 +513,19 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
 
   def format_http_event_batch(events)
     parsed_lines = []
-    log_size = 0
+    @log_size = 0
+    @old_log_size=0
     if @logtype_config.has_key?'jsonPath'
-      parsed_lines, log_size = json_log_parser(events)
+      parsed_lines = json_log_parser(events)
+    elsif @logtype_config.has_key?'ml_regex'
+      parsed_lines = ml_log_parser(events)
     else
-       parsed_lines, log_size = parse_lines(events)
+      parsed_lines = parse_lines(events)
+    end
+    if (parsed_lines.length > 0)
+      return Yajl.dump(parsed_lines), @log_size
     end
-    return Yajl.dump(parsed_lines), log_size
+    return [],0
   end
 
   def gzip_compress(payload)
@@ -307,7 +543,7 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
   def send_logs_to_s247(gzipped_parsed_lines, log_size)
       request = Net::HTTP::Post.new @uri.request_uri
       request.body = gzipped_parsed_lines
-      @s247_http_client.override_headers["Log-Size"] = log_size
+      @s247_http_client.override_headers["Log-Size"] = @log_size
       sleep_interval = @retry_interval
      begin
         @max_retry.times do |counter|
@@ -320,17 +556,17 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
               if resp_headers.has_key?'LOG_LICENSE_EXCEEDS' && resp_headers['LOG_LICENSE_EXCEEDS'] == 'True'
                 log.error "Log license limit exceeds so not able to send logs"
                 @log_upload_allowed = false
-		@log_upload_stopped_time =Time.now.to_i
+		            @log_upload_stopped_time =Time.now.to_i
               elsif resp_headers.has_key?'BLOCKED_LOGTYPE' && resp_headers['BLOCKED_LOGTYPE'] == 'True'
                 log.error "Max upload limit reached for log type"
                 @log_upload_allowed = false
-		@log_upload_stopped_time =Time.now.to_i
+		            @log_upload_stopped_time =Time.now.to_i
               elsif resp_headers.has_key?'INVALID_LOGTYPE' && resp_headers['INVALID_LOGTYPE'] == 'True'
                 log.error "Log type not present in this account so stopping log collection"
-		@valid_logtype = false
+		            @valid_logtype = false
               else
-		@log_upload_allowed = true
-                log.debug "Successfully sent logs with size #{gzipped_parsed_lines.size} / #{log_size} to site24x7. Upload Id : #{resp_headers['x-uploadid']}"
+		            @log_upload_allowed = true
+                log.debug "Successfully sent logs with size #{gzipped_parsed_lines.size} / #{@log_size} to site24x7. Upload Id : #{resp_headers['x-uploadid']}"
               end
             else
               log.error "Response Code #{resp_headers} from Site24x7, so retrying (#{counter + 1}/#{@max_retry})"
@@ -356,4 +592,161 @@ class Fluent::Site24x7Output < Fluent::Plugin::Output
       end
   end
 
-end
+  def log_the_holded_line()
+    @log_size = 0
+    if @logged == false
+      if (@ml_trace.length>0)
+        ml_regex_applier(@ml_trace, @ml_data)
+        if @ml_trace_buffer
+          if !(@formatted_line.empty?)
+            @formatted_line[@message_key] = @formatted_line[@message_key] + @ml_trace_buffer
+            @log_size += @ml_trace_buffer.bytesize  
+          else
+            @ml_trace += @ml_trace_buffer.gsub('\n', '<NewLine>')
+            ml_regex_applier(@ml_trace, @ml_data)
+          end
+          @ml_trace_buffer = ''          
+        end    
+        @ml_trace = ''
+      elsif (@json_data.length>0)
+        if !(json_log_applier(@json_data))
+          @formatted_line={}
+        end
+        @json_data = ''
+      elsif @old_formatted_line
+        @formatted_line.update(@old_formatted_line)
+        log_line_filter()
+        @log_size += @old_log_size
+        @old_formatted_line = {}
+        @old_log_size = 0
+      end
+      @logged = true
+      if @format_record
+        @custom_parser.format_record()
+      end
+      if !(@formatted_line.empty?)
+        return @formatted_line
+      end
+    end
+    return nil
+  end
+
+  def applyMasking()
+    if @masking_config
+      begin
+        for key,value in @masking_config do
+          adjust_length = 0
+          mask_regex = @masking_config[key]["regex"]
+          if @formatted_line.has_key?key
+            field_value = @formatted_line[key]
+            if !(mask_regex.eql?(@general_regex))
+              matcher = field_value.to_enum(:scan, mask_regex).map { Regexp.last_match }
+              if matcher
+                (0..(matcher.length)-1).map do |index| 
+                  start = matcher[index].offset(1)[0]  
+                  _end = matcher[index].offset(1)[1]
+                  if ((start >= 0) && (_end > 0))
+                    start = start - adjust_length
+                    _end = _end - adjust_length
+                    adjust_length += (_end - start) - @masking_config[key]['string'].bytesize
+                    field_value = field_value[0..(start-1)] + @masking_config[key]['string'] + field_value[_end..field_value.bytesize]
+                  end
+                end
+              end
+              @formatted_line[key] = field_value
+              @log_size -= adjust_length
+            else
+              @log_size -= (@formatted_line[key].bytesize - @masking_config[key]['string'].bytesize)
+              @formatted_line[key] = @masking_config[key]['string']
+            end
+          end
+        end
+      rescue Exception => e
+        log.error "Exception occurred in masking : #{e.backtrace}"
+      end        
+    end
+  end
+  
+  def applyHashing()
+    if @hashing_config
+      begin
+        for key,value in @hashing_config do
+          hash_regex = @hashing_config[key]["regex"]
+          if @formatted_line.has_key?key 
+            field_value = @formatted_line[key]
+            if (hash_regex.eql?(@general_regex))
+              hash_string =  Digest::SHA256.hexdigest(field_value)
+              field_value = hash_string
+            else 
+              adjust_length = 0 
+              matcher = field_value.to_enum(:scan, hash_regex).map { Regexp.last_match }
+              if matcher
+                (0..(matcher.length)-1).map do |index| 
+                  start = matcher[index].offset(1)[0] 
+                  _end = matcher[index].offset(1)[1] 
+                  if ((start >= 0) && (_end > 0))
+                    start = start - adjust_length
+                    _end = _end - adjust_length
+                    hash_string =  Digest::SHA256.hexdigest(field_value[start..(_end-1)])
+                    adjust_length += (_end - start) - hash_string.bytesize
+                    field_value = field_value[0..(start-1)] + hash_string + field_value[_end..field_value.bytesize]
+                  end
+                end
+              end
+            end 
+            if adjust_length
+              @log_size -= adjust_length
+            else
+              @log_size -= (@formatted_line[key].bytesize - field_value.bytesize)
+            end
+            @formatted_line[key] = field_value
+          end
+        end
+      rescue Exception => e
+        log.error "Exception occurred in hashing : #{e.backtrace}"
+      end
+    end
+  end
+  
+  def getDerivedFields()
+    if @derived_config
+      begin
+        for key,value in @derived_fields do
+          for each in @derived_fields[key] do
+              if @formatted_line.has_key?key
+                match_derived = each.match(@formatted_line[key])
+                if match_derived
+                  @formatted_line.update(match_derived.named_captures)
+                  for field_name,value in match_derived.named_captures do
+                    @log_size += @formatted_line[field_name].bytesize
+                  end  
+                end
+                break
+              end
+          end
+        end
+      rescue Exception => e
+        log.error "Exception occurred in derived fields : #{e.backtrace}"
+      end  
+    end
+  end
+  
+  def timer_task()
+    while true
+      @after_time = Time.now
+      if @before_time
+        diff = @after_time-@before_time
+        if diff.to_i > 29
+          out = log_the_holded_line()
+          if out != nil
+            out = Yajl.dump([out])
+            out = gzip_compress(out)
+            send_logs_to_s247(out, @log_size)
+          end
+        end
+      end
+      sleep(30)
+    end
+  end
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-site24x7
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.4
 platform: ruby
 authors:
 - Magesh Rajan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-12 00:00:00.000000000 Z
+date: 2022-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -152,7 +152,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.24
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Site24x7 output plugin for Fluent event collector.