fluent-plugin-mageshlogs 0.1.3 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aba9a82083adce89cc592536cffc069dbc17f0ef68ef8fdbe2ba41556b50bdc0
-  data.tar.gz: 1304f5b0e1762411a43d4ab8359e40b2143da21844236513eb42d1002816b249
+  metadata.gz: 8e1ccf593a6773573f2d03f10bd7927f73c537f4f3f6e7450843680acf07f3b3
+  data.tar.gz: 6d796f72c9cf59b18fa76a3f93a2b481f806f4b7dddf13b23e8e4d7be076d18b
 SHA512:
-  metadata.gz: 03d96572f114fc37ce05b9da44ac4319a4eccb5c9074d42471c18421619e26942f88a8645333b969ad896440aa3acb5df51b705a814bfc3c5ecccd4ff4e160b5
-  data.tar.gz: fc3b490505728166b99d45bc23c152a6ddd039e3116b6ac95cd4473c6cb88978e03536cca97a0c893bc37d08b268f9a06e439b667dae3385174c59b157847e0b
+  metadata.gz: 9b9afb3099a91664bb995c49ee25e3f24ff1a34920559fc9ba9eef4bce6aba6be46563f6d847669eeb631c90ad6e6a8ebca3e51105669c49121f0a10e3283a9b
+  data.tar.gz: 2cb975f206aa8ce23b6c8cdce35b57e278df0474df876d8dddd5fef94e407b2e1d1298f20048c3dd458e57f90611da8e69c220049689223bff6e131737c1cabc

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,51 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-mageshlogs (0.1.0)
+      fluentd (>= 0.14.10, < 2)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    concurrent-ruby (1.1.9)
+    cool.io (1.7.1)
+    fluentd (1.13.3)
+      bundler
+      cool.io (>= 1.4.5, < 2.0.0)
+      http_parser.rb (>= 0.5.1, < 0.8.0)
+      msgpack (>= 1.3.1, < 2.0.0)
+      serverengine (>= 2.2.2, < 3.0.0)
+      sigdump (~> 0.2.2)
+      strptime (>= 0.2.2, < 1.0.0)
+      tzinfo (>= 1.0, < 3.0)
+      tzinfo-data (~> 1.0)
+      webrick (>= 1.4.2, < 1.8.0)
+      yajl-ruby (~> 1.0)
+    http_parser.rb (0.7.0)
+    msgpack (1.4.2)
+    power_assert (1.2.0)
+    rake (13.0.3)
+    serverengine (2.2.4)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.4)
+    strptime (0.2.5)
+    test-unit (3.3.7)
+      power_assert
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    tzinfo-data (1.2021.1)
+      tzinfo (>= 1.0.0)
+    webrick (1.7.0)
+    yajl-ruby (1.4.1)
+
+PLATFORMS
+  x86_64-linux
+
+DEPENDENCIES
+  bundler (~> 2.2.24)
+  fluent-plugin-mageshlogs!
+  rake (~> 13.0.3)
+  test-unit (~> 3.3.7)
+
+BUNDLED WITH
+   2.2.24

data/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,43 @@
+# fluent-plugin-mageshlogs
+
+[Fluentd](https://fluentd.org/) output plugin to do something.
+
+TODO: write description for you plugin.
+
+## Installation
+
+### RubyGems
+
+```
+$ gem install fluent-plugin-mageshlogs
+```
+
+### Bundler
+
+Add following line to your Gemfile:
+
+```ruby
+gem "fluent-plugin-mageshlogs"
+```
+
+And then execute:
+
+```
+$ bundle
+```
+
+## Configuration
+
+You can generate configuration template:
+
+```
+$ fluent-plugin-config-format output mageshlogs
+```
+
+You can copy and paste generated documents here.
+
+## Copyright
+
+* Copyright(c) 2021- magesh-1870
+* License
+  * Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,13 @@
+require "bundler"
+Bundler::GemHelper.install_tasks
+
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs.push("lib", "test")
+  t.test_files = FileList["test/**/test_*.rb"]
+  t.verbose = true
+  t.warning = true
+end
+
+task default: [:test]

data/fluent-plugin-mageshlogs.gemspec

@@ -0,0 +1,37 @@
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name    = "fluent-plugin-mageshlogs"
+  spec.version = "0.1.7"
+  spec.authors = ["Magesh Rajan"]
+  spec.email   = ["mageshrajan0007@gmail.com"]
+
+  spec.summary       = %q{Magesh output plugin for Fluent event collector}
+  spec.homepage      = "https://www.site24x7.com/log-management.html"
+  spec.license       = ""
+
+  test_files, files  = `git ls-files -z`.split("\x0").partition do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.files         = files
+  spec.executables   = files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = test_files
+  spec.require_paths = ["lib"]
+
+  #spec.add_development_dependency "bundler", "~> 2.2.24"
+  #spec.add_development_dependency "rake", "~> 13.0.3"
+  #spec.add_development_dependency "test-unit", "~> 3.3.7"
+  #spec.add_development_dependency "yajl-ruby", "~> 1.2"
+
+  #spec.add_runtime_dependency "fluentd", [">= 1", "< 2"]
+  #spec.add_runtime_dependency "net-http-persistent", '~> 3.1'
+  spec.add_runtime_dependency "fluentd", [">= 1", "< 2"]
+  spec.add_runtime_dependency "net-http-persistent", '~> 3.1'
+
+  spec.add_development_dependency "bundler", "~> 2.1"
+  spec.add_development_dependency "test-unit", '~> 3.1'
+  spec.add_development_dependency "rake", "~> 12.0"
+  spec.add_development_dependency "yajl-ruby", "~> 1.2"
+  spec.add_development_dependency 'webmock', '~> 3.5', '>= 3.5.0'
+end

data/lib/fluent/plugin/out_mageshlogs.rb

@@ -0,0 +1,353 @@
+#
+# Copyright 2021- magesh-1870
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "fluent/plugin/output"
+
+module Fluent
+  module Plugin
+    class MageshLogsOutput < Fluent::Plugin::Output
+    	Fluent::Plugin.register_output('mageshlogs', self)
+  
+  S247_MAX_RECORD_COUNT = 500
+  S247_MAX_RECORD_SIZE = 1000000
+  S247_MAX_BATCH_SIZE = 5000000
+  S247_TRUNCATION_SUFFIX = "##TRUNCATED###"
+
+  helpers :compat_parameters
+
+  config_param :log_type_config, :string
+  config_param :max_retry, :integer, :default => 3
+  config_param :retry_interval, :integer, :default => 2
+  config_param :http_proxy, :string, :default => nil
+
+  config_section :buffer do
+    config_set_default :@type, "memory"
+  end
+
+  def initialize
+    super
+  end
+
+  def configure(conf)
+    compat_parameters_convert(conf, :buffer)
+    super
+  end
+
+  def multi_workers_ready?
+    true
+  end
+
+  def formatted_to_msgpack_binary?
+    true
+  end
+
+  def start
+    super
+    init_variables()
+    init_http_client(@logtype_config)
+  end
+
+  def shutdown
+    super
+  end
+
+  def terminate
+    super
+    @s247_http_client.shutdown if @s247_http_client
+  end
+
+def init_variables()
+    @logtype_config = Yajl::Parser.parse(Base64.decode64(@log_type_config))
+    @s247_custom_regex = if @logtype_config.has_key? 'regex' then Regexp.compile(@logtype_config['regex'].gsub('?P<','?<')) else nil end
+    @s247_ignored_fields = if @logtype_config.has_key? 'ignored_fields' then @logtype_config['ignored_fields'] else [] end
+    @s247_tz = {'hrs': 0, 'mins': 0} #UTC
+    @log_source = Socket.gethostname
+	@s247_datetime_format_string = @logtype_config['dateFormat']
+        @s247_datetime_format_string = @s247_datetime_format_string.sub('%f', '%N')
+	if !@s247_datetime_format_string.include? 'unix'
+	    is_year_present = if @s247_datetime_format_string.include?('%y') || @s247_datetime_format_string.include?('%Y') then true else false end
+	    if !is_year_present
+		@s247_datetime_format_string = @s247_datetime_format_string+ ' %Y'
+	    end   
+	    is_timezone_present = if @s247_datetime_format_string.include? '%z' then true else false end
+	    if !is_timezone_present && @logtype_config.has_key?('timezone')
+		tz_value = @logtype_config['timezone']
+		if tz_value.start_with?('+')
+		    @s247_tz['hrs'] = Integer('-' + tz_value[1..4])
+		    @s247_tz['mins'] = Integer('-' + tz_value[3..6])
+		elsif tz_value.start_with?('-')
+		    @s247_tz['hrs'] = Integer('+' + tz_value[1..4])
+		    @s247_tz['mins'] = Integer('+' + tz_value[3..6])
+		end
+	    end
+	end
+end
+
+def init_http_client(logtype_config)
+     require 'net/http/persistent'
+     upload_url = 'https://'+logtype_config['uploadDomain']+'/upload'
+     @uri = URI(upload_url)
+      proxy_uri = :ENV
+      if http_proxy
+        proxy_uri = URI.parse(http_proxy)
+      elsif ENV['HTTP_PROXY'] || ENV['http_proxy']
+        log.info "Using HTTP proxy defined in `HTTP_PROXY`/`http_proxy` env vars"
+      end
+      log.info "Starting HTTP connection to #{upload_url}"
+      @s247_http_client = Net::HTTP::Persistent.new name: "fluent-plugin-site24x7", proxy: proxy_uri
+      @s247_http_client.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      @s247_http_client.override_headers["Content-Type"] = "application/json"
+      @s247_http_client.override_headers["Content-Encoding"] = "gzip"
+      @s247_http_client.override_headers["X-DeviceKey"] = logtype_config['apiKey']
+      @s247_http_client.override_headers["X-LogType"] = logtype_config['logType']
+      @s247_http_client.override_headers["X-StreamMode"] = 1
+      @s247_http_client.override_headers["User-Agent"] = 'Fluentd'
+      if !@s247_http_client.proxy_uri.nil?
+        # Log the proxy settings as resolved by the HTTP client
+        log.info "Using HTTP proxy #{@s247_http_client.proxy_uri.scheme}://#{@s247_http_client.proxy_uri.host}:#{@s247_http_client.proxy_uri.port} username: #{@s247_http_client.proxy_uri.user ? "set" : "unset"}, password: #{@s247_http_client.proxy_uri.password ? "set" : "unset"}"
+      end
+end
+
+def get_timestamp(datetime_string)
+    begin
+        # If the date value is in unix format the no need to process the date string
+        if @s247_datetime_format_string.include? 'unix'
+            return (if @s247_datetime_format_string == 'unix' then datetime_string+'000' else datetime_string end)
+        end
+        datetime_string += if !is_year_present then ' '+String(Time.new.year) else '' end
+        if !is_timezone_present
+            @s247_datetime_format_string += '%z'
+            time_zone = String(@s247_tz['hrs'])+':'+String(@s247_tz['mins'])
+            datetime_string += if time_zone.start_with?('-') then time_zone else '+'+time_zone end
+        end
+        datetime_data = DateTime.strptime(datetime_string, @s247_datetime_format_string)
+        return Integer(datetime_data.strftime('%Q'))
+    rescue
+        return 0
+    end
+end
+
+def parse_lines(lines)
+ parsed_lines = []
+ log_size = 0
+ lines.each do |line|
+    if !line.empty?
+	    begin
+		if match = line.match(@s247_custom_regex)
+                    log_size += line.bytesize
+		    log_fields = match&.named_captures#matcher.groupdict(default='-')
+		    removed_log_size=0
+		    @s247_ignored_fields.each do |field_name|
+		        removed_log_size += if log_fields.has_key?field_name then log_fields.delete(field_name).bytesize else 0 end
+		    end
+		    formatted_line = {'_zl_timestamp' => get_timestamp(log_fields[@logtype_config['dateField']]), 's247agentuid' => @log_source}
+		    formatted_line.merge!(log_fields)
+                    parsed_lines.push(formatted_line)
+		    log_size -= removed_log_size
+		end
+	    rescue Exception => e
+		log.info "Exception in parse_line #{e.backtrace}"
+	    end
+    end
+  end
+  return parsed_lines, log_size
+end
+
+def is_filters_matched(formatted_line)
+    if @logtype_config.has_key?'filterConfig'
+        @logtype_config['filterConfig'].each do |config|
+            if formatted_line.has_key?config && (filter_config[config]['match'] ^ (filter_config[config]['values'].include?formatted_line[config]))
+                return false
+	    end
+        end
+    end
+    return true
+end
+
+def get_json_value(obj, key, datatype=nil)
+    if obj != nil && (obj.has_key?key)
+       if datatype and datatype == 'json-object'
+	  arr_json = []
+          child_obj = obj[key]
+          if child_obj.class == String
+             child_obj = Yajl::Parser.parse(child_obj.gsub('\\','\\\\'))
+          end             
+          child_obj.each do |key, value|
+             arr_json.push({'key' => key, 'value' => String(value)})
+          end
+          return arr_json
+       else
+         return (if obj.has_key?key then obj[key] else obj[key.downcase] end)
+       end
+    elsif key.include?'.'
+	parent_key = key[0..key.index('.')-1]
+	child_key = key[key.index('.')+1..-1]
+        child_obj = obj[if obj.has_key?parent_key then parent_key else parent_key.capitalize() end]
+	if child_obj.class == String
+            child_obj = Yajl::Parser.parse(child_obj.replace('\\','\\\\'))
+        end
+        return get_json_value(child_obj, child_key)
+    end
+end
+
+def json_log_parser(lines_read)
+    log_size = 0
+    parsed_lines = []
+    lines_read.each do |line|
+        if !line.empty?
+          current_log_size = 0
+	  formatted_line = {}
+	  event_obj = Yajl::Parser.parse(line)
+	  @logtype_config['jsonPath'].each do |path_obj|
+	    value = get_json_value(event_obj, path_obj[if path_obj.has_key?'key' then 'key' else 'name' end], path_obj['type'])
+            if value
+	      formatted_line[path_obj['name']] = value 
+	      current_log_size+= String(value).size - (if value.class == Array then value.size*20 else 0 end)
+            end
+          end
+	  if is_filters_matched(formatted_line)
+	    formatted_line['_zl_timestamp'] = get_timestamp(formatted_line[@logtype_config['dateField']])
+	    formatted_line['s247agentuid'] = @log_source
+	    parsed_lines.push(formatted_line)
+            log_size+=current_log_size
+          end
+	end
+    end
+    return parsed_lines, log_size
+end
+
+  def format(tag, time, record)
+    [record['message']].to_msgpack
+  end
+
+
+  def write(chunk)
+    begin
+	events = Array.new
+	chunk.msgpack_each do |record|
+	  next if record.empty?
+	  events.push record[0]
+	end
+	process_http_events(events)
+    rescue Exception => e
+      log.info "Exception #{e.backtrace}"
+    end
+  end
+
+  def process_http_events(events)
+    batches = batch_http_events(events)
+    batches.each do |batched_event|
+      formatted_events, log_size = format_http_event_batch(batched_event)
+      formatted_events = gzip_compress(formatted_events)
+      send_logs_to_s247(formatted_events, log_size)
+    end
+  end
+
+  def batch_http_events(encoded_events)
+    batches = []
+    current_batch = []
+    current_batch_size = 0
+    encoded_events.each_with_index do |encoded_event, i|
+      current_event_size = encoded_event.bytesize
+      if current_event_size > S247_MAX_RECORD_SIZE
+        encoded_event = encoded_event[0..(S247_MAX_RECORD_SIZE-DD_TRUNCATION_SUFFIX.length)]+DD_TRUNCATION_SUFFIX
+        current_event_size = encoded_event.bytesize
+      end
+
+      if (i > 0 and i % S247_MAX_RECORD_COUNT == 0) or (current_batch_size + current_event_size > S247_MAX_BATCH_SIZE)
+        batches << current_batch
+        current_batch = []
+        current_batch_size = 0
+      end
+
+      current_batch_size += current_event_size
+      current_batch << encoded_event
+    end
+    batches << current_batch
+    batches
+  end
+
+  def format_http_event_batch(events)
+    parsed_lines = []
+    log_size = 0
+    if @logtype_config.has_key?'jsonPath'
+      parsed_lines, log_size = json_log_parser(events)
+    else
+       parsed_lines, log_size = parse_lines(events)
+    end
+    return Yajl.dump(parsed_lines), log_size
+  end
+
+  def gzip_compress(payload)
+    gz = StringIO.new
+    gz.set_encoding("BINARY")
+    z = Zlib::GzipWriter.new(gz, 9)
+    begin
+      z.write(payload)
+    ensure
+      z.close
+    end
+    gz.string
+  end
+
+  def send_logs_to_s247(gzipped_parsed_lines, log_size)
+      request = Net::HTTP::Post.new @uri.request_uri
+      request.body = gzipped_parsed_lines
+      @s247_http_client.override_headers["Log-Size"] = log_size
+      sleep_interval = @retry_interval
+     begin
+        @max_retry.times do |counter|
+          need_retry = false
+          begin
+            response = @s247_http_client.request @uri, request
+            resp_headers = response.each_header.to_h
+
+            if response.code == '200'
+              if resp_headers.has_key?'LOG_LICENSE_EXCEEDS' && resp_headers['LOG_LICENSE_EXCEEDS'] == 'True'
+                log.error "Log license limit exceeds so not able to send logs"
+              elsif resp_headers.has_key?'BLOCKED_LOGTYPE' && resp_headers['BLOCKED_LOGTYPE'] == 'True'
+                log.error "Max upload limit reached for log type"
+              elsif resp_headers.has_key?'INVALID_LOGTYPE' && resp_headers['INVALID_LOGTYPE'] == 'True'
+                log.error "Log type not present in this account so stopping log collection"
+              else
+                log.info "Successfully sent logs with size #{gzipped_parsed_lines.size} / #{log_size} to site24x7. Upload Id : #{resp_headers['x-uploadid']}"
+              end
+            else
+              log.info "Response Code #{resp_headers} from Site24x7, so retrying (#{counter + 1}/#{@max_retry})"
+              need_retry = true
+            end
+          rescue StandardError => e
+            log.info "Error connecting to Site24x7. exception: #{e} (#{counter + 1}/#{@max_retry})"
+          end
+
+          if need_retry
+            if counter == @max_retry - 1
+              log.error "Could not send your logs after #{max_retry} tries"
+              break
+            end
+            sleep(sleep_interval)
+            sleep_interval *= 2
+          else
+            return
+          end
+        end
+      rescue Exception => e
+        log.error "Exception occurred in sendig logs : #{e}"
+      end
+
+      end	
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,8 @@
+$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
+require "test-unit"
+require "fluent/test"
+require "fluent/test/driver/output"
+require "fluent/test/helpers"
+
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
+Test::Unit::TestCase.extend(Fluent::Test::Helpers)

data/test/plugin/test_out_mageshlogs.rb

@@ -0,0 +1,18 @@
+require "helper"
+require "fluent/plugin/out_mageshlogs.rb"
+
+class MageshlogsOutputTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+  end
+
+  test "failure" do
+    flunk
+  end
+
+  private
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::MageshlogsOutput).configure(conf)
+  end
+end

metadata

@@ -1,112 +1,145 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mageshlogs
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.7
 platform: ruby
 authors:
 - Magesh Rajan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-07-29 00:00:00.000000000 Z
+date: 2021-07-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: net-http-persistent
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.2.24
-  type: :development
+        version: '3.1'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.2.24
+        version: '3.1'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 13.0.3
+        version: '2.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 13.0.3
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.3.7
+        version: '3.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.3.7
+        version: '3.1'
 - !ruby/object:Gem::Dependency
-  name: yajl-ruby
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '12.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '12.0'
 - !ruby/object:Gem::Dependency
-  name: fluentd
+  name: yajl-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1'
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
-  type: :runtime
+        version: '1.2'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1'
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '1.2'
 - !ruby/object:Gem::Dependency
-  name: net-http-persistent
+  name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.1'
-  type: :runtime
+        version: '3.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.5.0
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: '3.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.5.0
 description: 
 email:
 - mageshrajan0007@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
-files: []
+files:
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-mageshlogs.gemspec
+- lib/fluent/plugin/2
+- lib/fluent/plugin/3
+- lib/fluent/plugin/e
+- lib/fluent/plugin/nil
+- lib/fluent/plugin/out_mageshlogs.rb
+- test/helper.rb
+- test/plugin/test_out_mageshlogs.rb
 homepage: https://www.site24x7.com/log-management.html
 licenses:
 - ''
@@ -130,4 +163,6 @@ rubygems_version: 3.2.24
 signing_key: 
 specification_version: 4
 summary: Magesh output plugin for Fluent event collector
-test_files: []
+test_files:
+- test/helper.rb
+- test/plugin/test_out_mageshlogs.rb