fluent-plugin-logzio 0.0.14 → 0.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +47 -9
- data/fluent-plugin-logzio.gemspec +10 -10
- data/lib/fluent/plugin/out_logzio_buffered.rb +63 -17
- data/spec/lib/fluent/plugin/out_logzio_buffered_spec.rb +8 -8
- data/spec/lib/fluent/plugin/out_logzio_multi_bulk_spec.rb +36 -0
- data/spec/lib/fluent/plugin/out_logzio_record_too_large_spec.rb +33 -0
- data/spec/spec_helper.rb +8 -7
- metadata +29 -19
- data/spec/lib/fluent/plugin/out_logzio_spec.rb +0 -30
    
        checksums.yaml
    CHANGED
    
    | @@ -1,7 +1,7 @@ | |
| 1 1 | 
             
            ---
         | 
| 2 2 | 
             
            SHA1:
         | 
| 3 | 
            -
              metadata.gz:  | 
| 4 | 
            -
              data.tar.gz:  | 
| 3 | 
            +
              metadata.gz: c7d5868baf67c48aa8bffb672b4031d4ef11fd27
         | 
| 4 | 
            +
              data.tar.gz: e5fb7b2b84dfb1b37b86f6a168eefbbb6982efbb
         | 
| 5 5 | 
             
            SHA512:
         | 
| 6 | 
            -
              metadata.gz:  | 
| 7 | 
            -
              data.tar.gz:  | 
| 6 | 
            +
              metadata.gz: 196b6bd32d1e7a8e3abf0df03e612e49b0e5745f55635995fb4e8204653199aec79ccf95871699990c3989322b7e57bca0a0269569e00587bff14afead9b2935
         | 
| 7 | 
            +
              data.tar.gz: 8f35a877790aeb6d4ccded569cdab71775d36a52651e29330629da7bd4e5ca631848e57b26eba2256d36628bd83f395fee9cdd9e70ff7f8ca043d0b1d8701d0c
         | 
    
        data/README.md
    CHANGED
    
    | @@ -4,12 +4,47 @@ Logz.io plugin for [Fluentd](http://www.fluentd.org) | |
| 4 4 | 
             
            =============
         | 
| 5 5 | 
             
            With fluent-plugin-logzio you will be able to use [Logz.io](http://logz.io) as output the logs you collect with Fluentd.
         | 
| 6 6 |  | 
| 7 | 
            +
            ## Requirements
         | 
| 8 | 
            +
             | 
| 9 | 
            +
            | fluent-plugin-logzio      | Fluentd     | Ruby   |
         | 
| 10 | 
            +
            |---------------------------|-------------|--------|
         | 
| 11 | 
            +
            | >= 0.0.15                 | >= v0.14.0  | >= 2.1 |
         | 
| 12 | 
            +
            |  < 0.0.15                 | >= v0.12.0  | >= 1.9 |
         | 
| 13 | 
            +
             | 
| 7 14 | 
             
            ## Getting Started
         | 
| 8 15 | 
             
            * Install [Fluentd](http://www.fluentd.org/download)
         | 
| 9 16 | 
             
            * gem install fluent-plugin-logzio
         | 
| 10 17 | 
             
            * Make sure you have an account with Logz.io.
         | 
| 11 18 | 
             
            * Configure Fluentd as below:
         | 
| 12 19 |  | 
| 20 | 
            +
            ### FluentD 1.0-style Example Configuration
         | 
| 21 | 
            +
             | 
| 22 | 
            +
            This is an **example** only. Your needs in production may vary!
         | 
| 23 | 
            +
             | 
| 24 | 
            +
            ```
         | 
| 25 | 
            +
                <match **>
         | 
| 26 | 
            +
                  @type logzio_buffered
         | 
| 27 | 
            +
             | 
| 28 | 
            +
                  endpoint_url https://listener.logz.io:8071?token=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&type=my_type
         | 
| 29 | 
            +
             | 
| 30 | 
            +
                  output_include_time true
         | 
| 31 | 
            +
                  output_include_tags true
         | 
| 32 | 
            +
                  http_idle_timeout 10
         | 
| 33 | 
            +
             | 
| 34 | 
            +
                  <buffer>
         | 
| 35 | 
            +
                      @type memory
         | 
| 36 | 
            +
                      flush_thread_count 4
         | 
| 37 | 
            +
                      flush_interval 3s
         | 
| 38 | 
            +
                      chunk_limit_size 16m      # Logz.io bulk limit is decoupled from chunk_limit_size. Set whatever you want.
         | 
| 39 | 
            +
                      queue_limit_length 4096
         | 
| 40 | 
            +
                  </buffer>
         | 
| 41 | 
            +
                </match>
         | 
| 42 | 
            +
            ```
         | 
| 43 | 
            +
             | 
| 44 | 
            +
            ### FluentD 0.12-style Example Configuration
         | 
| 45 | 
            +
             | 
| 46 | 
            +
            This is an **example** only. Your needs in production may vary!
         | 
| 47 | 
            +
             | 
| 13 48 | 
             
            ```
         | 
| 14 49 | 
             
                <match your_match>
         | 
| 15 50 | 
             
                  @type logzio_buffered
         | 
| @@ -20,19 +55,22 @@ With fluent-plugin-logzio you will be able to use [Logz.io](http://logz.io) as o | |
| 20 55 | 
             
                  buffer_type    file
         | 
| 21 56 | 
             
                  buffer_path    /path/to/buffer/file
         | 
| 22 57 | 
             
                  flush_interval 10s
         | 
| 23 | 
            -
                  buffer_chunk_limit  | 
| 58 | 
            +
                  buffer_chunk_limit 64m   # Logz.io bulk limit is decoupled from buffer_chunk_limit. Set whatever you want.
         | 
| 24 59 | 
             
                </match>
         | 
| 25 60 | 
             
            ```
         | 
| 26 61 |  | 
| 27 62 | 
             
            ## Parameters
         | 
| 28 | 
            -
            * **endpoint_url** the url to Logz.io input where `xxx-xxxx...` is your Logz.io access token, and `my_type` is the type of your logs in  | 
| 29 | 
            -
            * **output_include_time** should the appender add a timestamp to your logs on their process time. (recommended)
         | 
| 30 | 
            -
            * **output_include_tags** should the appender add the fluentd tag to the document, called "fluentd_tag" (which can be renamed, see next point)
         | 
| 31 | 
            -
            * **output_tags_fieldname** set the tag's fieldname, defaults to "fluentd_tag"
         | 
| 32 | 
            -
            * **http_idle_timeout** timeout in seconds that the http persistent connection will stay open without traffic
         | 
| 33 | 
            -
             | 
| 63 | 
            +
            * **endpoint_url** the url to Logz.io input where `xxx-xxxx...` is your Logz.io access token, and `my_type` is the type of your logs in Logz.io.
         | 
| 64 | 
            +
            * **output_include_time** should the appender add a timestamp to your logs on their process time. (recommended).
         | 
| 65 | 
            +
            * **output_include_tags** should the appender add the fluentd tag to the document, called "fluentd_tag" (which can be renamed, see next point).
         | 
| 66 | 
            +
            * **output_tags_fieldname** set the tag's fieldname, defaults to "fluentd_tag".
         | 
| 67 | 
            +
            * **http_idle_timeout** timeout in seconds that the http persistent connection will stay open without traffic.
         | 
| 68 | 
            +
            * **retry_count** How many times to resend failed bulks. Defaults to 4 times.
         | 
| 69 | 
            +
            * **retry_sleep** How long to sleep initially between retries, exponential step-off. Initial default is 2s.
         | 
| 70 | 
            +
            * **bulk_limit** Limit to the size of the Logz.io upload bulk. Defaults to 1000000 bytes leaving about 24kB for overhead.
         | 
| 34 71 |  | 
| 35 72 | 
             
            ## Release Notes
         | 
| 36 | 
            -
            - 0.0. | 
| 73 | 
            +
            - 0.0.15: Support FluentD 1.0+. Split the chunk into bulk uploads, decoupling `chunk_limit_size`/`buffer_chunk_limit` from Logz.io bulk limit. Tunable `bulk_limit` and initial `retry_sleep`.
         | 
| 74 | 
            +
            - 0.0.14: Refactor send function to handle more cases, and retry in case of logzio connection failure.
         | 
| 37 75 | 
             
            - 0.0.13: BREAKING - Removed non-buffered version. It's really not efficient, and should just not be used. If you are using this version, you should change to the buffered one.
         | 
| 38 | 
            -
            - 0.0.12: Catch exception when parsing YAML to ignore (instead of crash) not valid logs
         | 
| 76 | 
            +
            - 0.0.12: Catch exception when parsing YAML to ignore (instead of crash) not valid logs.
         | 
| @@ -4,9 +4,9 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) | |
| 4 4 |  | 
| 5 5 | 
             
            Gem::Specification.new do |s|
         | 
| 6 6 | 
             
              s.name        = 'fluent-plugin-logzio'
         | 
| 7 | 
            -
              s.version     = '0.0. | 
| 8 | 
            -
              s.authors     = ['Yury Kotov', 'Roi Rav-Hon']
         | 
| 9 | 
            -
              s.email       = ['bairkan@gmail.com', 'roi@logz.io']
         | 
| 7 | 
            +
              s.version     = '0.0.15'
         | 
| 8 | 
            +
              s.authors     = ['Yury Kotov', 'Roi Rav-Hon', 'Arcadiy Ivanov']
         | 
| 9 | 
            +
              s.email       = ['bairkan@gmail.com', 'roi@logz.io', 'arcadiy@ivanov.biz']
         | 
| 10 10 | 
             
              s.homepage    = 'https://github.com/logzio/fluent-plugin-logzio'
         | 
| 11 11 | 
             
              s.summary     = %q{Fluentd plugin for output to Logz.io}
         | 
| 12 12 | 
             
              s.description = %q{Fluentd pluging (fluent.org) for output to Logz.io (logz.io)}
         | 
| @@ -17,12 +17,12 @@ Gem::Specification.new do |s| | |
| 17 17 | 
             
              s.files         = `git ls-files`.split("\n")
         | 
| 18 18 | 
             
              s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
         | 
| 19 19 | 
             
              s.require_paths = ['lib']
         | 
| 20 | 
            -
              s.required_ruby_version = Gem::Requirement.new('>= 1. | 
| 20 | 
            +
              s.required_ruby_version = Gem::Requirement.new('>= 2.1.0')
         | 
| 21 21 |  | 
| 22 | 
            -
              s.add_dependency 'net-http-persistent', '~> 2. | 
| 23 | 
            -
              s. | 
| 24 | 
            -
              s.add_development_dependency 'rake', '~>  | 
| 25 | 
            -
              s.add_development_dependency 'bundler', '~> 1. | 
| 26 | 
            -
              s.add_development_dependency 'rspec', '~> 3. | 
| 27 | 
            -
              s.add_development_dependency 'test-unit', '~> 3. | 
| 22 | 
            +
              s.add_dependency 'net-http-persistent', '~> 2.9'
         | 
| 23 | 
            +
              s.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 2']
         | 
| 24 | 
            +
              s.add_development_dependency 'rake', '~> 12.3'
         | 
| 25 | 
            +
              s.add_development_dependency 'bundler', '~> 1.16'
         | 
| 26 | 
            +
              s.add_development_dependency 'rspec', '~> 3.7'
         | 
| 27 | 
            +
              s.add_development_dependency 'test-unit', '~> 3.2'
         | 
| 28 28 | 
             
            end
         | 
| @@ -1,10 +1,18 @@ | |
| 1 | 
            +
            require 'time'
         | 
| 2 | 
            +
            require 'fluent/plugin/output'
         | 
| 3 | 
            +
             | 
| 1 4 | 
             
            module Fluent
         | 
| 2 | 
            -
              class LogzioOutputBuffered <  | 
| 5 | 
            +
              class LogzioOutputBuffered < Output
         | 
| 3 6 | 
             
                Fluent::Plugin.register_output('logzio_buffered', self)
         | 
| 7 | 
            +
             | 
| 8 | 
            +
                helpers :compat_parameters
         | 
| 9 | 
            +
             | 
| 4 10 | 
             
                config_param :endpoint_url, :string, default: nil
         | 
| 5 11 | 
             
                config_param :output_include_time, :bool, default: true
         | 
| 6 12 | 
             
                config_param :output_include_tags, :bool, default: true
         | 
| 7 13 | 
             
                config_param :retry_count, :integer, default: 4 # How many times to resend failed bulks. Undocumented because not suppose to be changed
         | 
| 14 | 
            +
                config_param :retry_sleep, :integer, default: 2 # How long to sleep initially between retries, exponential step-off
         | 
| 15 | 
            +
                config_param :bulk_limit, :integer, default: 1000000 # Make sure submission to LogzIO does not exceed 1MB limit and leave some overhead
         | 
| 8 16 | 
             
                config_param :http_idle_timeout, :integer, default: 5
         | 
| 9 17 | 
             
                config_param :output_tags_fieldname, :string, default: 'fluentd_tags'
         | 
| 10 18 |  | 
| @@ -14,7 +22,9 @@ module Fluent | |
| 14 22 |  | 
| 15 23 | 
             
                def configure(conf)
         | 
| 16 24 | 
             
                  super
         | 
| 17 | 
            -
                   | 
| 25 | 
            +
                  compat_parameters_convert(conf, :buffer)
         | 
| 26 | 
            +
             | 
| 27 | 
            +
                  $log.debug "Logz.io URL #{@endpoint_url}"
         | 
| 18 28 | 
             
                end
         | 
| 19 29 |  | 
| 20 30 | 
             
                def start
         | 
| @@ -25,39 +35,75 @@ module Fluent | |
| 25 35 | 
             
                  @http.headers['Content-Type'] = 'text/plain'
         | 
| 26 36 | 
             
                  @http.idle_timeout = @http_idle_timeout
         | 
| 27 37 | 
             
                  @http.socket_options << [Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, 1]
         | 
| 28 | 
            -
             | 
| 38 | 
            +
             | 
| 39 | 
            +
                  log.debug "Started Logz.io shipper.."
         | 
| 29 40 | 
             
                end
         | 
| 30 41 |  | 
| 31 42 | 
             
                def shutdown
         | 
| 32 43 | 
             
                  super
         | 
| 33 44 | 
             
                end
         | 
| 34 45 |  | 
| 46 | 
            +
                def formatted_to_msgpack_binary?
         | 
| 47 | 
            +
                  true
         | 
| 48 | 
            +
                end
         | 
| 49 | 
            +
             | 
| 35 50 | 
             
                def format(tag, time, record)
         | 
| 36 | 
            -
                   | 
| 51 | 
            +
                  if time.is_a?(Fluent::EventTime)
         | 
| 52 | 
            +
                    sec_frac = time.sec + time.nsec / 10.0 ** 9
         | 
| 53 | 
            +
                  else
         | 
| 54 | 
            +
                    sec_frac = time * 1.0
         | 
| 55 | 
            +
                  end
         | 
| 56 | 
            +
                  [tag, sec_frac, record].to_msgpack
         | 
| 37 57 | 
             
                end
         | 
| 38 58 |  | 
| 39 59 | 
             
                def write(chunk)
         | 
| 60 | 
            +
                  encode_chunk(chunk) { |bulk_records, bulk_size|
         | 
| 61 | 
            +
                    send_bulk(bulk_records, bulk_size)
         | 
| 62 | 
            +
                  }
         | 
| 63 | 
            +
                end
         | 
| 64 | 
            +
             | 
| 65 | 
            +
                def encode_chunk(chunk)
         | 
| 40 66 | 
             
                  records = []
         | 
| 67 | 
            +
                  bulk_size = 0
         | 
| 68 | 
            +
                  chunk.each { |tag, time, record|
         | 
| 69 | 
            +
                    record['@timestamp'] ||= Time.at(time).iso8601(3) if @output_include_time
         | 
| 70 | 
            +
                    record[@output_tags_fieldname] ||= tag.to_s if @output_include_tags
         | 
| 41 71 |  | 
| 42 | 
            -
                  chunk.msgpack_each {|tag,time,record|
         | 
| 43 72 | 
             
                    begin
         | 
| 44 | 
            -
                       | 
| 45 | 
            -
                       | 
| 46 | 
            -
                      records.push(Yajl.dump(record))
         | 
| 73 | 
            +
                      json_record = Yajl.dump(record)
         | 
| 74 | 
            +
                      record_size = json_record.size + (1 if !records.empty?).to_i # Accounting for trailing "\n"
         | 
| 47 75 | 
             
                    rescue
         | 
| 48 | 
            -
                      log.error | 
| 76 | 
            +
                      log.error "Adding record #{record} to buffer failed. Exception: #{$!}"
         | 
| 77 | 
            +
                      next
         | 
| 78 | 
            +
                    end
         | 
| 79 | 
            +
             | 
| 80 | 
            +
                    if record_size > @bulk_limit
         | 
| 81 | 
            +
                      log.warn "Record with size #{record_size} exceeds #{@bulk_limit} and can't be sent to Logz.io. Record is: #{json_record}"
         | 
| 82 | 
            +
                      next
         | 
| 83 | 
            +
                    end
         | 
| 84 | 
            +
                    if bulk_size + record_size > @bulk_limit
         | 
| 85 | 
            +
                      yield(records, bulk_size)
         | 
| 86 | 
            +
                      records = []
         | 
| 87 | 
            +
                      bulk_size = 0
         | 
| 49 88 | 
             
                    end
         | 
| 89 | 
            +
                    records.push(json_record)
         | 
| 90 | 
            +
                    bulk_size += record_size
         | 
| 50 91 | 
             
                  }
         | 
| 92 | 
            +
                  if records
         | 
| 93 | 
            +
                    yield(records, bulk_size)
         | 
| 94 | 
            +
                  end
         | 
| 95 | 
            +
                end
         | 
| 51 96 |  | 
| 52 | 
            -
             | 
| 97 | 
            +
                def send_bulk(bulk_records, bulk_size)
         | 
| 98 | 
            +
                  log.debug "Sending a bulk of #{bulk_records.size} records, size #{bulk_size}B to Logz.io"
         | 
| 53 99 |  | 
| 54 100 | 
             
                  # Setting our request
         | 
| 55 101 | 
             
                  post = Net::HTTP::Post.new @uri.request_uri
         | 
| 56 102 |  | 
| 57 103 | 
             
                  # Logz.io bulk http endpoint expecting log line with \n delimiter
         | 
| 58 | 
            -
                  post.body =  | 
| 104 | 
            +
                  post.body = bulk_records.join("\n")
         | 
| 59 105 |  | 
| 60 | 
            -
                  sleep_interval =  | 
| 106 | 
            +
                  sleep_interval = @retry_sleep
         | 
| 61 107 |  | 
| 62 108 | 
             
                  begin
         | 
| 63 109 | 
             
                    @retry_count.times do |counter|
         | 
| @@ -72,20 +118,20 @@ module Fluent | |
| 72 118 | 
             
                            log.info "Got 400 code from Logz.io. This means that some of your logs are too big, or badly formatted. Response: #{response.body}"
         | 
| 73 119 | 
             
                            should_retry = false
         | 
| 74 120 | 
             
                          else
         | 
| 75 | 
            -
                            log.debug "Got HTTP #{response.code} from  | 
| 121 | 
            +
                            log.debug "Got HTTP #{response.code} from Logz.io, not giving up just yet (Try #{counter + 1}/#{@retry_count})"
         | 
| 76 122 | 
             
                          end
         | 
| 77 123 | 
             
                        else
         | 
| 78 | 
            -
                          log.debug " | 
| 124 | 
            +
                          log.debug "Successfully sent bulk of #{bulk_records.size} records, size #{bulk_size}B to Logz.io"
         | 
| 79 125 | 
             
                          should_retry = false
         | 
| 80 126 | 
             
                        end
         | 
| 81 127 | 
             
                      rescue StandardError => e
         | 
| 82 | 
            -
                        log.debug "Error connecting to  | 
| 128 | 
            +
                        log.debug "Error connecting to Logz.io. Got exception: #{e} (Try #{counter + 1}/#{@retry_count})"
         | 
| 83 129 | 
             
                      end
         | 
| 84 130 |  | 
| 85 131 | 
             
                      if should_retry
         | 
| 86 132 | 
             
                        if counter == @retry_count - 1
         | 
| 87 | 
            -
             | 
| 88 | 
            -
             | 
| 133 | 
            +
                          log.error "Could not send your bulk after #{retry_count} tries Sorry! Your bulk is: #{post.body}"
         | 
| 134 | 
            +
                          break
         | 
| 89 135 | 
             
                        end
         | 
| 90 136 | 
             
                        sleep(sleep_interval)
         | 
| 91 137 | 
             
                        sleep_interval *= 2
         | 
| @@ -1,7 +1,7 @@ | |
| 1 1 | 
             
            require 'spec_helper'
         | 
| 2 2 |  | 
| 3 3 | 
             
            describe 'Fluent::LogzioOutputBuffered' do
         | 
| 4 | 
            -
              let(:driver) { Fluent::Test:: | 
| 4 | 
            +
              let(:driver) { Fluent::Test::Driver::Output.new(Fluent::LogzioOutputBuffered).configure(config) }
         | 
| 5 5 | 
             
              let(:config) do
         | 
| 6 6 | 
             
                %[
         | 
| 7 7 | 
             
                  endpoint_url         https://logz.io?token=123
         | 
| @@ -12,7 +12,7 @@ describe 'Fluent::LogzioOutputBuffered' do | |
| 12 12 | 
             
              include_context 'output context'
         | 
| 13 13 | 
             
              include_examples 'output examples'
         | 
| 14 14 |  | 
| 15 | 
            -
              describe ' | 
| 15 | 
            +
              describe 'feed' do
         | 
| 16 16 | 
             
                before(:each) do
         | 
| 17 17 | 
             
                  expect(request).to receive(:body=).with('{"field1":50,"otherfield":99,"fluentd_tags":"test"}' + "\n" + '{"field1":150,"otherfield":199,"fluentd_tags":"test"}')
         | 
| 18 18 | 
             
                  expect(Net::HTTP::Post).to receive(:new).with('/?token=123').once.and_return(request)
         | 
| @@ -20,13 +20,13 @@ describe 'Fluent::LogzioOutputBuffered' do | |
| 20 20 | 
             
                end
         | 
| 21 21 |  | 
| 22 22 | 
             
                it 'adds messages to the buffer' do
         | 
| 23 | 
            -
                  driver. | 
| 24 | 
            -
             | 
| 23 | 
            +
                  driver.run(default_tag: 'test') do
         | 
| 24 | 
            +
                    driver.feed(time, record1)
         | 
| 25 | 
            +
                    driver.feed(time, record2)
         | 
| 26 | 
            +
                  end
         | 
| 25 27 |  | 
| 26 | 
            -
                  driver. | 
| 27 | 
            -
             | 
| 28 | 
            -
             | 
| 29 | 
            -
                  driver.run
         | 
| 28 | 
            +
                  expect(driver.formatted).to eq([['test', 0.0, {'field1' => 50, 'otherfield' => 99}].to_msgpack,
         | 
| 29 | 
            +
                                                  ['test', 0.0, {'field1' => 150, 'otherfield' => 199}].to_msgpack])
         | 
| 30 30 | 
             
                end
         | 
| 31 31 | 
             
              end
         | 
| 32 32 | 
             
            end
         | 
| @@ -0,0 +1,36 @@ | |
| 1 | 
            +
            require 'spec_helper'
         | 
| 2 | 
            +
             | 
| 3 | 
            +
            describe 'Fluent::LogzioOutputBuffered' do
         | 
| 4 | 
            +
              let(:driver) { Fluent::Test::Driver::Output.new(Fluent::LogzioOutputBuffered).configure(config) }
         | 
| 5 | 
            +
              let(:config) do
         | 
| 6 | 
            +
                %[
         | 
| 7 | 
            +
                  endpoint_url         https://logz.io?token=123
         | 
| 8 | 
            +
                  output_include_time  false
         | 
| 9 | 
            +
                  bulk_limit           70
         | 
| 10 | 
            +
                ]
         | 
| 11 | 
            +
              end
         | 
| 12 | 
            +
             | 
| 13 | 
            +
              include_context 'output context'
         | 
| 14 | 
            +
              include_examples 'output examples'
         | 
| 15 | 
            +
             | 
| 16 | 
            +
              describe 'feed' do
         | 
| 17 | 
            +
                before(:each) do
         | 
| 18 | 
            +
                  expect(request).to receive(:body=).with('{"field1":50,"otherfield":99,"fluentd_tags":"test"}')
         | 
| 19 | 
            +
                  expect(Net::HTTP::Post).to receive(:new).with('/?token=123').once.and_return(request)
         | 
| 20 | 
            +
                  expect_any_instance_of(Net::HTTP::Persistent).to receive(:request).and_return(response)
         | 
| 21 | 
            +
                  expect(request).to receive(:body=).with('{"field1":150,"otherfield":199,"fluentd_tags":"test"}')
         | 
| 22 | 
            +
                  expect(Net::HTTP::Post).to receive(:new).with('/?token=123').once.and_return(request)
         | 
| 23 | 
            +
                  expect_any_instance_of(Net::HTTP::Persistent).to receive(:request).and_return(response)
         | 
| 24 | 
            +
                end
         | 
| 25 | 
            +
             | 
| 26 | 
            +
                it 'adds messages to the buffer' do
         | 
| 27 | 
            +
                  driver.run(default_tag: 'test') do
         | 
| 28 | 
            +
                    driver.feed(time, record1)
         | 
| 29 | 
            +
                    driver.feed(time, record2)
         | 
| 30 | 
            +
                  end
         | 
| 31 | 
            +
             | 
| 32 | 
            +
                  expect(driver.formatted).to eq([['test', 0.0, {'field1' => 50, 'otherfield' => 99}].to_msgpack,
         | 
| 33 | 
            +
                                                  ['test', 0.0, {'field1' => 150, 'otherfield' => 199}].to_msgpack])
         | 
| 34 | 
            +
                end
         | 
| 35 | 
            +
              end
         | 
| 36 | 
            +
            end
         | 
| @@ -0,0 +1,33 @@ | |
| 1 | 
            +
            require 'spec_helper'
         | 
| 2 | 
            +
             | 
| 3 | 
            +
            describe 'Fluent::LogzioOutputBuffered' do
         | 
| 4 | 
            +
              let(:driver) { Fluent::Test::Driver::Output.new(Fluent::LogzioOutputBuffered).configure(config) }
         | 
| 5 | 
            +
              let(:config) do
         | 
| 6 | 
            +
                %[
         | 
| 7 | 
            +
                  endpoint_url         https://logz.io?token=123
         | 
| 8 | 
            +
                  output_include_time  false
         | 
| 9 | 
            +
                  bulk_limit           52
         | 
| 10 | 
            +
                ]
         | 
| 11 | 
            +
              end
         | 
| 12 | 
            +
             | 
| 13 | 
            +
              include_context 'output context'
         | 
| 14 | 
            +
              include_examples 'output examples'
         | 
| 15 | 
            +
             | 
| 16 | 
            +
              describe 'feed' do
         | 
| 17 | 
            +
                before(:each) do
         | 
| 18 | 
            +
                  expect(request).to receive(:body=).with('{"field1":50,"otherfield":99,"fluentd_tags":"test"}').once
         | 
| 19 | 
            +
                  expect(Net::HTTP::Post).to receive(:new).with('/?token=123').once.and_return(request)
         | 
| 20 | 
            +
                  expect_any_instance_of(Net::HTTP::Persistent).to receive(:request).once.and_return(response)
         | 
| 21 | 
            +
                end
         | 
| 22 | 
            +
             | 
| 23 | 
            +
                it 'adds messages to the buffer' do
         | 
| 24 | 
            +
                  driver.run(default_tag: 'test') do
         | 
| 25 | 
            +
                    driver.feed(time, record1)
         | 
| 26 | 
            +
                    driver.feed(time, record2)
         | 
| 27 | 
            +
                  end
         | 
| 28 | 
            +
             | 
| 29 | 
            +
                  expect(driver.formatted).to eq([['test', 0.0, {'field1' => 50, 'otherfield' => 99}].to_msgpack,
         | 
| 30 | 
            +
                                                  ['test', 0.0, {'field1' => 150, 'otherfield' => 199}].to_msgpack])
         | 
| 31 | 
            +
                end
         | 
| 32 | 
            +
              end
         | 
| 33 | 
            +
            end
         | 
    
        data/spec/spec_helper.rb
    CHANGED
    
    | @@ -40,8 +40,8 @@ RSpec.configure do |config| | |
| 40 40 | 
             
                mocks.verify_partial_doubles = true
         | 
| 41 41 | 
             
              end
         | 
| 42 42 |  | 
| 43 | 
            -
            # The settings below are suggested to provide a good initial experience
         | 
| 44 | 
            -
            # with RSpec, but feel free to customize to your heart's content.
         | 
| 43 | 
            +
              # The settings below are suggested to provide a good initial experience
         | 
| 44 | 
            +
              # with RSpec, but feel free to customize to your heart's content.
         | 
| 45 45 | 
             
            =begin
         | 
| 46 46 | 
             
              # These two settings work together to allow you to limit a spec run
         | 
| 47 47 | 
             
              # to individual examples or groups you care about by tagging them with
         | 
| @@ -94,9 +94,11 @@ RSpec.configure do |config| | |
| 94 94 | 
             
              Kernel.srand config.seed
         | 
| 95 95 | 
             
            =end
         | 
| 96 96 | 
             
              require 'net/http/persistent'
         | 
| 97 | 
            +
              require 'serverengine'
         | 
| 98 | 
            +
             | 
| 97 99 | 
             
              require 'fluent/test'
         | 
| 100 | 
            +
              require 'fluent/test/driver/output'
         | 
| 98 101 |  | 
| 99 | 
            -
              require 'fluent/plugin/out_logzio'
         | 
| 100 102 | 
             
              require 'fluent/plugin/out_logzio_buffered'
         | 
| 101 103 | 
             
            end
         | 
| 102 104 |  | 
| @@ -112,9 +114,9 @@ RSpec.shared_context 'output context' do | |
| 112 114 | 
             
              before(:all) { Fluent::Test.setup }
         | 
| 113 115 | 
             
              let(:instance) { driver.instance }
         | 
| 114 116 |  | 
| 115 | 
            -
              let(:time) {0}
         | 
| 116 | 
            -
              let(:record1) {{ | 
| 117 | 
            -
              let(:record2) {{ | 
| 117 | 
            +
              let(:time) { 0 }
         | 
| 118 | 
            +
              let(:record1) { {'field1' => 50, 'otherfield' => 99} }
         | 
| 119 | 
            +
              let(:record2) { {'field1' => 150, 'otherfield' => 199} }
         | 
| 118 120 | 
             
              let(:request) { double }
         | 
| 119 121 | 
             
              let(:response) do
         | 
| 120 122 | 
             
                response = double
         | 
| @@ -124,4 +126,3 @@ RSpec.shared_context 'output context' do | |
| 124 126 | 
             
                response
         | 
| 125 127 | 
             
              end
         | 
| 126 128 | 
             
            end
         | 
| 127 | 
            -
             | 
    
        metadata
    CHANGED
    
    | @@ -1,15 +1,16 @@ | |
| 1 1 | 
             
            --- !ruby/object:Gem::Specification
         | 
| 2 2 | 
             
            name: fluent-plugin-logzio
         | 
| 3 3 | 
             
            version: !ruby/object:Gem::Version
         | 
| 4 | 
            -
              version: 0.0. | 
| 4 | 
            +
              version: 0.0.15
         | 
| 5 5 | 
             
            platform: ruby
         | 
| 6 6 | 
             
            authors:
         | 
| 7 7 | 
             
            - Yury Kotov
         | 
| 8 8 | 
             
            - Roi Rav-Hon
         | 
| 9 | 
            +
            - Arcadiy Ivanov
         | 
| 9 10 | 
             
            autorequire: 
         | 
| 10 11 | 
             
            bindir: bin
         | 
| 11 12 | 
             
            cert_chain: []
         | 
| 12 | 
            -
            date:  | 
| 13 | 
            +
            date: 2018-02-05 00:00:00.000000000 Z
         | 
| 13 14 | 
             
            dependencies:
         | 
| 14 15 | 
             
            - !ruby/object:Gem::Dependency
         | 
| 15 16 | 
             
              name: net-http-persistent
         | 
| @@ -17,88 +18,95 @@ dependencies: | |
| 17 18 | 
             
                requirements:
         | 
| 18 19 | 
             
                - - ~>
         | 
| 19 20 | 
             
                  - !ruby/object:Gem::Version
         | 
| 20 | 
            -
                    version: '2. | 
| 21 | 
            +
                    version: '2.9'
         | 
| 21 22 | 
             
              type: :runtime
         | 
| 22 23 | 
             
              prerelease: false
         | 
| 23 24 | 
             
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 24 25 | 
             
                requirements:
         | 
| 25 26 | 
             
                - - ~>
         | 
| 26 27 | 
             
                  - !ruby/object:Gem::Version
         | 
| 27 | 
            -
                    version: '2. | 
| 28 | 
            +
                    version: '2.9'
         | 
| 28 29 | 
             
            - !ruby/object:Gem::Dependency
         | 
| 29 30 | 
             
              name: fluentd
         | 
| 30 31 | 
             
              requirement: !ruby/object:Gem::Requirement
         | 
| 31 32 | 
             
                requirements:
         | 
| 32 | 
            -
                - -  | 
| 33 | 
            +
                - - '>='
         | 
| 34 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 35 | 
            +
                    version: 0.14.0
         | 
| 36 | 
            +
                - - <
         | 
| 33 37 | 
             
                  - !ruby/object:Gem::Version
         | 
| 34 | 
            -
                    version: ' | 
| 38 | 
            +
                    version: '2'
         | 
| 35 39 | 
             
              type: :runtime
         | 
| 36 40 | 
             
              prerelease: false
         | 
| 37 41 | 
             
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 38 42 | 
             
                requirements:
         | 
| 39 | 
            -
                - -  | 
| 43 | 
            +
                - - '>='
         | 
| 44 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 45 | 
            +
                    version: 0.14.0
         | 
| 46 | 
            +
                - - <
         | 
| 40 47 | 
             
                  - !ruby/object:Gem::Version
         | 
| 41 | 
            -
                    version: ' | 
| 48 | 
            +
                    version: '2'
         | 
| 42 49 | 
             
            - !ruby/object:Gem::Dependency
         | 
| 43 50 | 
             
              name: rake
         | 
| 44 51 | 
             
              requirement: !ruby/object:Gem::Requirement
         | 
| 45 52 | 
             
                requirements:
         | 
| 46 53 | 
             
                - - ~>
         | 
| 47 54 | 
             
                  - !ruby/object:Gem::Version
         | 
| 48 | 
            -
                    version: ' | 
| 55 | 
            +
                    version: '12.3'
         | 
| 49 56 | 
             
              type: :development
         | 
| 50 57 | 
             
              prerelease: false
         | 
| 51 58 | 
             
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 52 59 | 
             
                requirements:
         | 
| 53 60 | 
             
                - - ~>
         | 
| 54 61 | 
             
                  - !ruby/object:Gem::Version
         | 
| 55 | 
            -
                    version: ' | 
| 62 | 
            +
                    version: '12.3'
         | 
| 56 63 | 
             
            - !ruby/object:Gem::Dependency
         | 
| 57 64 | 
             
              name: bundler
         | 
| 58 65 | 
             
              requirement: !ruby/object:Gem::Requirement
         | 
| 59 66 | 
             
                requirements:
         | 
| 60 67 | 
             
                - - ~>
         | 
| 61 68 | 
             
                  - !ruby/object:Gem::Version
         | 
| 62 | 
            -
                    version: '1. | 
| 69 | 
            +
                    version: '1.16'
         | 
| 63 70 | 
             
              type: :development
         | 
| 64 71 | 
             
              prerelease: false
         | 
| 65 72 | 
             
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 66 73 | 
             
                requirements:
         | 
| 67 74 | 
             
                - - ~>
         | 
| 68 75 | 
             
                  - !ruby/object:Gem::Version
         | 
| 69 | 
            -
                    version: '1. | 
| 76 | 
            +
                    version: '1.16'
         | 
| 70 77 | 
             
            - !ruby/object:Gem::Dependency
         | 
| 71 78 | 
             
              name: rspec
         | 
| 72 79 | 
             
              requirement: !ruby/object:Gem::Requirement
         | 
| 73 80 | 
             
                requirements:
         | 
| 74 81 | 
             
                - - ~>
         | 
| 75 82 | 
             
                  - !ruby/object:Gem::Version
         | 
| 76 | 
            -
                    version: '3. | 
| 83 | 
            +
                    version: '3.7'
         | 
| 77 84 | 
             
              type: :development
         | 
| 78 85 | 
             
              prerelease: false
         | 
| 79 86 | 
             
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 80 87 | 
             
                requirements:
         | 
| 81 88 | 
             
                - - ~>
         | 
| 82 89 | 
             
                  - !ruby/object:Gem::Version
         | 
| 83 | 
            -
                    version: '3. | 
| 90 | 
            +
                    version: '3.7'
         | 
| 84 91 | 
             
            - !ruby/object:Gem::Dependency
         | 
| 85 92 | 
             
              name: test-unit
         | 
| 86 93 | 
             
              requirement: !ruby/object:Gem::Requirement
         | 
| 87 94 | 
             
                requirements:
         | 
| 88 95 | 
             
                - - ~>
         | 
| 89 96 | 
             
                  - !ruby/object:Gem::Version
         | 
| 90 | 
            -
                    version: '3. | 
| 97 | 
            +
                    version: '3.2'
         | 
| 91 98 | 
             
              type: :development
         | 
| 92 99 | 
             
              prerelease: false
         | 
| 93 100 | 
             
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 94 101 | 
             
                requirements:
         | 
| 95 102 | 
             
                - - ~>
         | 
| 96 103 | 
             
                  - !ruby/object:Gem::Version
         | 
| 97 | 
            -
                    version: '3. | 
| 104 | 
            +
                    version: '3.2'
         | 
| 98 105 | 
             
            description: Fluentd pluging (fluent.org) for output to Logz.io (logz.io)
         | 
| 99 106 | 
             
            email:
         | 
| 100 107 | 
             
            - bairkan@gmail.com
         | 
| 101 108 | 
             
            - roi@logz.io
         | 
| 109 | 
            +
            - arcadiy@ivanov.biz
         | 
| 102 110 | 
             
            executables: []
         | 
| 103 111 | 
             
            extensions: []
         | 
| 104 112 | 
             
            extra_rdoc_files: []
         | 
| @@ -112,7 +120,8 @@ files: | |
| 112 120 | 
             
            - fluent-plugin-logzio.gemspec
         | 
| 113 121 | 
             
            - lib/fluent/plugin/out_logzio_buffered.rb
         | 
| 114 122 | 
             
            - spec/lib/fluent/plugin/out_logzio_buffered_spec.rb
         | 
| 115 | 
            -
            - spec/lib/fluent/plugin/ | 
| 123 | 
            +
            - spec/lib/fluent/plugin/out_logzio_multi_bulk_spec.rb
         | 
| 124 | 
            +
            - spec/lib/fluent/plugin/out_logzio_record_too_large_spec.rb
         | 
| 116 125 | 
             
            - spec/spec_helper.rb
         | 
| 117 126 | 
             
            homepage: https://github.com/logzio/fluent-plugin-logzio
         | 
| 118 127 | 
             
            licenses:
         | 
| @@ -126,7 +135,7 @@ required_ruby_version: !ruby/object:Gem::Requirement | |
| 126 135 | 
             
              requirements:
         | 
| 127 136 | 
             
              - - '>='
         | 
| 128 137 | 
             
                - !ruby/object:Gem::Version
         | 
| 129 | 
            -
                  version: 1. | 
| 138 | 
            +
                  version: 2.1.0
         | 
| 130 139 | 
             
            required_rubygems_version: !ruby/object:Gem::Requirement
         | 
| 131 140 | 
             
              requirements:
         | 
| 132 141 | 
             
              - - '>='
         | 
| @@ -140,5 +149,6 @@ specification_version: 4 | |
| 140 149 | 
             
            summary: Fluentd plugin for output to Logz.io
         | 
| 141 150 | 
             
            test_files:
         | 
| 142 151 | 
             
            - spec/lib/fluent/plugin/out_logzio_buffered_spec.rb
         | 
| 143 | 
            -
            - spec/lib/fluent/plugin/ | 
| 152 | 
            +
            - spec/lib/fluent/plugin/out_logzio_multi_bulk_spec.rb
         | 
| 153 | 
            +
            - spec/lib/fluent/plugin/out_logzio_record_too_large_spec.rb
         | 
| 144 154 | 
             
            - spec/spec_helper.rb
         | 
| @@ -1,30 +0,0 @@ | |
| 1 | 
            -
            require 'spec_helper'
         | 
| 2 | 
            -
             | 
| 3 | 
            -
            describe 'Fluent::LogzioOutput' do
         | 
| 4 | 
            -
              let(:driver) { Fluent::Test::OutputTestDriver.new(Fluent::LogzioOutput).configure(config) }
         | 
| 5 | 
            -
              let(:config) do
         | 
| 6 | 
            -
                %[
         | 
| 7 | 
            -
                  endpoint_url         https://logz.io?token=123
         | 
| 8 | 
            -
                  output_include_time  false
         | 
| 9 | 
            -
                ]
         | 
| 10 | 
            -
              end
         | 
| 11 | 
            -
             | 
| 12 | 
            -
              include_context 'output context'
         | 
| 13 | 
            -
              include_examples 'output examples'
         | 
| 14 | 
            -
             | 
| 15 | 
            -
              describe 'emit' do
         | 
| 16 | 
            -
                before(:each) do
         | 
| 17 | 
            -
                  expect(request).to receive(:body=).with('{"field1":50,"otherfield":99,"fluentd_tags":"test"}')
         | 
| 18 | 
            -
                  expect(request).to receive(:body=).with('{"field1":150,"otherfield":199,"fluentd_tags":"test"}')
         | 
| 19 | 
            -
                  expect(Net::HTTP::Post).to receive(:new).with('/?token=123').twice.and_return(request)
         | 
| 20 | 
            -
                  expect_any_instance_of(Net::HTTP::Persistent).to receive(:request).twice.and_return(response)
         | 
| 21 | 
            -
                end
         | 
| 22 | 
            -
             | 
| 23 | 
            -
                it 'sends http requests' do
         | 
| 24 | 
            -
                  driver.run
         | 
| 25 | 
            -
             | 
| 26 | 
            -
                  driver.emit(record1, Time.at(time))
         | 
| 27 | 
            -
                  driver.emit(record2, Time.at(time))
         | 
| 28 | 
            -
                end
         | 
| 29 | 
            -
              end
         | 
| 30 | 
            -
            end
         |