fluent-plugin-newrelic 1.1.8 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: caf9616fa22e2338c4df33d70571d6a18850a17dafe98bb78c1e916aa7339ab5
-  data.tar.gz: a23147d65d03ac2de83765856f3ee84cc2afe26fbe24bc8c7e3c8f028316fb37
+  metadata.gz: 11df01d6cf7ef5b768dabccfc1d57f0aa7bf3bbd96010ab0d9109b0dae65434e
+  data.tar.gz: bd8b44e862c08b809fb7ad08913d679dca2ff13d3029dca8c6c31656fa317676
 SHA512:
-  metadata.gz: 991cab2d7e3060134f2f31784077d6e409c505a878a2d44a9eb1bf1d6ec8bc79752e3840881b0e7e0cddbb8547d9067bb153f76c650cded1d0c330eb9fc184bf
-  data.tar.gz: 716cb1b8150874ecb42c09bf678baa339d0f7bc65e4ebeb30badd9c31a9258d6a60d31d6376649bbbc64329bdcfc95e3d45fbb9b1bc022ea657940c3c4be9106
+  metadata.gz: 2f5b0a413c051418c0635352cac3bbac7b039c00c4fef14d694cd3a531111e07f85dc9f44d87641b774cc00869a51574e161644b3fc3392c257284e0342aeead
+  data.tar.gz: b8da217e0801af5053fad0539680348e5e94eeec56ddf4a67872c63afd968c53647ba2cde5b98424392b2104675298ee62a29e74bbc41d5790a0aad9911c23b7

data/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,36 @@
+---
+name: Bug report
+about: Report a bug so we can take care of it
+title: ''
+labels: bug
+assignees: ''
+---
+
+[NOTE]: # ( ^^ Provide a general summary of the issue in the title above ^^ )
+
+## Description
+
+[NOTE]: # ( Describe the problem you're encountering )
+[TIP]:  # ( Do NOT share sensitive information, whether personal, proprietary, or otherwise! )
+
+## Expected Behavior
+
+[NOTE]: # ( Tell us what you expected to happen )
+
+## [Troubleshooting](https://discuss.newrelic.com/t/troubleshooting-frameworks/108787) or [NR Diag](https://docs.newrelic.com/docs/using-new-relic/cross-product-functions/troubleshooting/new-relic-diagnostics) results
+
+[NOTE]: # ( Provide any other relevant log data )
+[TIP]:  # ( Scrub logs and diagnostic information for sensitive information )
+
+## Steps to Reproduce
+
+[NOTE]: # ( Please be as specific as possible )
+[TIP]:  # ( Link a sample application that demonstrates the issue )
+
+## Your Environment
+
+[TIP]:  # ( Include as many relevant details about your environment as possible including the running version of New Relic software and any relevant configurations. )
+
+## Additional context
+
+[TIP]:  # ( Add any other context about the problem here. For example, relevant community posts, support tickets, screenshots... )

data/.github/workflows/merge-to-master.yml

@@ -0,0 +1,40 @@
+name: New Relic Fluentd Output Plugin - Merge to master
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  ci:
+    name: Continuous Delivery pipeline
+    runs-on: ubuntu-18.04
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Setup Ruby, bundler and install dependencies
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ruby-2.7.2
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+
+      - name: Run unit tests
+        run: bundle exec rake
+
+      - name: Publish Unit Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v1.5
+        if: always()
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          files: '**/TEST-*.xml'
+
+      - name: Build gem
+        run: gem build newrelic-fluentd-output.gemspec
+
+      - name: Publish fluent-plugin-newrelic to rubygems.org
+        env:
+          GEM_HOST_API_KEY: ${{ secrets.GEM_HOST_API_KEY }}
+        run: |
+          gem push fluent-plugin-newrelic-*.gem

data/.github/workflows/pr.yml

@@ -0,0 +1,28 @@
+name: New Relic Fluentd Output Plugin - Pull Request
+
+on: [pull_request]
+
+jobs:
+  ci:
+    name: Continuous Integration pipeline
+    runs-on: ubuntu-18.04
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Setup Ruby, bundler and install dependencies
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ruby-2.5.8
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+
+      - name: Run unit tests
+        run: bundle exec rake
+
+      - name: Publish Unit Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v1.5
+        if: always()
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          files: '**/TEST-*.xml'

data/.github/workflows/repolinter.yml

@@ -0,0 +1,31 @@
+# NOTE: This file should always be named `repolinter.yml` to allow
+# workflow_dispatch to work properly
+name: Repolinter Action
+
+# NOTE: This workflow will ONLY check the default branch!
+# Currently there is no elegant way to specify the default
+# branch in the event filtering, so branches are instead
+# filtered in the "Test Default Branch" step.
+on: [push, workflow_dispatch]
+
+jobs:
+  repolint:
+    name: Run Repolinter
+    runs-on: ubuntu-latest
+    steps:
+      - name: Test Default Branch
+        id: default-branch
+        uses: actions/github-script@v2
+        with:
+          script: |
+            const data = await github.repos.get(context.repo)
+            return data.data && data.data.default_branch === context.ref.split('/').slice(-1)[0]
+      - name: Checkout Self
+        if: ${{ steps.default-branch.outputs.result == 'true' }}
+        uses: actions/checkout@v2
+      - name: Run Repolinter
+        if: ${{ steps.default-branch.outputs.result == 'true' }}
+        uses: newrelic/repolinter-action@v1
+        with:
+          config_url: https://raw.githubusercontent.com/newrelic/.github/main/repolinter-rulesets/community-plus.yml
+          output_type: issue

data/README.md

@@ -1,3 +1,5 @@
+[![Community Plus header](https://github.com/newrelic/opensource-website/raw/master/src/images/categories/Community_Plus.png)](https://opensource.newrelic.com/oss-category/#community-plus)
+
 # fluent-plugin-newrelic
 
 A [Fluentd](https://fluentd.org/) output plugin that sends logs to New Relic
@@ -29,12 +31,12 @@ For more info, review [Fluentd's official documentation](https://docs.fluentd.or
 
 ### Required plugin configuration
 
-Exactly one of the following:
+This plugin must be configured with either a New Relic API Insert key, or a New Relic License key.
+If both types of keys are specified, the API Insert key will take precedence.
+
+To specify an API Insert key, either set the `api_key` property in the configuration, or set the `NEW_RELIC_API_KEY` environment variable. If both are specified, the configuration property will take precedence.
 
-| Property | Description |
-|---|---|
-| api_key | your New Relic API Insert key |
-| license_key | your New Relic License key |
+To specify a license key, either set the `license_key` property in the configuration, or set the `NEW_RELIC_LICENSE_KEY` environment variable. If both are specified, the configuration property will take precedence.
 
 ### Optional plugin configuration
 
@@ -84,6 +86,30 @@ Example using License key:
 Getting your New Relic license key:
 `https://rpm.newrelic.com/accounts/<ACCOUNT_ID>`
 
+## Playground
+
+To provide a sandbox environment where you can play with the plugin while development or testing
+we've added a docker-compose environment that would help to run fluend with the plugin and send
+some logs. [More info](playground/README.md)
+
+## Community
+
+New Relic hosts and moderates an online forum where customers can interact with New Relic employees as well as other customers to get help and share best practices. Like all official New Relic open source projects, there's a related Community topic in the New Relic Explorers Hub: [Log forwarding](https://discuss.newrelic.com/tag/log-forwarding)
+
+
+**A note about vulnerabilities**
+
+As noted in our [security policy](../../security/policy), New Relic is committed to the privacy and security of our customers and their data. We believe that providing coordinated disclosure by security researchers and engaging with the security community are important means to achieve our security goals.
+
+If you believe you have found a security vulnerability in this project or any of New Relic's products or websites, we welcome and greatly appreciate you reporting it to New Relic through [HackerOne](https://hackerone.com/newrelic).
+
+If you would like to contribute to this project, **please create a branch directly in this repository** and review [these guidelines](https://opensource.newrelic.com/code-of-conduct/). 
+
+## License
+
+newrelic-fluentd-output is licensed under the [Apache 2.0](http://apache.org/licenses/LICENSE-2.0.txt) License.
+
+
 ## Copyright
 
 * Copyright(c) 2019 - New Relic

data/examples/Windows-IIS.conf

@@ -0,0 +1,85 @@
+####
+## New Relic Logs - Basic Windows Event + IIS config v1.5
+## Built in FluentD v1.7.4
+## by AI of NR 2/4/2020
+##
+## Don't forget to add your NR license key to the <match> statement at the bottom of this file.
+##
+
+## Windows Event Log Input
+<source>
+  @type windows_eventlog
+  @id windows_eventlog
+  tag winevt.raw
+  channels application,system,security
+  <storage>
+    @type local
+    persistent true
+    path c:/opt/td-agent/winevt.pos
+  </storage>
+</source>
+
+#
+# Windows IIS log parsing.  This config uses the standard W3C format and the standard location for IIS logs.
+# It expects the log timestamp to be UTC.
+# Change the path below if you store your logs elsewhere.
+#
+<source>
+  @type tail
+  tag iislog.raw
+  path c:/inetpub/logs/logfiles/*/*
+  pos_file c:/opt/td-agent/iislog.pos
+  <parse>
+    @type regexp
+    expression /(?<time>\d{4}-\d{2}-\d{2} [\d:]+) (?<message>.+)/
+    time_format %Y-%m-%d %H:%M:%S
+	utc true
+  </parse>
+</source>
+
+<filter iislog.raw>
+  @type parser
+  key_name message
+  remove_key_name_field false
+  reserve_data true
+  reserve_time true
+  <parse>
+    @type csv
+    delimiter ' '
+    keys hostname,req_method,req_uri,cs-uri-query,s_port,cs-username,c_ip,req_ua,req_referer,http_code,sc-substatus,sc-win32-status,time-taken
+    null_value_pattern -
+    null_empty_string true
+  </parse>
+</filter>
+
+#
+# For a slightly nicer experience, add Service Name (s-sitename) to your log output, comment out the filter above and use this one instead.
+#
+#<filter iislog.raw>
+#  @type parser
+#  key_name message
+#  remove_key_name_field false
+#  reserve_data true
+#  reserve_time true
+#  <parse>
+#    @type csv
+#    delimiter ' '
+#    keys service_name,hostname,req_method,req_uri,cs-uri-query,s_port,cs-username,c_ip,req_ua,req_referer,http_code,sc-substatus,sc-win32-status,time-taken
+#    null_value_pattern -
+#    null_empty_string true
+#  </parse>
+#</filter>
+
+<filter winevt.raw>
+  @type record_transformer
+  <record>
+    message ${record["description"]}
+	  hostname ${record["computer_name"]}
+  </record>
+</filter>
+
+# New Relic output
+<match **>
+  @type newrelic
+  api_key <YOUR INSERT KEY>
+</match>

data/examples/copy_output.conf

@@ -0,0 +1,43 @@
+#Tail and parse Apache access logs
+
+<source>
+   @type tail
+   @id input_tail_apache
+   tag apache_access
+   path /var/log/apache2/access.log
+   pos_file /var/log/apache2/access.pos
+   path_key filename
+   <parse>
+   @type apache2
+   </parse>
+ </source>
+ 
+ #Add hostname and tag fields to all events ("records") with a Fluentd tag of apache_access
+ 
+ <filter apache_access>
+  @type record_transformer
+  <record>
+    hostname "#{Socket.gethostname}"
+    tag ${tag}
+  </record>
+</filter>
+
+#Output (https://docs.fluentd.org/output/copy) events to both New Relic and a local file.
+
+<match **>
+  @type copy
+  <store>
+  @type newrelic
+  api_key <YOUR INSERT KEY>
+  </store>
+  <store>
+  @type file
+  path /var/log/apacheout.log
+  <buffer>
+    #Buffer settings are for testing and not recommended for use in a production environment.
+    timekey 10s
+    timekey_use_utc true
+    timekey_wait 15s
+  </buffer>
+  </store>
+</match>

data/examples/custom_field.conf

@@ -0,0 +1,32 @@
+#Tail and parse Docker log files
+
+<source>
+  @type tail
+  path /var/lib/docker/containers/*/*-json.log
+  pos_file /var/log/docker-log.pos
+  read_from_head true
+  tag containers
+  <parse>
+    @type json
+    time_format %Y-%m-%dT%H:%M:%S.%NZ
+    keep_time_key true
+    time_key time
+  </parse>
+</source>
+
+<filter containers>
+  @type record_transformer
+  enable_ruby true
+  <record>
+    #Add hostname and tag fields to all records
+    fluentd_host "#{Socket.gethostname}"
+    tag ${tag}
+  </record>
+</filter>
+
+# Forward events to New Relic
+
+<match containers>
+  @type newrelic
+  api_key <YOUR INSERT KEY>
+  </match>

data/examples/file_input.conf

@@ -0,0 +1,29 @@
+#Tail arbitrary text/log file
+
+<source>
+  @type tail
+  <parse>
+    @type none
+  </parse>
+  path /var/log/backend-app*.log
+  pos_file /var/log/backend.application.pos
+  path_key filename # Add watched file path to path_key field for every event/record.
+  tag backend.application
+</source>
+
+ #Add hostname and tag fields to all events ("records") with a Fluentd tag of backend.application
+ 
+ <filter backend.application>
+  @type record_transformer
+  <record>
+    hostname "#{Socket.gethostname}"
+    tag ${tag}
+  </record>
+</filter>
+
+#Write events to New Relic
+
+<match backend.application>
+  @type newrelic
+  api_key <YOUR INSERT KEY>
+</match>

data/examples/filter_logs.conf

@@ -0,0 +1,52 @@
+#Tail and parse arbitrary text/log file
+
+<source>
+  @type tail
+  <parse> #Parse timestamp, everything else to be stored in message field
+    @type regexp
+    expression /^\[(?<logtime>[^\]]*)\] (?<message>.*)$/
+    time_key logtime
+    time_format %Y-%m-%d %H:%M:%S %z
+  </parse>
+  path /var/log/backend-app*.log
+  pos_file /var/log/backend.application.pos
+  path_key filename # Add watched file path to path_key field for every event/record.
+  tag backend.application
+</source>
+
+#Add hostname and service_name fields to all events ("records") with a Fluentd tag of backend.application
+ 
+<filter backend.application>
+  @type record_transformer
+  <record>
+    hostname "#{Socket.gethostname}"
+    service_name ${tag}
+  </record>
+</filter>
+
+# For all events with a tag of backend.application:
+# Keep ONLY events where service_name field contains a value matching /backend.application/ AND where message field contains a value matching /Cannot connect to/
+# Discard any events where value of hostname field matches /staging/
+
+<filter backend.application>
+  @type grep
+  <regexp>
+    key service_name
+    pattern /backend.application/
+  </regexp>
+   <regexp>
+    key message
+    pattern /Cannot connect to/
+  </regexp>
+  <exclude>
+    key hostname
+    pattern /staging/
+  </exclude>
+</filter>
+
+#Write events with backend.application tag to New Relic
+
+<match backend.application>
+  @type newrelic
+  api_key <YOUR INSERT KEY>
+</match>

data/examples/grok_parser.conf

@@ -0,0 +1,43 @@
+#Tail arbitrary log file and parse using grok pattern
+#Install the required plugin: fluent-gem install fluent-plugin-grok-parser
+
+<source>
+  @type tail
+  <parse>
+    @type grok
+    <grok>
+      pattern %{SYSLOGTIMESTAMP:timestamp} %{LOGLEVEL:loglevel}: %{GREEDYDATA:message}
+    </grok>
+  </parse>
+  path /var/log/customapp.log
+  pos_file /var/log/customapp.pos
+  path_key filename
+  tag custom.application
+</source>
+
+# Drop events with custom.application tag where loglevel field contains "debug" or "info" (case-insensitive match)
+
+<filter custom.application>
+  @type grep
+  <exclude>
+    key loglevel
+    pattern /debug|info/i
+  </exclude>
+</filter>
+
+#Add hostname and tag fields to all events ("records") with a Fluentd tag of custom.application
+ 
+ <filter custom.application>
+  @type record_transformer
+  <record>
+    hostname "#{Socket.gethostname}"
+    tag ${tag}
+  </record>
+</filter>
+
+#Write custom.application events to New Relic
+
+<match custom.application>
+  @type newrelic
+  api_key <YOUR INSERT KEY>
+</match>

data/examples/minimal_complete_config.conf

@@ -0,0 +1,27 @@
+#Tail arbitrary text/log file
+
+<source>
+  @type tail
+  <parse>
+    @type none
+  </parse>
+  path /home/logs/*
+  path_key file
+  tag sample.tag
+</source>
+
+#Add service_name field to all events ("records") with a Fluentd tag of sample.tag
+
+<filter sample.tag>
+  @type record_transformer
+  <record>
+    service_name ${tag}
+  </record>
+</filter>
+
+#Write sample.tag events to New Relic
+
+<match sample.tag>
+  @type newrelic
+  api_key <YOUR INSERT KEY>
+</match>

data/examples/multiline_log_parse.conf

@@ -0,0 +1,11 @@
+ <filter backend.application>
+    @type parser
+    <parse>
+      @type multiline_grok
+      grok_failure_key grokfailure
+      multiline_start_regex ^abc
+      <grok>
+        pattern %{GREEDYDATA:message}
+      </grok>
+    </parse>
+  </filter>

data/examples/parse_nginx.conf

@@ -0,0 +1,17 @@
+#Tail and parse NGINX log file
+
+<source>
+  @type tail
+  <parse>
+    @type nginx
+  </parse>
+  path /path/to/access.log
+  tag nginx.access
+</source>
+
+#Write events with tag matching nginx.* to New Relic
+
+<match nginx.*>
+  @type newrelic
+  api_key <YOUR INSERT KEY>
+</match>

data/examples/readme.md

@@ -61,9 +61,9 @@ sudo mkdir /etc/fluentd
 sudo nano /etc/fluentd/fluentd.conf
 ```
 
-#### 2. Add the contents from the [`syslog\fluentd.conf`](syslog\fluentd.config)
+#### 2. Add the contents from the [`syslog/fluentd.conf`](syslog/fluentd.config)
 
-You can find the contents of the [`syslog\fluentd.conf`](syslog\fluentd.config) in the sub folder `syslog`. These contents should provide a quick start to getting started. In the provided
+You can find the contents of the [`syslog/fluentd.conf`](syslog/fluentd.config) in the sub folder `syslog`. These contents should provide a quick start to getting started. In the provided
 example, the syslog details are coming from the above mentioned devices. You may need to tweak the configuration according to the server sending the syslog traffic.
 
 #### 3. Check New Relic for New Logs

data/examples/syslog_input.conf

@@ -0,0 +1,5 @@
+<source>
+  @type syslog
+  port 5140
+  tag syslog.messages
+</source>

data/lib/fluent/plugin/out_newrelic.rb

@@ -33,10 +33,15 @@ module Fluent
       config_param :license_key, :string, :default => nil
 
       DEFAULT_BUFFER_TYPE = 'memory'.freeze
+      DEFAULT_TIMEKEY = 5
+      DEFAULT_TIMEKEY_WAIT = 0
+      MAX_PAYLOAD_SIZE = 1000000 # bytes
 
       config_section :buffer do
         config_set_default :@type, DEFAULT_BUFFER_TYPE
-        config_set_default :chunk_keys, ['timestamp']
+        config_set_default :chunk_keys, ['time']
+        config_set_default :timekey, DEFAULT_TIMEKEY
+        config_set_default :timekey_wait, DEFAULT_TIMEKEY_WAIT
       end
 
       define_method('log') {$log} unless method_defined?(:log)
@@ -49,8 +54,11 @@ module Fluent
 
       def configure(conf)
         super
+
+        @api_key ||= ENV["NEW_RELIC_API_KEY"]
+        @license_key ||= ENV["NEW_RELIC_LICENSE_KEY"]
         if @api_key.nil? && @license_key.nil?
-          raise Fluent::ConfigError.new("'api_key' or 'license_key' parameter is required") 
+          raise Fluent::ConfigError.new("'api_key' or 'license_key' parameter is required")
         end
 
         # create initial sockets hash and socket based on config param
@@ -89,39 +97,23 @@ module Fluent
           packaged['message'] = record['log']
           packaged['attributes'].delete('log')
         end
-        
+
         packaged
       end
 
       def write(chunk)
-        payload = {
-          'common' => {
-            'attributes' => {
-              'plugin' => {
-                'type' => 'fluentd',
-                'version' => NewrelicFluentdOutput::VERSION,
-              }
-            }
-          },
-          'logs' => []
-        }
+        logs = []
         chunk.msgpack_each do |ts, record|
           next unless record.is_a? Hash
           next if record.empty?
-          payload['logs'].push(package_record(record, ts))
+          logs.push(package_record(record, ts))
         end
-        io = StringIO.new
-        gzip = Zlib::GzipWriter.new(io)
 
-        # Fluentd can run with a version of Ruby (2.1.0) whose to_json method doesn't support non-ASCII characters.
-        # So we use Yajl, which can handle all Unicode characters. Apparently this library is what Fluentd uses
-        # internally, so it is installed by default with td-agent.
-        # See https://github.com/fluent/fluentd/issues/215
-        gzip << Yajl.dump([payload])
-        gzip.close
-        send_payload(io.string)
+
+        payloads = get_compressed_payloads(logs)
+        payloads.each { |payload| send_payload(payload) }
       end
-      
+
       def handle_response(response)
         if !(200 <= response.code.to_i && response.code.to_i < 300)
           log.error("Response was " + response.code + " " + response.body)
@@ -137,6 +129,58 @@ module Fluent
         handle_response(http.request(request))
       end
 
+      private
+
+      def get_compressed_payloads(logs)
+        return [] if logs.length == 0
+
+        payload = create_payload(logs)
+        compressed_payload = compress(payload)
+
+        if compressed_payload.bytesize <= MAX_PAYLOAD_SIZE
+          return [compressed_payload]
+        end
+
+        compressed_payload = nil # Free for GC
+
+        if logs.length > 1 # we can split
+          # let's split logs array by half, and try to create payloads again
+          midpoint = logs.length / 2
+          first_half = get_compressed_payloads(logs.slice(0, midpoint))
+          second_half = get_compressed_payloads(logs.slice(midpoint, logs.length))
+          return first_half + second_half
+        else
+          log.error("Can't compress record below required maximum packet size and it will be discarded. Record: #{logs[0]}")
+          return []
+        end
+      end
+
+      def create_payload(logs)
+        {
+          'common' => {
+            'attributes' => {
+              'plugin' => {
+                'type' => 'fluentd',
+                'version' => NewrelicFluentdOutput::VERSION,
+              }
+            }
+          },
+          'logs' => logs
+        }
+      end
+
+      def compress(payload)
+        io = StringIO.new
+        gzip = Zlib::GzipWriter.new(io)
+
+        # Fluentd can run with a version of Ruby (2.1.0) whose to_json method doesn't support non-ASCII characters.
+        # So we use Yajl, which can handle all Unicode characters. Apparently this library is what Fluentd uses
+        # internally, so it is installed by default with td-agent.
+        # See https://github.com/fluent/fluentd/issues/215
+        gzip << Yajl.dump([payload])
+        gzip.close
+        io.string
+      end
     end
   end
 end

data/lib/newrelic-fluentd-output/version.rb

@@ -1,3 +1,3 @@
 module NewrelicFluentdOutput
-  VERSION = "1.1.8"
+  VERSION = "1.2.1"
 end

data/playground/.env.example

@@ -0,0 +1 @@
+NEW_RELIC_API_KEY=REPLACE_WITH_YOUR_API_KEY

data/playground/.gitignore

@@ -0,0 +1 @@
+.env

data/playground/Dockerfile

@@ -0,0 +1,7 @@
+FROM fluent/fluentd:v1.12-1
+
+WORKDIR /usr/share/fluentd
+
+USER root
+
+RUN fluent-gem install fluent-plugin-newrelic

data/playground/README.md

@@ -0,0 +1,40 @@
+# New Relict Fluentd output plugin playground
+
+## Starting the environment
+
+To run the environment first go to `./playground` folder:
+
+1. Copy `.env.example` to `.env` and fill your own New Relic API KEY or LICENSE KEY.
+2. Run `docker-compose up -d`
+
+Thats it, you should have a running docker with latests new relic output plugin version.
+
+## Writing logs
+
+Since the file `./testlogs/test.log` are mounted as a volume on the docker instance and
+is being tailed by fluentd you could write some text to the file and it should be picked
+up by fluentd.
+
+This command should write a line on the log file and it will reach new relic in a while.
+
+`echo "Hello world" >> ./testlogs/test.log`
+
+If what you need is send logs every .05s, for example, you can have a big file like
+[1mb of text](https://gist.github.com/khaykov/a6105154becce4c0530da38e723c2330) and
+use some awk magic to send each line with .05s delay.
+
+`awk '{print $0; system("sleep .05");}' 1mbofdata.txt >> ./testlogs/test.log`
+
+## Troubleshooting
+
+If you need to go inside the container to debug something just run `docker-compose exec fluentd sh`
+and you'll be inside the instance. This is a development image so you'll find that you're
+logged in as root, this allows you to install the tools you need with `apk`. For example
+`apk add vim`.
+
+The plugin is located on `/usr/lib/ruby/gems/2.7.0/gems/fluent-plugin-newrelic-{version}`
+
+## More examples
+
+For more examples, you could adapt the configurations on [examples](../examples/readme.md)
+to use them with this playground and test whatever you need.

data/playground/config/fluent.conf

@@ -0,0 +1,23 @@
+<source>
+  @type tail
+  <parse>
+    @type none
+  </parse>
+  path /testlogs/*
+  path_key file
+  tag sample.tag
+</source>
+
+# Add service_name field to all events ("records") with a Fluentd tag of sample.tag
+<filter sample.tag>
+  @type record_transformer
+  <record>
+    logtype test
+    service_name fluentd-test
+  </record>
+</filter>
+
+# Write sample.tag events to New Relic
+<match sample.tag>
+  @type newrelic
+</match>

data/playground/docker-compose.yml

@@ -0,0 +1,12 @@
+version: "3.0"
+
+services:
+  fluentd:
+    env_file: .env
+    build:
+      dockerfile: ./Dockerfile
+      context: .
+    volumes:
+      - "./config:/fluentd/etc"
+      - "./testlogs/:/testlogs/:rw"
+    command: fluentd -c /fluentd/etc/fluent.conf -v

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-newrelic
 version: !ruby/object:Gem::Version
-  version: 1.1.8
+  version: 1.2.1
 platform: ruby
 authors:
 - New Relic Logging Team
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-03-09 00:00:00.000000000 Z
+date: 2021-07-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -102,6 +102,10 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".dockerignore"
+- ".github/ISSUE_TEMPLATE/bug_report.md"
+- ".github/workflows/merge-to-master.yml"
+- ".github/workflows/pr.yml"
+- ".github/workflows/repolinter.yml"
 - ".gitignore"
 - DEVELOPER.md
 - Dockerfile
@@ -110,12 +114,30 @@ files:
 - README.md
 - Rakefile
 - examples/Dockerfile
+- examples/Windows-IIS.conf
+- examples/copy_output.conf
+- examples/custom_field.conf
+- examples/file_input.conf
+- examples/filter_logs.conf
+- examples/grok_parser.conf
+- examples/minimal_complete_config.conf
+- examples/multiline_log_parse.conf
+- examples/parse_nginx.conf
 - examples/readme.md
 - examples/syslog/fluentd.config
+- examples/syslog_input.conf
 - lib/fluent/plugin/out_newrelic.rb
 - lib/newrelic-fluentd-output/version.rb
 - merge-to-master-pipeline.yml
 - newrelic-fluentd-output.gemspec
+- playground/.env.example
+- playground/.gitignore
+- playground/Dockerfile
+- playground/README.md
+- playground/config/fluent.conf
+- playground/docker-compose.yml
+- playground/testlogs/.gitkeep
+- playground/testlogs/test.log
 - pr-pipeline.yml
 homepage: https://github.com/newrelic/newrelic-fluentd-output
 licenses:
@@ -136,7 +158,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Sends FluentD events to New Relic