fluent-plugin-viaq_data_model 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ffc8861a24c80f6044ffe80d9e38c5f675a46554
-  data.tar.gz: 26a1dfb59dfacaac6975c8a76ef03aff6896380e
+  metadata.gz: f4ecf55fa6bf6e1001738e41876a8ba70f4fee67
+  data.tar.gz: 88d75546acb5b4db61dc09046a7873a2382c60d7
 SHA512:
-  metadata.gz: 5b7314c78d6b2443aff7d31f5008e95dd2dad8192e248cd1a34d09f31c7828aef16d94fd4e7a4effd283b627ef2a6f641ee014cf6fa9e1fe5acfc0937246fd99
-  data.tar.gz: b118bb6a73721dbb05e82492222b03530722ad6539bd10026bdc66a051fc5f9a2a8fa7772838b2ec2c81450f7edec100a1461096a5c77f472ef26b0e6e980c3a
+  metadata.gz: 9f2fd967b4d57cb6afe9c8a46d723e8e30c53f71a056bdb436e6fceb5dad489c52a8eded2068d95c6ccb47f7deb728f6a5d2b568034cc29d7cb8e3e11a9e4503
+  data.tar.gz: af3c09cabf0b8ece8a659bb8c1116be62d53f90da62768ccdc2d6be2cc87ce257fb12e051f1cd9d52565e29344c40e5f6aa4cebaee34f5ac70d60dccda27badb

data/.travis.yml

@@ -0,0 +1,12 @@
+language: ruby
+
+rvm:
+  - 2.1
+  - 2.2
+  - 2.3.1
+
+gemfile:
+ - Gemfile
+
+script: bundle exec rake test
+sudo: false

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+gem 'codeclimate-test-reporter', :group => :test, :require => nil
+gem 'rubocop', require: false
+
+# Specify your gem's dependencies in fluent-plugin-add.gemspec
+gemspec

data/README.md

@@ -1,2 +1,118 @@
-# fluent-plugin-viaq_data_model
-helps format records into viaq data model
+# fluent-plugin-viaq_data_model - a ViaQ data model filter plugin for [Fluentd](http://fluentd.org)
+[![Travis CI](https://secure.travis-ci.org/ViaQ/fluent-plugin-viaq_data_model.png)](http://travis-ci.org/#!/ViaQ/fluent-plugin-viaq_data_model)
+
+## Introduction
+
+This plugin formats Fluentd records in the proper [ViaQ data
+model](https://github.com/ViaQ/elasticsearch-templates).  It does the
+following:
+
+* Removes empty fields
+  * fields with a value of `nil`
+  * string fields with a value of `''` or the empty string
+  * hash valued fields with a value of `{}`
+  * hash valued fields which contain only empty fields as described above
+  * FixNum, Boolean and other field values are not removed - type must respond
+    to `:empty?` to be considered empty
+
+* Moves "undefined" values to a top level field called `undefined`
+
+The ViaQ data model wants all top level fields defined and described.  These
+can conflict with the fields defined by ViaQ.  You can "move" these fields to
+be under a hash valued top level field called `undefined` so as not to conflict
+with the "well known" ViaQ top level fields.  You can optionally keep some
+fields as top level fields while moving others to the `undefined` container.
+
+* Rename a time field to `@timestamp`
+
+You cannot set the `@timestamp` field in a Fluentd `record_transformer` filter.
+The plugin allows you to use some other field e.g. `time` and have that "moved"
+to a top level field called `@timestamp`.
+
+## Configuration
+
+NOTE: All fields are Optional - no required fields.
+
+See `filter-viaq_data_model.conf` for an example filter configuration.
+
+* `default_keep_fields` - comma delimited string - default: `''`
+  * This is the default list of fields to keep as top level fields in the record
+  * `default_keep_fields message,@timestamp,ident` - do not move these fields into the `undefined` field
+* `extra_keep_fields` - comma delimited string - default: `''`
+  * This is an extra list of fields to keep in addition to
+  `default_keep_fields` - mostly useful as a way to hard code the
+  `default_keep_fields` list for configuration management purposes, but at the
+  same time allow customization in certain cases
+  * `extra_keep_fields myspecialfield1,myspecialfield2`
+* `keep_empty_fields` - comma delimited string - default `''`
+  * Always keep these top level fields, even if they are empty
+  * `keep_empty_fields message` - keep the `message` field, even if empty
+* `use_undefined` - boolean - default `false`
+  * If `true`, move fields not specified in `default_keep_fields` and
+  `extra_keep_fields` to the `undefined` top level field.  If you use
+  `use_undefined` you should specify the fields you want to keep out of
+  `undefined` by using `default_keep_fields` and/or `extra_keep_fields`
+* `undefined_name` - string - default `"undefined"`
+  * Name of undefined top level field to use if `use_undefined true` is set
+  * `undefined_name myfields` - keep undefined fields under field `myfields`
+* `rename_time` - boolean - default `true`
+  * Rename the time field e.g. when you need to set `@timestamp` in the record
+  * NOTE: This will overwrite the `dest_time_name` if already set
+* `rename_time_if_missing` - boolean - default `false`
+  * Rename the time field only if it is not present.  For example, if some
+  records already have the `@timestamp` field and you do not want to overwrite
+  them, use `rename_time_if_missing true`
+* `src_time_name` - string - default `time`
+  * Use this field to get the value of the time field in the resulting record.
+    This field will be removed from the record.
+  * NOTE: This field must be present in the `default_keep_fields` or
+  `extra_keep_fields` if `use_undefined true`
+* `dest_time_name` - string - default `@timestamp`
+  * This is the name of the top level field to hold the time value.  The value
+  is taken from the value of the `src_time_name` field.
+
+## Example
+
+If the input record looks like this:
+
+    {
+      "a": "b",
+      "c": "d",
+      "e": '',
+      "f": {
+        "g": '',
+        "h": {}
+      },
+      "i": {
+        "j": 0,
+        "k": False,
+        "l": ''
+      },
+      "time": "2017-02-13 15:30:10.259106596-07:00"
+    }
+
+The resulting record, using the defaults, would look like this:
+
+    {
+      "a": "b",
+      "c": "d",
+      "i": {
+        "j": 0,
+        "k": False,
+      },
+      "@timestamp": "2017-02-13 15:30:10.259106596-07:00"
+    }
+
+
+## Installation
+
+    gem install fluent-plugin-viaq_data_model
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Test it (`GEM_HOME=vendor bundle install; GEM_HOME=vendor bundle exec rake test`)
+5. Push to the branch (`git push origin my-new-feature`)
+6. Create new Pull Request

data/filter-viaq_data_model.conf

@@ -6,6 +6,7 @@
   use_undefined "#{ENV['CDM_USE_UNDEFINED'] || false}"
   undefined_name "#{ENV['CDM_UNDEFINED_NAME'] || 'undefined'}"
   rename_time "#{ENV['CDM_RENAME_TIME'] || true}"
+  rename_time_if_missing "#{ENV['CDM_RENAME_TIME_IF_MISSING'] || false}"
   src_time_name "#{ENV['CDM_SRC_TIME_NAME'] || 'time'}"
   dest_time_name "#{ENV['CDM_DEST_TIME_NAME'] || '@timestamp'}"
 </filter>

data/fluent-plugin-viaq_data_model.gemspec

@@ -4,12 +4,12 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-viaq_data_model"
-  gem.version       = "0.0.1"
+  gem.version       = "0.0.2"
   gem.authors       = ["Rich Megginson"]
   gem.email         = ["rmeggins@redhat.com"]
   gem.description   = %q{Filter plugin to ensure data is in the ViaQ common data model}
   gem.summary       = %q{Filter plugin to ensure data is in the ViaQ common data model}
-  gem.homepage      = "https://github.com/ViaQ/fluent-plugin-viaq_data_mode"
+  gem.homepage      = "https://github.com/ViaQ/fluent-plugin-viaq_data_model"
   gem.license       = "Apache-2.0"
 
   gem.files         = `git ls-files`.split($/)

data/lib/fluent/plugin/filter_viaq_data_model.rb

@@ -1,3 +1,20 @@
+#
+# Fluentd ViaQ data model Filter Plugin
+#
+# Copyright 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
 require 'fluent/filter'
 require 'fluent/log'
 
@@ -33,6 +50,9 @@ module Fluent
     desc 'Rename timestamp field to Elasticsearch compatible name'
     config_param :rename_time, :bool, default: true
 
+    desc 'Rename timestamp field to Elasticsearch compatible name only if the destination field does not already exist'
+    config_param :rename_time_if_missing, :bool, default: false
+
     desc 'Name of source timestamp field'
     config_param :src_time_name, :string, default: 'time'
 
@@ -53,7 +73,7 @@ module Fluent
       if @use_undefined && @keep_fields.key?(@undefined_name)
         raise Fluent::ConfigError, "Do not put [#{@undefined_name}] in default_keep_fields or extra_keep_fields"
       end
-      if @rename_time && @use_undefined && !@keep_fields.key?(@src_time_name)
+      if (@rename_time || @rename_time_if_not_exist) && @use_undefined && !@keep_fields.key?(@src_time_name)
         raise Fluent::ConfigError, "Field [#{@src_time_name}] must be listed in default_keep_fields or extra_keep_fields"
       end
     end
@@ -102,8 +122,11 @@ module Fluent
       # probably shouldn't remove everything . . .
       log.warn("Empty record! tag [#{tag}] time [#{time}]") if record.empty?
       # rename the time field
-      if @rename_time && record.key?(@src_time_name)
-        record[@dest_time_name] = record.delete(@src_time_name)
+      if (@rename_time || @rename_time_if_missing) && record.key?(@src_time_name)
+        val = record.delete(@src_time_name)
+        unless @rename_time_if_missing && record.key?(@dest_time_name)
+          record[@dest_time_name] = val
+        end
       end
       if ENV['CDM_DEBUG']
         unless tag == ENV['CDM_DEBUG_IGNORE_TAG']

data/test/test_filter_viaq_data_model.rb

@@ -164,6 +164,33 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
       assert_equal('b', rec['c'])
       assert_nil(rec['a'])
     end
+    test 'see if time field is renamed when checking if missing' do
+      rec = emit_with_tag('tag', {'a'=>'b'}, '
+        rename_time_if_missing true
+        src_time_name a
+        dest_time_name c
+      ')
+      assert_equal('b', rec['c'])
+      assert_nil(rec['a'])
+    end
+    test 'see if time field is renamed when already present' do
+      rec = emit_with_tag('tag', {'a'=>'b','c'=>'d'}, '
+        rename_time true
+        src_time_name a
+        dest_time_name c
+      ')
+      assert_equal('b', rec['c'])
+      assert_nil(rec['a'])
+    end
+    test 'see if time field is preserved when already present' do
+      rec = emit_with_tag('tag', {'a'=>'b','c'=>'d'}, '
+        rename_time_if_missing true
+        src_time_name a
+        dest_time_name c
+      ')
+      assert_equal('d', rec['c'])
+      assert_nil(rec['a'])
+    end
     test 'see if deeply nested empty fields are removed or preserved' do
       msg = {'a'=>{'b'=>{'c'=>{'d'=>{'e'=>'','f'=>{},'g'=>''}}}},'h'=>{'i'=>{'j'=>'','k'=>'l','m'=>99,'n'=>true}}}
       rec = emit_with_tag('tag', msg)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-viaq_data_model
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Rich Megginson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-13 00:00:00.000000000 Z
+date: 2017-02-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -116,6 +116,8 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".travis.yml"
+- Gemfile
 - LICENSE
 - README.md
 - Rakefile
@@ -124,7 +126,7 @@ files:
 - fluent-plugin-viaq_data_model.gemspec
 - lib/fluent/plugin/filter_viaq_data_model.rb
 - test/test_filter_viaq_data_model.rb
-homepage: https://github.com/ViaQ/fluent-plugin-viaq_data_mode
+homepage: https://github.com/ViaQ/fluent-plugin-viaq_data_model
 licenses:
 - Apache-2.0
 metadata: {}