fluent-plugin-viaq_data_model 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 129a8f3798e87df17888fe973bc9215a4fe9c42c
-  data.tar.gz: a22524738b672742e90dd2a626c6126e8b00f97d
+  metadata.gz: '05957e3bf287a122835a57a9c8664d689a618d64'
+  data.tar.gz: 26d5f3dca6e874e40553f1965d096f0669954e41
 SHA512:
-  metadata.gz: 00f295b2a060138b84c041b59785f785cfce8a51f1a6754574c9fb0a1160e85a1ba16757806373f23721781939ad824ae871cfbdd99e6140655b26ac6b0a782c
-  data.tar.gz: 0abc2a05dda5dc57a29c8982176827a7dc3e654415c13a26af7d8e5ad9ec98228a3666fd9a281a59155509c428a97ba6290e8a9d498fc41dd3485b3cf31a4886
+  metadata.gz: c25c5635ac44f09da3b6eacd863b7753a9fa3f1ed460c676d85eb293d80766177f0d7e32edec23598bc54fc980188cca51e0d2a63ff7bbade9c4e92fac06e917
+  data.tar.gz: 8e17b6ed4ea137598c1831f1de606fcfe3f679550204547bf5194f70c2e0f04f438f255e826298addcd9e16aa24de22cb2fb474b2b74c67afe644c68ca154c8f

data/README.md

@@ -96,6 +96,9 @@ See `filter-viaq_data_model.conf` for an example filter configuration.
   * This is the name of the top level field to hold the time value.  The value
   is taken from the value of the `src_time_name` field.
 * `formatter` - a formatter for a well known common data model source
+  * `enabled` - default `true` - is this formatter enabled?  **NOTE** if the
+    formatter is disabled, it will still match, it just won't do anything, and
+    it will skip the other formatters.
   * `type` - one of the well known sources
     * `sys_journal` - a record read from the systemd journal
     * `k8s_journal` - a Kubernetes container record read from the systemd
@@ -109,6 +112,9 @@ See `filter-viaq_data_model.conf` for an example filter configuration.
   `normalizer` - the default is `collector`
 * `elasticsearch_index_name` - how to construct Elasticsearch index names or
   prefixes for given tags
+  * `enabled` - default `true` - is this item enabled?  **NOTE** if the
+    item is disabled, it will still match, it just won't do anything, and
+    it will skip the other index name items.
   * `tag` - the Fluentd tag pattern to match for these records
   * `name_type` - the well known type of index name or prefix to create -
     `operations_full, project_full, operations_prefix, project_prefix` - The
@@ -231,7 +237,6 @@ Given a configuration like this:
       tag "**"
       name_type project_full
     </elasticsearch_index_name>
-    elasticsearch_index_field viaq_index_name
 
 A record with tag `journal.system` like this:
 
@@ -261,6 +266,54 @@ will end up looking like this:
       "viaq_index_name":"project.myproject.000000.2017.07.07"
     }
 
+
+### Note about using enabled false
+
+Given a configuration like this:
+
+    <elasticsearch_index_name>
+      enabled false
+      tag "journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops"
+      name_type operations_full
+    </elasticsearch_index_name>
+    <elasticsearch_index_name>
+      tag "**"
+      name_type project_full
+    </elasticsearch_index_name>
+
+A record with tag `journal.system` like this:
+
+    {
+      "@timestamp":"2017-07-27T17:27:46.216527+00:00"
+    }
+
+will end up looking like this:
+
+    {
+      "@timestamp":"2017-07-27T17:27:46.216527+00:00",
+    }
+
+That is, the tag will match the first `elasticsearch_index_name`, but since it
+is disabled, no index name will be created, and it will _not_ fall through to
+the `**` match below.  Using `enabled false` in this case allows you to not
+generate index names for operations indices, but still continue to generate
+index names for project indices.
+
+A record with tag `kubernetes.journal.container` like this:
+
+    {
+      "@timestamp":"2017-07-27T17:27:46.216527+00:00",
+      "kubernetes":{"namespace_name":"myproject","namespace_id":"000000"}
+    }
+
+will end up looking like this:
+
+    {
+      "@timestamp":"2017-07-27T17:27:46.216527+00:00",
+      "kubernetes":{"namespace_name":"myproject","namespace_id":"000000"}
+      "viaq_index_name":"project.myproject.000000.2017.07.07"
+    }
+
 ## Installation
 
     gem install fluent-plugin-viaq_data_model

data/fluent-plugin-viaq_data_model.gemspec

@@ -7,7 +7,7 @@ FLUENTD_VERSION = ENV['FLUENTD_VERSION'] || "0.12.0"
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-viaq_data_model"
-  gem.version       = "0.0.6"
+  gem.version       = "0.0.7"
   gem.authors       = ["Rich Megginson"]
   gem.email         = ["rmeggins@redhat.com"]
   gem.description   = %q{Filter plugin to ensure data is in the ViaQ common data model}

data/lib/fluent/plugin/filter_viaq_data_model.rb

@@ -99,6 +99,8 @@ module Fluent
     # come before more general matches
     desc 'Formatters for common data model, for well known record types'
     config_section :formatter, param_name: :formatters do
+      desc 'is this formatter enabled?'
+      config_param :enabled, :bool, default: true
       desc 'one of the well known formatter types'
       config_param :type, :enum, list: [:sys_journal, :k8s_journal, :sys_var_log, :k8s_json_file]
       desc 'process records with this tag pattern'
@@ -127,7 +129,11 @@ module Fluent
     # come before more general matches e.g. make sure tag "**" is last
     desc 'Construct Elasticsearch index names or prefixes based on the matching tags pattern and type'
     config_section :elasticsearch_index_name, param_name: :elasticsearch_index_names do
+      desc 'is this index name enabled?'
+      config_param :enabled, :bool, default: true
+      desc 'create index names for records with this tag pattern'
       config_param :tag, :string
+      desc 'type of index name to create'
       config_param :name_type, :enum, list: [:operations_full, :project_full, :operations_prefix, :project_prefix]
     end
     desc 'Store the Elasticsearch index name in this field'
@@ -219,12 +225,14 @@ module Fluent
 
     def process_sys_var_log_fields(tag, time, record, fmtr_type = nil)
       record['systemd'] = {"t" => {"PID" => record['pid']}, "u" => {"SYSLOG_IDENTIFIER" => record['ident']}}
-      rectime = record['time'] || time
-      # handle the case where the time reported in /var/log/messages is for a previous year
-      if Time.at(rectime) > Time.now
-        record['time'] = Time.new((rectime.year - 1), rectime.month, rectime.day, rectime.hour, rectime.min, rectime.sec, rectime.utc_offset).utc.to_datetime.rfc3339(6)
-      else
-        record['time'] = rectime.utc.to_datetime.rfc3339(6)
+      unless record[@dest_time_name] # e.g. already has @timestamp
+        rectime = record['time'] || time
+        # handle the case where the time reported in /var/log/messages is for a previous year
+        if Time.at(rectime) > Time.now
+          record['time'] = Time.new((rectime.year - 1), rectime.month, rectime.day, rectime.hour, rectime.min, rectime.sec, rectime.utc_offset).utc.to_datetime.rfc3339(6)
+        else
+          record['time'] = rectime.utc.to_datetime.rfc3339(6)
+        end
       end
       if record['host'].eql?('localhost') && @docker_hostname
         record['hostname'] = @docker_hostname
@@ -241,7 +249,9 @@ module Fluent
       elsif @docker_hostname
         record['hostname'] = @docker_hostname
       end
-      record['time'] = record['time'].utc.to_datetime.rfc3339(6)
+      unless record[@dest_time_name] # e.g. already has @timestamp
+        record['time'] = record['time'].utc.to_datetime.rfc3339(6)
+      end
     end
 
     def check_for_match_and_format(tag, time, record)
@@ -250,8 +260,7 @@ module Fluent
       fmtr = @formatter_cache[tag]
       unless fmtr
         idx = @formatters.index{|fmtr| fmtr.matcher.match(tag)}
-        if idx
-          fmtr = @formatters[idx]
+        if idx && (fmtr = @formatters[idx]).enabled
           @formatter_cache[tag] = fmtr
         else
           @formatter_cache_nomatch[tag] = true
@@ -260,7 +269,7 @@ module Fluent
       end
       fmtr.fmtr_func.call(tag, time, record, fmtr.fmtr_type)
 
-      if record['time'].nil?
+      if record[@dest_time_name].nil? && record['time'].nil?
         record['time'] = Time.at(time).utc.to_datetime.rfc3339(6)
       end
 
@@ -275,7 +284,7 @@ module Fluent
         "ipaddr6"     => @ipaddr6,
         "inputname"   => "fluent-plugin-systemd",
         "name"        => "fluentd",
-        "received_at" => Time.at(time).utc.to_datetime.rfc3339(6),
+        "received_at" => Time.now.utc.to_datetime.rfc3339(6),
         "version"     => @pipeline_version
       }
     end
@@ -285,6 +294,7 @@ module Fluent
       @elasticsearch_index_names.each do |ein|
         if ein.matcher.match(tag)
           found = true
+          return unless ein.enabled
           if ein.name_type == :operations_full || ein.name_type == :project_full
             field_name = @elasticsearch_index_name_field
             need_time = true
@@ -333,7 +343,11 @@ module Fluent
         end
       end
       unless found
-        log.warn("no match for tag #{tag}")
+        if ENV['CDM_DEBUG']
+          unless tag == ENV['CDM_DEBUG_IGNORE_TAG']
+            log.error("no match for tag #{tag}")
+          end
+        end
       end
     end
 

data/test/test_filter_viaq_data_model.rb

@@ -18,6 +18,7 @@
 #
 #require_relative '../helper'
 require 'fluent/test'
+require 'flexmock/test_unit'
 
 require 'fluent/plugin/filter_viaq_data_model'
 
@@ -28,6 +29,9 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
     Fluent::Test.setup
     @time = Fluent::Engine.now
     log = Fluent::Engine.log
+    @timestamp = Time.now
+    @timestamp_str = @timestamp.utc.to_datetime.rfc3339(6)
+    flexmock(Time).should_receive(:now).and_return(@timestamp)
   end
 
   def create_driver(conf = '')
@@ -368,10 +372,28 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
       assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
       assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
       assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
-      assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['pipeline_metadata']['normalizer']['received_at'])
+      assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
       dellist = 'log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID'.split(',')
       dellist.each{|field| assert_nil(rec[field])}
     end
+    test 'disable journal record processing' do
+      ENV['IPADDR4'] = '127.0.0.1'
+      ENV['IPADDR6'] = '::1'
+      ENV['FLUENTD_VERSION'] = 'fversion'
+      ENV['DATA_VERSION'] = 'dversion'
+      rec = emit_with_tag('journal.system', normal_input, '
+        <formatter>
+          enabled false
+          tag "journal.system**"
+          type sys_journal
+          remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
+        </formatter>
+        pipeline_type normalizer
+      ')
+      assert_nil(rec['systemd'])
+      notdellist = 'log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID'.split(',')
+      notdellist.each{|field| assert_equal(normal_input[field], rec[field])}
+    end
     test 'process a journal record, override remove_keys' do
       ENV['IPADDR4'] = '127.0.0.1'
       ENV['IPADDR6'] = '::1'
@@ -397,7 +419,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
       assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
       assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
       assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
-      assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['pipeline_metadata']['normalizer']['received_at'])
+      assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
       keeplist = 'log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID'.split(',')
       keeplist.each{|field| normal_input[field] && assert_not_nil(rec[field])}
       dellist = 'CONTAINER_NAME,PRIORITY'.split(',')
@@ -490,10 +512,28 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
       assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
       assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
       assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
-      assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['pipeline_metadata']['normalizer']['received_at'])
+      assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
       dellist = 'log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID'.split(',')
       dellist.each{|field| assert_nil(rec[field])}
     end
+    test 'disable kubernetes journal record processing' do
+      ENV['IPADDR4'] = '127.0.0.1'
+      ENV['IPADDR6'] = '::1'
+      ENV['FLUENTD_VERSION'] = 'fversion'
+      ENV['DATA_VERSION'] = 'dversion'
+      rec = emit_with_tag('kubernetes.journal.container', normal_input, '
+        <formatter>
+          enabled false
+          tag "kubernetes.journal.container**"
+          type k8s_journal
+          remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
+        </formatter>
+        pipeline_type normalizer
+      ')
+      assert_nil(rec['systemd'])
+      notdellist = 'log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID'.split(',')
+      notdellist.each{|field| assert_equal(normal_input[field], rec[field])}
+    end
     test 'process a kubernetes journal record, given kubernetes.host' do
       input = normal_input.merge({})
       input['kubernetes'] = {'host' => 'k8shost'}
@@ -521,7 +561,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
       assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
       assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
       assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
-      assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['pipeline_metadata']['normalizer']['received_at'])
+      assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
       dellist = 'log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID'.split(',')
       dellist.each{|field| assert_nil(rec[field])}
     end
@@ -552,7 +592,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
       assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
       assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
       assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
-      assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['pipeline_metadata']['normalizer']['received_at'])
+      assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
       dellist = 'log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID'.split(',')
       dellist.each{|field| assert_nil(rec[field])}
     end
@@ -581,7 +621,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
       assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
       assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
       assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
-      assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['pipeline_metadata']['normalizer']['received_at'])
+      assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
       dellist = 'host,pid,ident'.split(',')
       dellist.each{|field| assert_nil(rec[field])}
     end
@@ -612,7 +652,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
       assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
       assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
       assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
-      assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['pipeline_metadata']['normalizer']['received_at'])
+      assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
       dellist = 'host,pid,ident'.split(',')
       dellist.each{|field| assert_nil(rec[field])}
     end
@@ -640,7 +680,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
       assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
       assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
       assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
-      assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['pipeline_metadata']['normalizer']['received_at'])
+      assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
       dellist = 'host,pid,ident'.split(',')
       dellist.each{|field| assert_nil(rec[field])}
     end
@@ -716,6 +756,30 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
       ')
       assert_equal('.operations.2017.07.27', rec['my_index_name'])
     end
+    test 'disable operations index name' do
+      rec = emit_with_tag('journal.system', normal_input, '
+        <formatter>
+          tag "journal.system**"
+          type sys_journal
+          remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
+        </formatter>
+        <formatter>
+          tag "kubernetes.journal.container**"
+          type k8s_journal
+          remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
+        </formatter>
+        <elasticsearch_index_name>
+          enabled false
+          tag "journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops"
+          name_type operations_full
+        </elasticsearch_index_name>
+        <elasticsearch_index_name>
+          tag "**"
+          name_type project_full
+        </elasticsearch_index_name>
+      ')
+      assert_nil(rec['viaq_index_name'])
+    end
     test 'log error if missing kubernetes field' do
       rec = emit_with_tag('kubernetes.journal.container.something', normal_input, '
         <formatter>
@@ -866,5 +930,53 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
       ')
       assert_equal('project.name.uuid.2017.07.27', rec['my_index_name'])
     end
+    test 'disable kubernetes index names but allow operations index names' do
+      input = normal_input.merge({})
+      input['kubernetes'] = {'namespace_name'=>'name', 'namespace_id'=>'uuid'}
+      rec = emit_with_tag('kubernetes.journal.container.something', input, '
+        <formatter>
+          tag "journal.system**"
+          type sys_journal
+          remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
+        </formatter>
+        <formatter>
+          tag "kubernetes.journal.container**"
+          type k8s_journal
+          remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
+        </formatter>
+        <elasticsearch_index_name>
+          tag "journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops"
+          name_type operations_full
+        </elasticsearch_index_name>
+        <elasticsearch_index_name>
+          enabled false
+          tag "**"
+          name_type project_full
+        </elasticsearch_index_name>
+      ')
+      assert_nil(rec['viaq_index_name'])
+      rec = emit_with_tag('journal.system.something', normal_input, '
+        <formatter>
+          tag "journal.system**"
+          type sys_journal
+          remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
+        </formatter>
+        <formatter>
+          tag "kubernetes.journal.container**"
+          type k8s_journal
+          remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
+        </formatter>
+        <elasticsearch_index_name>
+          tag "journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops"
+          name_type operations_full
+        </elasticsearch_index_name>
+        <elasticsearch_index_name>
+          enabled false
+          tag "**"
+          name_type project_full
+        </elasticsearch_index_name>
+      ')
+      assert_equal('.operations.2017.07.27', rec['viaq_index_name'])
+   end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-viaq_data_model
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - Rich Megginson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-29 00:00:00.000000000 Z
+date: 2017-09-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd