fluent-plugin-viaq_data_model 0.0.9 → 0.0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a18e41903af1963acc60e14cfa67a97e1150c2e4
-  data.tar.gz: 16197edf63812241935d2c2d29e806865bfbe7eb
+  metadata.gz: b9e299245761d3debf06c0c4ae48413be713f5be
+  data.tar.gz: aa684c6a42046d65849734c7989e3ab4f5435029
 SHA512:
-  metadata.gz: 67b5c1ac90d3b88aa6e0d7bf27a613fe9480adec00ae22d4e7ab1ecf5287ff292744799dfafb7abdd43165d6fa7d8f0ab4ea4fe93997781e804853261ad541fa
-  data.tar.gz: 8a459c3fa776e944a397a0bb8443ec7d245696f86ba3df9c670d4d9604855142fee65b2061f433d862ad147f3981909a2a65a5ef7578f16327a6606112fa7de5
+  metadata.gz: 150204ed3baf62ed86df87117935753d9f99a94cabcd8f19f75233a5f29ed353f71f0245d93f129c67b2b46c753013edba80a77bf9106d2595065502d7fd66c8
+  data.tar.gz: 8513d7f9af3799646c8e670c45716ab288e6f128af022d78757c36298fbce3d7c62a3702e6c8ea4b0b0b7b33bce352cf030c575d12c41b04db1e2d1f84feb669

data/fluent-plugin-viaq_data_model.gemspec

@@ -7,7 +7,7 @@ FLUENTD_VERSION = ENV['FLUENTD_VERSION'] || "0.12.0"
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-viaq_data_model"
-  gem.version       = "0.0.9"
+  gem.version       = "0.0.11"
   gem.authors       = ["Rich Megginson"]
   gem.email         = ["rmeggins@redhat.com"]
   gem.description   = %q{Filter plugin to ensure data is in the ViaQ common data model}

data/lib/fluent/plugin/filter_viaq_data_model.rb

@@ -140,6 +140,8 @@ module Fluent
     config_param :elasticsearch_index_name_field, :string, default: 'viaq_index_name'
     desc 'Store the Elasticsearch index prefix in this field'
     config_param :elasticsearch_index_prefix_field, :string, default: 'viaq_index_prefix'
+    desc 'Optionally turn off processing of kubernetes events'
+    config_param :process_kubernetes_events, :bool, default: true
 
     def configure(conf)
       super
@@ -259,6 +261,7 @@ module Fluent
         end
         record['time'] = rectime.utc.to_datetime.rfc3339(6)
       end
+      transform_eventrouter(tag, record)
     end
 
     def check_for_match_and_format(tag, time, record)
@@ -286,14 +289,16 @@ module Fluent
     end
 
     def add_pipeline_metadata (tag, time, record)
-      (record['pipeline_metadata'] ||= {})[@pipeline_type.to_s] = {
-        "ipaddr4"     => @ipaddr4,
-        "ipaddr6"     => @ipaddr6,
-        "inputname"   => "fluent-plugin-systemd",
-        "name"        => "fluentd",
-        "received_at" => Time.now.utc.to_datetime.rfc3339(6),
-        "version"     => @pipeline_version
-      }
+      record['pipeline_metadata'] = {} unless record.key?('pipeline_metadata')
+      pipeline_type = @pipeline_type.to_s
+      # this will catch the case where pipeline_type doesn't exist, or is not a Hash
+      record['pipeline_metadata'][pipeline_type] = {} unless record['pipeline_metadata'][pipeline_type].respond_to?(:fetch)
+      record['pipeline_metadata'][pipeline_type]['ipaddr4'] = @ipaddr4
+      record['pipeline_metadata'][pipeline_type]['ipaddr6'] = @ipaddr6
+      record['pipeline_metadata'][pipeline_type]['inputname'] = 'fluent-plugin-systemd'
+      record['pipeline_metadata'][pipeline_type]['name'] = 'fluentd'
+      record['pipeline_metadata'][pipeline_type]['received_at'] = Time.now.utc.to_datetime.rfc3339(6)
+      record['pipeline_metadata'][pipeline_type]['version'] = @pipeline_version
     end
 
     def add_elasticsearch_index_name_field(tag, time, record)
@@ -358,6 +363,22 @@ module Fluent
       end
     end
 
+    def transform_eventrouter(tag, record)
+      return unless @process_kubernetes_events
+      if record.key?("event")
+        if record.key?("verb")
+          record["event"]["verb"] = record.delete("verb")
+        end
+        record["kubernetes"] = {} unless record.key?("kubernetes")
+        record["kubernetes"]["event"] = record.delete("event")
+        if record["kubernetes"]["event"].key?('message')
+          ((record['pipeline_metadata'] ||= {})[@pipeline_type.to_s] ||= {})['original_raw_message'] = record['message']
+        end
+        record['message'] = record["kubernetes"]["event"].delete("message")
+        record['time'] = record["kubernetes"]["event"]["metadata"].delete("creationTimestamp") 
+      end
+    end
+
     def filter(tag, time, record)
       if ENV['CDM_DEBUG']
         unless tag == ENV['CDM_DEBUG_IGNORE_TAG']

data/lib/fluent/plugin/filter_viaq_data_model_systemd.rb

@@ -107,7 +107,8 @@ module ViaqDataModelFilterSystemd
     record['level'] = ["emerg", "alert", "crit", "err", "warning", "notice", "info", "debug", "trace", "unknown"][pri_index]
     JOURNAL_TIME_FIELDS.each do |field|
       if (val = record[field])
-        record['time'] = Time.at(val.to_f / 1000000.0).utc.to_datetime.rfc3339(6)
+        vali = val.to_i
+        record['time'] = Time.at(vali / 1000000, vali % 1000000).utc.to_datetime.rfc3339(6)
         break
       end
     end
@@ -129,6 +130,7 @@ module ViaqDataModelFilterSystemd
       else
         record['hostname'] = record['_HOSTNAME']
       end
+      transform_eventrouter(tag, record)
     end
   end
 end

data/test/test_filter_viaq_data_model.rb

@@ -317,6 +317,12 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
         "UDEV_DEVLINK"     =>"UDEV_DEVLINK"
       }
     end
+    def add_event(input)
+      input = input.merge({})
+      input['verb'] = 'ADDED'
+      input['event'] = {'message'=>'event message','metadata'=>{'creationTimestamp'=>'2017-07-27T17:23:46.216527+00:00'}}
+      input
+    end
     test 'match records with journal_system_record_tag' do
       rec = emit_with_tag('journal.system', {'a'=>'b', 'MESSAGE'=>'here'}, '
         <formatter>
@@ -516,6 +522,72 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
       dellist = 'log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID'.split(',')
       dellist.each{|field| assert_nil(rec[field])}
     end
+    test 'process a kubernetes journal record with event from eventrouter, default settings' do
+      ENV['IPADDR4'] = '127.0.0.1'
+      ENV['IPADDR6'] = '::1'
+      ENV['FLUENTD_VERSION'] = 'fversion'
+      ENV['DATA_VERSION'] = 'dversion'
+      input = add_event(normal_input)
+      rec = emit_with_tag('kubernetes.journal.container', input, '
+        <formatter>
+          tag "kubernetes.journal.container**"
+          type k8s_journal
+          remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
+        </formatter>
+        pipeline_type collector
+      ')
+      assert_equal(normal_output_t, rec['systemd']['t'])
+      assert_equal(normal_output_u, rec['systemd']['u'])
+      assert_equal(normal_output_k, rec['systemd']['k'])
+      assert_equal('ADDED', rec['kubernetes']['event']['verb'])
+      assert_equal('event message', rec['message'])
+      assert_equal('hello world', rec['pipeline_metadata']['collector']['original_raw_message'])
+      assert_equal('info', rec['level'])
+      assert_equal('myhost', rec['hostname'])
+      assert_equal('2017-07-27T17:23:46.216527+00:00', rec['@timestamp'])
+      assert_equal('127.0.0.1', rec['pipeline_metadata']['collector']['ipaddr4'])
+      assert_equal('::1', rec['pipeline_metadata']['collector']['ipaddr6'])
+      assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['collector']['inputname'])
+      assert_equal('fluentd', rec['pipeline_metadata']['collector']['name'])
+      assert_equal('fversion dversion', rec['pipeline_metadata']['collector']['version'])
+      assert_equal(@timestamp_str, rec['pipeline_metadata']['collector']['received_at'])
+      dellist = 'log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID,event,verb'.split(',')
+      dellist.each{|field| assert_nil(rec[field])}
+    end
+    test 'process a kubernetes journal record with event from eventrouter, disable event processing' do
+      ENV['IPADDR4'] = '127.0.0.1'
+      ENV['IPADDR6'] = '::1'
+      ENV['FLUENTD_VERSION'] = 'fversion'
+      ENV['DATA_VERSION'] = 'dversion'
+      input = add_event(normal_input)
+      rec = emit_with_tag('kubernetes.journal.container', input, '
+        <formatter>
+          tag "kubernetes.journal.container**"
+          type k8s_journal
+          remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
+        </formatter>
+        process_kubernetes_events false
+        pipeline_type collector
+      ')
+      assert_equal('ADDED', rec['verb'])
+      assert_equal({'message'=>'event message','metadata'=>{'creationTimestamp'=>'2017-07-27T17:23:46.216527+00:00'}}, rec['event'])
+      assert_equal(normal_output_t, rec['systemd']['t'])
+      assert_equal(normal_output_u, rec['systemd']['u'])
+      assert_equal(normal_output_k, rec['systemd']['k'])
+      assert_equal('hello world', rec['message'])
+      assert_equal('info', rec['level'])
+      assert_equal('myhost', rec['hostname'])
+      assert_equal('2017-07-27T17:27:46.216527+00:00', rec['@timestamp'])
+      assert_equal('127.0.0.1', rec['pipeline_metadata']['collector']['ipaddr4'])
+      assert_equal('::1', rec['pipeline_metadata']['collector']['ipaddr6'])
+      assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['collector']['inputname'])
+      assert_equal('fluentd', rec['pipeline_metadata']['collector']['name'])
+      assert_equal('fversion dversion', rec['pipeline_metadata']['collector']['version'])
+      assert_equal(@timestamp_str, rec['pipeline_metadata']['collector']['received_at'])
+      dellist = 'log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID'.split(',')
+      dellist.each{|field| assert_nil(rec[field])}
+      assert_nil(rec['pipeline_metadata']['collector']['original_raw_message'])
+    end
     test 'disable kubernetes journal record processing' do
       ENV['IPADDR4'] = '127.0.0.1'
       ENV['IPADDR6'] = '::1'
@@ -717,6 +789,35 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
       dellist = 'host,pid,ident'.split(',')
       dellist.each{|field| assert_nil(rec[field])}
     end
+    test 'process a k8s json-file record with event from eventrouter, default settings' do
+      ENV['IPADDR4'] = '127.0.0.1'
+      ENV['IPADDR6'] = '::1'
+      ENV['FLUENTD_VERSION'] = 'fversion'
+      ENV['DATA_VERSION'] = 'dversion'
+      input = {'kubernetes'=>{'host'=>'k8shost'},'stream'=>'stderr','time'=>@timestamp_str,'log'=>'mymessage'}
+      input = add_event(input)
+      rec = emit_with_tag('kubernetes.var.log.containers.name.name_this_that_other_log', input, '
+        <formatter>
+          tag "kubernetes.var.log.containers**"
+          type k8s_json_file
+          remove_keys log,stream
+        </formatter>
+        pipeline_type collector 
+      ')
+      assert_equal('ADDED', rec['kubernetes']['event']['verb'])
+      assert_equal('event message', rec['message'])
+      assert_equal('mymessage', rec['pipeline_metadata']['collector']['original_raw_message'])
+      assert_equal('err', rec['level'])
+      assert_equal('2017-07-27T17:23:46.216527+00:00', rec['@timestamp'])
+      assert_equal('127.0.0.1', rec['pipeline_metadata']['collector']['ipaddr4'])
+      assert_equal('::1', rec['pipeline_metadata']['collector']['ipaddr6'])
+      assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['collector']['inputname'])
+      assert_equal('fluentd', rec['pipeline_metadata']['collector']['name'])
+      assert_equal('fversion dversion', rec['pipeline_metadata']['collector']['version'])
+      assert_equal(@timestamp_str, rec['pipeline_metadata']['collector']['received_at'])
+      dellist = 'host,pid,ident,event,verb'.split(',')
+      dellist.each{|field| assert_nil(rec[field])}
+    end
     test 'process a k8s json-file record with a string valued timestamp' do
       ENV['IPADDR4'] = '127.0.0.1'
       ENV['IPADDR6'] = '::1'
@@ -1126,6 +1227,31 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
         </elasticsearch_index_name>
       ')
       assert_equal('.operations.2017.07.27', rec['viaq_index_name'])
-   end
+    end
+    test 'construct an operations index name from a kubernetes record in an operations namespace' do
+      input = normal_input.merge({})
+      input['kubernetes'] = {'namespace_name'=>'default', 'namespace_id'=>'uuid'}
+      rec = emit_with_tag('kubernetes.journal.container._default_', input, '
+        <formatter>
+          tag "journal.system**"
+          type sys_journal
+          remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
+        </formatter>
+        <formatter>
+          tag "kubernetes.journal.container**"
+          type k8s_journal
+          remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
+        </formatter>
+        <elasticsearch_index_name>
+          tag "journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops"
+          name_type operations_full
+        </elasticsearch_index_name>
+        <elasticsearch_index_name>
+          tag "**"
+          name_type project_full
+        </elasticsearch_index_name>
+      ')
+      assert_equal('.operations.2017.07.27', rec['viaq_index_name'])
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-viaq_data_model
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.0.11
 platform: ruby
 authors:
 - Rich Megginson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-13 00:00:00.000000000 Z
+date: 2017-09-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -147,7 +147,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Filter plugin to ensure data is in the ViaQ common data model