fluent-plugin-viaq_data_model 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 72e1031bd59c7056bff5416a8877658c4850d7fb
-  data.tar.gz: e9e76b523fca3e2c3eb00800a7e00256adc90db3
+  metadata.gz: 129a8f3798e87df17888fe973bc9215a4fe9c42c
+  data.tar.gz: a22524738b672742e90dd2a626c6126e8b00f97d
 SHA512:
-  metadata.gz: 9cc93bc210c483b56dd05ce87efd6b9f54dc067c619dbe90d087002f10b01729b3ca21ac7c80024ec07b88aa852609590b1f5b83479287fb0228d2bcc0438801
-  data.tar.gz: 6a398beddcc833b48ba06bec215b70cd415a5b8d661df7d0fa4b7d832e9ecc23f2d558623b9817055956a90874d896e55ed966f3b8a6f1040b47f7e8540084af
+  metadata.gz: 00f295b2a060138b84c041b59785f785cfce8a51f1a6754574c9fb0a1160e85a1ba16757806373f23721781939ad824ae871cfbdd99e6140655b26ac6b0a782c
+  data.tar.gz: 0abc2a05dda5dc57a29c8982176827a7dc3e654415c13a26af7d8e5ad9ec98228a3666fd9a281a59155509c428a97ba6290e8a9d498fc41dd3485b3cf31a4886

data/.travis.yml

@@ -6,7 +6,11 @@ rvm:
   - 2.3.1
 
 gemfile:
- - Gemfile
+  - Gemfile
+
+env:
+  - FLUENTD_VERSION=0.12.0
+  - FLUENTD_VERSION=0.14.0
 
 script: bundle exec rake test
 sudo: false

data/README.md

@@ -29,6 +29,31 @@ You cannot set the `@timestamp` field in a Fluentd `record_transformer` filter.
 The plugin allows you to use some other field e.g. `time` and have that "moved"
 to a top level field called `@timestamp`.
 
+* Converts systemd and json-file logs to ViaQ data model format
+
+Doing this conversion in a `record_transformer` with embedded ruby code is very
+resource intensive.  The ViaQ plugin can convert common input formats such as
+Kubernetes `json-file`, `/var/log/messages`, and systemd `journald` into their
+corresponding ViaQ `_default_`, `systemd`, `kubernetes`, and
+`pipeline_metadata` namespaced fields.  The `pipeline_metadata` will be added
+to all records, regardless of tag.  Use the `pipeline_type` parameter to
+specify which part of the pipeline this is, `collector` or `normalizer`.
+The ViaQ data model conversion will only be applied to matching `tag`s
+specified in a `formatter` section.
+
+* Creates Elasticsearch index names or prefixes
+
+You can create either a full Elasticsearch index name for the record (to be
+used with the `fluent-plugin-elasticsearch` `target_index_key` parameter), or
+create an index name prefix (missing the date/timestamp part of the index
+name - to be used with `logstash_prefix_key`).  In order to use this, create an
+`elasticsearch_index_name` section, and specify the `tag` to match, and the
+`name_type` type of index name to create.  By default, a prefix name will be
+stored in the `viaq_index_prefix` field in the record, and a full name will be
+stored in the `viaq_index_name` field.  Configure
+`elasticsearch_index_name_field` or `elasticsearch_index_prefix_field` to use a
+different field name.
+
 ## Configuration
 
 NOTE: All fields are Optional - no required fields.
@@ -70,6 +95,49 @@ See `filter-viaq_data_model.conf` for an example filter configuration.
 * `dest_time_name` - string - default `@timestamp`
   * This is the name of the top level field to hold the time value.  The value
   is taken from the value of the `src_time_name` field.
+* `formatter` - a formatter for a well known common data model source
+  * `type` - one of the well known sources
+    * `sys_journal` - a record read from the systemd journal
+    * `k8s_journal` - a Kubernetes container record read from the systemd
+      journal - should have `CONTAINER_NAME`, `CONTAINER_ID_FULL`
+    * `sys_var_log` - a record read from `/var/log/messages`
+    * `k8s_json_file` - a record read from a `/var/log/containers/*.log` JSON
+      formatted container log file
+    * `tag` - the Fluentd tag pattern to match for these records
+    * `remove_keys` - comma delimited list of keys to remove from the record
+* `pipeline_type` - which part of the pipeline is this? `collector` or
+  `normalizer` - the default is `collector`
+* `elasticsearch_index_name` - how to construct Elasticsearch index names or
+  prefixes for given tags
+  * `tag` - the Fluentd tag pattern to match for these records
+  * `name_type` - the well known type of index name or prefix to create -
+    `operations_full, project_full, operations_prefix, project_prefix` - The
+    `operations_*` types will create a name like `.operations`, and the
+    `project_*` types will create a name like
+    `project.record['kubernetes']['namespace_name'].record['kubernetes']['namespace_id']`.
+    When using the `full` types, a delimiter `.` followed by the date in
+    `YYYY.MM.DD` format will be added to the string to make a full index name.
+    When using the `prefix` types, it is assumed that the
+    `fluent-plugin-elaticsearch` is used with the `logstash_prefix_key` to
+    create the full index name.
+* `elasticsearch_index_name_field` - name of the field in the record which stores
+  the index name - you should remove this field in the elasticsearch output
+  plugin using the `remove_keys` config parameter - default is `viaq_idnex_name`
+* `elasticsearch_index_prefix_field` - name of the field in the record which stores
+  the index prefix - you should remove this field in the elasticsearch output
+  plugin using the `remove_keys` config parameter - default is `viaq_idnex_prefix`
+
+**NOTE** The `formatter` blocks are matched in the given order in the file.
+  This means, don't use `tag "**"` as the first formatter or none of your
+  others will be matched or evaulated.
+
+**NOTE** The `elasticsearch_index_name` processing is done *last*, *after* the
+  formatting, removal of empty fields, `@timestamp` creation, etc., so use
+  e.g. `record['systemd']['t']['GID']` instead of `record['_GID']`
+
+**NOTE** The `elasticsearch_index_name` blocks are matched in the given order
+  in the file.  This means, don't use `tag "**"` as the first formatter or none
+  of your others will be matched or evaulated.
 
 ## Example
 
@@ -103,6 +171,95 @@ The resulting record, using the defaults, would look like this:
       "@timestamp": "2017-02-13 15:30:10.259106596-07:00"
     }
 
+## Formatter example
+
+Given a record like the following with a tag of `journal.system`
+
+    __REALTIME_TIMESTAMP=1502228121310282
+    __MONOTONIC_TIMESTAMP=722903835100
+    _BOOT_ID=d85e8a9d524c4a419bcfb6598db78524
+    _TRANSPORT=syslog
+    PRIORITY=6
+    SYSLOG_FACILITY=3
+    SYSLOG_IDENTIFIER=dnsmasq-dhcp
+    SYSLOG_PID=2289
+    _PID=2289
+    _UID=99
+    _GID=40
+    _COMM=dnsmasq
+    _EXE=/usr/sbin/dnsmasq
+    _CMDLINE=/sbin/dnsmasq --conf-file=/var/lib/libvirt/dnsmasq/default.conf --leasefile-ro --dhcp-script=/usr/libexec/libvirt_leaseshelper
+    _CAP_EFFECTIVE=3400
+    _SYSTEMD_CGROUP=/system.slice/libvirtd.service
+    MESSASGE=my message
+
+Using a configuration like this:
+
+    <formatter>
+      tag "journal.system**"
+      type sys_journal
+      remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
+    </formatter>
+
+The resulting record will look like this:
+
+    {
+    "systemd": {
+      "t": {
+        "BOOT_ID":"d85e8a9d524c4a419bcfb6598db78524",
+        "GID":40,
+        ...
+      },
+      "u": {
+        "SYSLOG_FACILITY":3,
+        "SYSLOG_IDENTIFIER":"dnsmasq-dhcp",
+        ...
+      },
+    "message":"my message",
+    ...
+    }
+
+## Elasticsearch index name example
+
+Given a configuration like this:
+
+    <elasticsearch_index_name>
+      tag "journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops"
+      name_type operations_full
+    </elasticsearch_index_name>
+    <elasticsearch_index_name>
+      tag "**"
+      name_type project_full
+    </elasticsearch_index_name>
+    elasticsearch_index_field viaq_index_name
+
+A record with tag `journal.system` like this:
+
+    {
+      "@timestamp":"2017-07-27T17:27:46.216527+00:00"
+    }
+
+will end up looking like this:
+
+    {
+      "@timestamp":"2017-07-27T17:27:46.216527+00:00",
+      "viaq_index_name":".operations.2017.07.07"
+    }
+
+A record with tag `kubernetes.journal.container` like this:
+
+    {
+      "@timestamp":"2017-07-27T17:27:46.216527+00:00",
+      "kubernetes":{"namespace_name":"myproject","namespace_id":"000000"}
+    }
+
+will end up looking like this:
+
+    {
+      "@timestamp":"2017-07-27T17:27:46.216527+00:00",
+      "kubernetes":{"namespace_name":"myproject","namespace_id":"000000"}
+      "viaq_index_name":"project.myproject.000000.2017.07.07"
+    }
 
 ## Installation
 

data/fluent-plugin-viaq_data_model.gemspec

@@ -2,9 +2,12 @@
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
+# can override for testing
+FLUENTD_VERSION = ENV['FLUENTD_VERSION'] || "0.12.0"
+
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-viaq_data_model"
-  gem.version       = "0.0.5"
+  gem.version       = "0.0.6"
   gem.authors       = ["Rich Megginson"]
   gem.email         = ["rmeggins@redhat.com"]
   gem.description   = %q{Filter plugin to ensure data is in the ViaQ common data model}
@@ -20,12 +23,13 @@ Gem::Specification.new do |gem|
 
   gem.required_ruby_version = '>= 2.0.0'
 
-  gem.add_runtime_dependency "fluentd", ">= 0.12.0"
+  gem.add_runtime_dependency "fluentd", "~> #{FLUENTD_VERSION}"
 
   gem.add_development_dependency "bundler"
-  gem.add_development_dependency("fluentd", ">= 0.12.0")
+  gem.add_development_dependency("fluentd", "~> #{FLUENTD_VERSION}")
   gem.add_development_dependency("rake", ["~> 11.0"])
   gem.add_development_dependency("rr", ["~> 1.0"])
   gem.add_development_dependency("test-unit", ["~> 3.2"])
   gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
+  gem.add_development_dependency("flexmock", ["~> 2.0"])
 end

data/lib/fluent/plugin/filter_viaq_data_model.rb

@@ -15,11 +15,42 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+require 'time'
+require 'date'
+
 require 'fluent/filter'
 require 'fluent/log'
+require 'fluent/match'
+
+require_relative 'filter_viaq_data_model_systemd'
+
+begin
+  ViaqMatchClass = Fluent::Match
+rescue
+  # Fluent::Match not provided with 0.14
+  class ViaqMatchClass
+    def initialize(pattern_str, unused)
+      patterns = pattern_str.split(/\s+/).map {|str|
+        Fluent::MatchPattern.create(str)
+      }
+      if patterns.length == 1
+        @pattern = patterns[0]
+      else
+        @pattern = Fluent::OrMatchPattern.new(patterns)
+      end
+    end
+    def match(tag)
+      @pattern.match(tag)
+    end
+    def to_s
+      "#{@pattern}"
+    end
+  end
+end
 
 module Fluent
   class ViaqDataModelFilter < Filter
+    include ViaqDataModelFilterSystemd
     Fluent::Plugin.register_filter('viaq_data_model', self)
 
     desc 'Default list of comma-delimited fields to keep in each record'
@@ -59,6 +90,50 @@ module Fluent
     desc 'Name of destination timestamp field'
     config_param :dest_time_name, :string, default: '@timestamp'
 
+    # <formatter>
+    #   type sys_journal
+    #   tag "journal.system**"
+    #   remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
+    # </formatter>
+    # formatters will be processed in the order specified, so make sure more specific matches
+    # come before more general matches
+    desc 'Formatters for common data model, for well known record types'
+    config_section :formatter, param_name: :formatters do
+      desc 'one of the well known formatter types'
+      config_param :type, :enum, list: [:sys_journal, :k8s_journal, :sys_var_log, :k8s_json_file]
+      desc 'process records with this tag pattern'
+      config_param :tag, :string
+      desc 'remove these keys from the record - same as record_transformer "remove_keys" field'
+      config_param :remove_keys, :string, default: nil
+    end
+
+    desc 'Which part of the pipeline is this - collector, normalizer, etc. for pipeline_metadata'
+    config_param :pipeline_type, :enum, list: [:collector, :normalizer], default: :collector
+
+    # e.g.
+    # <elasticsearch_index_name>
+    #   tag "journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops"
+    #   name_type operations_full
+    # </elasticsearch_index_name>
+    # <elasticsearch_index_name>
+    #   tag "**"
+    #   name_type project_full
+    # </elasticsearch_index_name>
+    # operations_full - ".operations.YYYY.MM.DD"
+    # operations_prefix - ".operations"
+    # project_full - "project.${kubernetes.namespace_name}.${kubernetes.namespace_id}.YYYY.MM.DD"
+    # project_prefix - "project.${kubernetes.namespace_name}.${kubernetes.namespace_id}"
+    # index names will be processed in the order specified, so make sure more specific matches
+    # come before more general matches e.g. make sure tag "**" is last
+    desc 'Construct Elasticsearch index names or prefixes based on the matching tags pattern and type'
+    config_section :elasticsearch_index_name, param_name: :elasticsearch_index_names do
+      config_param :tag, :string
+      config_param :name_type, :enum, list: [:operations_full, :project_full, :operations_prefix, :project_prefix]
+    end
+    desc 'Store the Elasticsearch index name in this field'
+    config_param :elasticsearch_index_name_field, :string, default: 'viaq_index_name'
+    desc 'Store the Elasticsearch index prefix in this field'
+    config_param :elasticsearch_index_prefix_field, :string, default: 'viaq_index_prefix'
 
     def configure(conf)
       super
@@ -76,6 +151,44 @@ module Fluent
       if (@rename_time || @rename_time_if_not_exist) && @use_undefined && !@keep_fields.key?(@src_time_name)
         raise Fluent::ConfigError, "Field [#{@src_time_name}] must be listed in default_keep_fields or extra_keep_fields"
       end
+      if @formatters
+        @formatters.each do |fmtr|
+          matcher = ViaqMatchClass.new(fmtr.tag, nil)
+          fmtr.instance_eval{ @params[:matcher] = matcher }
+          fmtr.instance_eval{ @params[:fmtr_type] = fmtr.type }
+          if fmtr.remove_keys
+            fmtr.instance_eval{ @params[:fmtr_remove_keys] = fmtr.remove_keys.split(',') }
+          else
+            fmtr.instance_eval{ @params[:fmtr_remove_keys] = nil }
+          end
+          case fmtr.type
+          when :sys_journal, :k8s_journal
+            fmtr_func = method(:process_journal_fields)
+          when :sys_var_log
+            fmtr_func = method(:process_sys_var_log_fields)
+          when :k8s_json_file
+            fmtr_func = method(:process_k8s_json_file_fields)
+          end
+          fmtr.instance_eval{ @params[:fmtr_func] = fmtr_func }
+        end
+        @formatter_cache = {}
+        @formatter_cache_nomatch = {}
+      end
+      begin
+        @docker_hostname = File.open('/etc/docker-hostname') { |f| f.readline }.rstrip
+      rescue
+        @docker_hostname = nil
+      end
+      @ipaddr4 = ENV['IPADDR4'] || '127.0.0.1'
+      @ipaddr6 = ENV['IPADDR6'] || '::1'
+      @pipeline_version = (ENV['FLUENTD_VERSION'] || 'unknown fluentd version') + ' ' + (ENV['DATA_VERSION'] || 'unknown data version')
+      # create the elasticsearch index name tag matchers
+      unless @elasticsearch_index_names.empty?
+        @elasticsearch_index_names.each do |ein|
+          matcher = ViaqMatchClass.new(ein.tag, nil)
+          ein.instance_eval{ @params[:matcher] = matcher }
+        end
+      end
     end
 
     def start
@@ -104,12 +217,135 @@ module Fluent
       thing
     end
 
+    def process_sys_var_log_fields(tag, time, record, fmtr_type = nil)
+      record['systemd'] = {"t" => {"PID" => record['pid']}, "u" => {"SYSLOG_IDENTIFIER" => record['ident']}}
+      rectime = record['time'] || time
+      # handle the case where the time reported in /var/log/messages is for a previous year
+      if Time.at(rectime) > Time.now
+        record['time'] = Time.new((rectime.year - 1), rectime.month, rectime.day, rectime.hour, rectime.min, rectime.sec, rectime.utc_offset).utc.to_datetime.rfc3339(6)
+      else
+        record['time'] = rectime.utc.to_datetime.rfc3339(6)
+      end
+      if record['host'].eql?('localhost') && @docker_hostname
+        record['hostname'] = @docker_hostname
+      else
+        record['hostname'] = record['host']
+      end
+    end
+
+    def process_k8s_json_file_fields(tag, time, record, fmtr_type = nil)
+      record['message'] = record['message'] || record['log']
+      record['level'] = (record['stream'] == 'stdout') ? 'info' : 'err'
+      if record['kubernetes'] && record['kubernetes']['host']
+        record['hostname'] = record['kubernetes']['host']
+      elsif @docker_hostname
+        record['hostname'] = @docker_hostname
+      end
+      record['time'] = record['time'].utc.to_datetime.rfc3339(6)
+    end
+
+    def check_for_match_and_format(tag, time, record)
+      return unless @formatters
+      return if @formatter_cache_nomatch[tag]
+      fmtr = @formatter_cache[tag]
+      unless fmtr
+        idx = @formatters.index{|fmtr| fmtr.matcher.match(tag)}
+        if idx
+          fmtr = @formatters[idx]
+          @formatter_cache[tag] = fmtr
+        else
+          @formatter_cache_nomatch[tag] = true
+          return
+        end
+      end
+      fmtr.fmtr_func.call(tag, time, record, fmtr.fmtr_type)
+
+      if record['time'].nil?
+        record['time'] = Time.at(time).utc.to_datetime.rfc3339(6)
+      end
+
+      if fmtr.fmtr_remove_keys
+        fmtr.fmtr_remove_keys.each{|k| record.delete(k)}
+      end
+    end
+
+    def add_pipeline_metadata (tag, time, record)
+      (record['pipeline_metadata'] ||= {})[@pipeline_type.to_s] = {
+        "ipaddr4"     => @ipaddr4,
+        "ipaddr6"     => @ipaddr6,
+        "inputname"   => "fluent-plugin-systemd",
+        "name"        => "fluentd",
+        "received_at" => Time.at(time).utc.to_datetime.rfc3339(6),
+        "version"     => @pipeline_version
+      }
+    end
+
+    def add_elasticsearch_index_name_field(tag, time, record)
+      found = false
+      @elasticsearch_index_names.each do |ein|
+        if ein.matcher.match(tag)
+          found = true
+          if ein.name_type == :operations_full || ein.name_type == :project_full
+            field_name = @elasticsearch_index_name_field
+            need_time = true
+          else
+            field_name = @elasticsearch_index_prefix_field
+            need_time = false
+          end
+
+          case ein.name_type
+          when :operations_full, :operations_prefix
+            prefix = ".operations"
+          when :project_full, :project_prefix
+            if (k8s = record['kubernetes']).nil?
+              log.error("record cannot use elasticsearch index name type #{ein.name_type}: record is missing kubernetes field: #{record}")
+              break
+            elsif (name = k8s['namespace_name']).nil?
+              log.error("record cannot use elasticsearch index name type #{ein.name_type}: record is missing kubernetes.namespace_name field: #{record}")
+              break
+            elsif (uuid = k8s['namespace_id']).nil?
+              log.error("record cannot use elasticsearch index name type #{ein.name_type}: record is missing kubernetes.namespace_id field: #{record}")
+              break
+            else
+              prefix = "project." + name + "." + uuid
+            end
+          end
+
+          if ENV['CDM_DEBUG']
+            unless tag == ENV['CDM_DEBUG_IGNORE_TAG']
+              log.error("prefix #{prefix} need_time #{need_time} time #{record[@dest_time_name]}")
+            end
+          end
+
+          if need_time
+            ts = DateTime.parse(record[@dest_time_name])
+            record[field_name] = prefix + "." + ts.strftime("%Y.%m.%d")
+          else
+            record[field_name] = prefix
+          end
+          if ENV['CDM_DEBUG']
+            unless tag == ENV['CDM_DEBUG_IGNORE_TAG']
+              log.error("record[#{field_name}] = #{record[field_name]}")
+            end
+          end
+
+          break
+        end
+      end
+      unless found
+        log.warn("no match for tag #{tag}")
+      end
+    end
+
     def filter(tag, time, record)
       if ENV['CDM_DEBUG']
         unless tag == ENV['CDM_DEBUG_IGNORE_TAG']
           log.error("input #{time} #{tag} #{record}")
         end
       end
+
+      check_for_match_and_format(tag, time, record)
+      add_pipeline_metadata(tag, time, record)
       if @use_undefined
         # undefined contains all of the fields not in keep_fields
         undefined = record.reject{|k,v| @keep_fields.key?(k)}
@@ -132,6 +368,14 @@ module Fluent
           record[@dest_time_name] = val
         end
       end
+
+      if !@elasticsearch_index_names.empty?
+        add_elasticsearch_index_name_field(tag, time, record)
+      elsif ENV['CDM_DEBUG']
+        unless tag == ENV['CDM_DEBUG_IGNORE_TAG']
+          log.error("not adding elasticsearch index name or prefix")
+        end
+      end
       if ENV['CDM_DEBUG']
         unless tag == ENV['CDM_DEBUG_IGNORE_TAG']
           log.error("output #{time} #{tag} #{record}")