fluent-plugin-mutate_filter 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 289f196651f835bc0e7ea5dab50346802c25c1ed
+  data.tar.gz: 3dda295dc953b5ff46804564e689618f29c75d59
+SHA512:
+  metadata.gz: ffa1c0787fc79a87f7d39cff9af9064b0a7d4c35d5c9b21057ee54147da95169c00f8231d7399a41866ffde2db5b60979627880e9e897cc94a9b7cfb25accfdb
+  data.tar.gz: 9f4231a628362dbb30d6c60a3d639bd5141ddb69c6f744965dad045b5e5c87109090d90c1965f6f98d784c29cfe391862e70671def104d141bcb4d839360dc8e

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.gem

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+# Changelog
+
+v0.1.0
+-------------
+* Initial release

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-mutate_filter.gemspec
+gemspec

data/README.md

@@ -0,0 +1,33 @@
+# Fluent::MutateFilter
+
+This gem provides the `mutate` filter for Fluentd which is designed to replicate the way `mutate` works in Logstash as much as possible. 
+
+To be honest, this is a translation of [logstash-filter-mutate](https://github.com/logstash-plugins/logstash-filter-mutate) bordering on copy-paste.
+
+## Requirements
+
+* Fluentd v0.12+
+
+## Installation
+
+```bash
+gem install fluent-plugin-mutate_filter
+```
+
+## Configuration Options
+
+All of the documentation and potential options are documented in the config_params section of the filter module. Below, only a subset of the options are displayed.
+
+```
+<filter *>
+  @type     mutate
+  rename {
+    "old_field_name": "new_field_name",
+    "old_nest.field_name": "new_nest.field_name"
+  }
+  replace {
+    "new_nest.field_name": "%{old_nest.field_name}"
+  }
+</filter>
+```
+

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "fluent/plugin/mutate_filter"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/fluent-plugin-mutate_filter.gemspec

@@ -0,0 +1,18 @@
+# coding: utf-8
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-mutate_filter"
+  spec.version       = "0.1.0"
+  spec.authors       = ["Jonathan Serafini"]
+  spec.email         = ["jonathan@serafini.ca"]
+  spec.summary       = %q{A mutate filter for Fluent which functions like Logstash.}
+  spec.description   = spec.description
+  spec.homepage      = "http://fuck.off"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.executables   = []
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "fluentd", [">= 0.12.0", "< 0.15.0"]
+  spec.add_development_dependency "bundler", "~> 1.11"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/lib/fluent/plugin/filter_mutate.rb

@@ -0,0 +1,453 @@
+require 'fluent/filter'
+require 'fluent/plugin_mixin/mutate_event'
+
+module Fluent
+  class MutateFilter < Filter
+    Fluent::Plugin.register_filter('mutate', self)
+
+    # Treat periods as nested field names
+    config_param :expand_nesting, :bool, default: true
+
+    # Remove any empty hashes or arrays
+    config_param :prune_empty,    :bool, default: true
+
+    # Rename one or more fields
+    # @example
+    #   rename {
+    #     "timestamp": "@timestamp"
+    #   }
+    config_param :rename,     :hash,  default: Hash.new
+
+    # Update an existing field with a new value.
+    # - If the field does not exist then no action will be taken.
+    # - If the new value contains a placeholder %{}, then the value will be 
+    #   expanded to the related event record field. 
+    # @example
+    #   update {
+    #     "message": "%{hostname}: new message"
+    #   }
+    config_param :update,     :hash,  default: Hash.new
+
+    # Remove an existing field
+    # @example
+    #   remove [
+    #     "dummy1", "placeholder1"
+    #   ]
+    config_param :remove,     :array, default: Array.new
+
+    # Replace a field with a new value
+    # - If the field does not exist, then it will be created.
+    # - If the new value contains a placeholder %{}, then the value will be
+    #   expanded to the related event record field.
+    # @example
+    #   replace {
+    #     "new_message": "a new field"
+    #   }
+    config_param :replace,    :hash,  default: Hash.new
+
+    # Convert a field's value to a different type, like turning a string to an
+    # integer.
+    # - If the field value is an array, all members will be converted.
+    # - If the field value is a hash, then no action will be taken.
+    # - Valid conversion types are integer, float, string, boolean
+    # @example
+    #   convert {
+    #     "id": "integer",
+    #     "message": "string"
+    #   }
+    config_param :convert,    :hash,  default: Hash.new
+
+    # Convert a string field by applying a regular expression and replacement.
+    # - If the field is not a string, then no action will be taken.
+    # 
+    # The configuration takes an array consisting of 3 elements per field/sub.
+    #
+    # @example
+    #   gsub [
+    #     "fieldname",  "/", "_",
+    #     "fieldname2", "[\\?#-]", "."
+    #   ]
+    config_param :gsub,       :hash,  default: Array.new
+
+    # Convert a string to it's uppercase equivalent
+    # @example
+    #   uppercase [
+    #     "field1", "field2"
+    #   ]
+    config_param :uppercase,  :array, default: Array.new
+
+    # Convert a string to it's lowercase equivalent
+    # @example
+    #   lowercase [
+    #     "field1", "field2"
+    #   ]
+    config_param :lowercase,  :array, default: Array.new
+
+    # Strip whitespace from field.
+    # @example 
+    #   strip [
+    #     "field1"
+    #   ]
+    config_param :strip,      :array, default: Array.new
+
+    # Split a field to an array using a separator character
+    # @example 
+    #   split {
+    #     "field1": ","
+    #   }
+    config_param :split,      :hash, default: Hash.new
+
+    # Join an array using a separator character
+    # @example
+    #   join {
+    #     "field1": " "
+    #   }
+    config_param :join,      :hash, default: Hash.new
+
+    # Merge two fields of arrays or hashes
+    # @example
+    #   merge {
+    #     "dest_field": "added_field"
+    #   }
+    config_param :merge,      :hash, default: Hash.new
+
+    # List of all possible mutate actions, in the order that we will apply 
+    # them. As it stands, this is the order in which Logstash would apply them.
+    MUTATE_ACTIONS = %w(
+      rename
+      update
+      replace
+      convert
+      gsub
+      uppercase
+      lowercase
+      strip
+      remove
+      split
+      join
+      merge
+    )
+
+    # Convert valid types
+    VALID_CONVERSIONS = %w(string integer float boolean)
+
+    # Convert helper method prefix
+    CONVERT_PREFIX = "convert_".freeze
+
+    # Convert boolean regex
+    TRUE_REGEX = (/^(true|t|yes|y|1)$/i).freeze
+    FALSE_REGEX = (/^(false|f|no|n|0)$/i).freeze
+
+    # Placeholder regex
+    TEMPLATE_TAG_REGEXP = /%\{[^}]+\}/
+
+    # Initialize attributes and parameters
+    # @since 0.1.0
+    # @return [NilClass]
+    def configure(conf)
+      super
+
+      @convert.nil? or @convert.each do |field, type|
+        if !VALID_CONVERSIONS.include?(type)
+          raise ConfigError, 
+            "convert #{type} is not one of #{VALID_CONVERSIONS.join(',')}."
+        end
+      end
+
+      @gsub_parsed = []
+      @gsub.nil? or 
+      @gsub.each_slice(3) do |field, needle, replacement|
+        if [field, needle, replacement].any? {|n| n.nil?}
+          raise ConfigError,
+            "gsub #{[field,needle,replacement]} requires 3 elements."
+        end
+
+        @gsub_parsed << {
+          field: field,
+          needle: (needle.index("%{").nil?? Regexp.new(needle): needle),
+          replacement: replacement
+        }
+      end
+    end
+
+    # Filter action which will manipulate records
+    # @since 0.1.0
+    # @return [Hash] the modified event record
+    def filter(tag, time, record)
+      # In order to more easily navigate the record, we wrap the record in a
+      # delegator. We additionally pass the `expand_nesting` option which
+      # determines whether we should treat periods as field separators.
+      result = Fluent::PluginMixin::MutateEvent.
+        new(record, expand_nesting: @expand_nesting)
+
+      MUTATE_ACTIONS.each do |action|
+        begin
+          send(action.to_sym, result) if instance_variable_get("@#{action}")
+        rescue => e
+          log.warn "failed to mutate #{action} action", error: e
+          log.warn_backtrace
+        end
+      end
+
+      result.prune if @prune_empty
+      result.to_record
+    end
+
+    protected
+
+    # Expand %{} strings to the related event fields.
+    # @since 0.1.0
+    # @return [String] the modified string
+    def expand_references(event, string)
+      new_string = ''
+
+      position = 0
+      matches = string.scan(TEMPLATE_TAG_REGEXP).map{|m| $~}
+
+      matches.each do |match|
+        reference_tag = match[0][2..-2]
+        reference_value = event.get(reference_tag.downcase).to_s
+        if reference_value.nil?
+          @log.error "failed to replace tag", field: reference_tag.downcase
+          reference_value = match.to_s
+        end
+
+        start = match.offset(0).first
+        new_string << string[position..(start-1)] if start > 0
+        new_string << reference_value
+        position = match.offset(0).last
+      end
+
+      if position < string.size
+        new_string << string[position..-1]
+      end
+
+      new_string
+    end
+
+    # Remove fields from the event hash
+    # @since 0.1.0
+    def remove(event)
+      @remove.each do |field|
+        event.remove(field)
+      end
+    end
+
+    # Rename fields in the event hash
+    # @since 0.1.0
+    def rename(event)
+      @rename.each do |old, new|
+        item = event.get(old)
+        next if item.nil?
+        event.set(new, item)
+        event.remove(old)
+      end
+    end
+
+    # Update (existing) fields in the event hash
+    # @since 0.1.0
+    def update(event)
+      @update.each do |field, newvalue|
+        newvalue = expand_references(event, newvalue)
+        next unless event.include?(field)
+        event.set(field, newvalue)
+      end
+    end
+
+    # Replace fields in the event hash
+    # @since 0.1.0
+    def replace(event)
+      @replace.each do |field, newvalue|
+        newvalue = expand_references(event, newvalue)
+        event.set(field, newvalue)
+      end
+    end
+
+    # Convert fields to given types in the record hash
+    # @since 0.1.0
+    def convert(event)
+      @convert.each do |field, type|
+        converter = method(CONVERT_PREFIX + type)
+
+        case original = event.get(field)
+        when NilClass
+          next
+        when Hash
+          @log.error("cannot convert hash", field: field, value: original)
+        when Array
+          event.set(field, original.map{|v| converter.call(v)})
+        else
+          event.set(field, converter.call(original))
+        end
+      end
+    end
+
+    def convert_string(value)
+      value.to_s.force_encoding(Encoding::UTF_8)
+    end
+
+    def convert_integer(value)
+      value.to_i
+    end
+
+    def convert_float(value)
+      value.to_f
+    end
+
+    def convert_boolean(value)
+      return true if value =~ TRUE_REGEX
+      return false if value.empty? || value =~ FALSE_REGEX
+      @log.error("failed to convert to boolean", value: value)
+    end
+
+    # Convert field values to uppercase in the record hash
+    # @since 0.1.0
+    def uppercase(event)
+      @uppercase.each do |field|
+        original = event.get(field)
+        result = case original
+                 when Array
+                   original.map do |elem|
+                     (elem.is_a?(String) ? elemen.upcase : elem)
+                   end
+                 when String
+                   original.upcase! || original
+                 else
+                   @log.error("can't uppercase field", 
+                              field: field, 
+                              value: original)
+                   original
+                 end
+        event.set(field, result)
+      end
+    end
+
+    # Convert field values to lowercase in the record hash
+    # @since 0.1.0
+    def lowercase(event)
+      @lowercase.each do |field|
+        original = event.get(field)
+        result = case original
+                 when Array
+                   original.map do |elem|
+                     (elem.is_a?(String) ? elemen.downcase : elem)
+                   end
+                 when String
+                   original.downcase! || original
+                 else
+                   @log.error("can't lowercase field", 
+                              field: field, 
+                              value: original)
+                   original
+                 end
+        event.set(field, result)
+      end
+    end
+
+    # Split fields based on delimiters in the record hash
+    # @since 0.1.0
+    def split(event)
+      @split.each do |field, separator|
+        value = event.get(field)
+        if value.is_a?(String)
+          event.set(field, value.split(separator))
+        else
+          @loger.error("can't split field", 
+                       field: field,
+                       value: value)
+        end
+      end
+    end
+
+    # Join fields based on delimiters in the record hash
+    # @since 0.1.0
+    def join(event)
+      @join.each do |field, separator|
+        value = event.get(field)
+        if value.is_a?(Array)
+          event.set(field, value.join(separator))
+        end
+      end
+    end
+
+    # Strip whitespace surrounding fields in the record hash
+    # @since 0.1.0
+    def strip(event)
+      @strip.each do |field|
+        value = event.get(field)
+        case value
+        when Array
+          event.set(field, value.map{|s| s.strip})
+        when String
+          event.set(field, value.strip)
+        end
+      end
+    end
+
+    # Merge hashes and arrays in the record hash
+    # @since 0.1.0
+    def merge(event)
+      @merge.each do |dest_field, added_fields|
+        dest_field_value = event.get(dest_field)
+
+        Array(added_fields).each do |added_field|
+          added_field_value = event.get(added_field)
+
+          if dest_field_value.is_a?(Hash) ^ added_field_value.is_a?(Hash)
+            @log.error('cannot merge an array and hash',
+                       dest_field: dest_field,
+                       added_field: added_field)
+            next
+          end
+
+          if dest_field_value.is_a?(Hash)
+            event.set(dest_field, dest_field_value.update(added_field_value))
+          else
+            event.set(dest_field, Array(dest_field_value).
+                                  concat(Array(added_field_value)))
+          end
+        end
+      end
+    end
+
+    # Perform regular expression substitutions in the record hahs
+    # @since 0.1.0
+    def gsub(event)
+      @gsub_parsed.each do |config|
+        field = config[:field]
+        needle = config[:needle]
+        replacement = config[:replacement]
+
+        value = event.get(field)
+        case value
+        when Array
+          result = value.map do |v|
+            if v.is_a?(String)
+              gsub_dynamic_fields(event, v, needle, replacement)
+            else 
+              @log.error('cannot gsub non Strings', 
+                         field: field, 
+                         value: v)
+            end
+            event.set(field, result)
+          end
+        when String
+          v = gsub_dynamic_fields(event, value, needle, replacement)
+          event.set(field, v)
+        else
+          @log.error('cannot gsub non Strings', field: field, value: value)
+        end
+      end
+    end
+
+    def gsub_dynamic_fields(event, original, needle, replacement)
+      replacement = expand_references(event, replacement)
+      if needle.is_a?(Regexp)
+        original.gsub(needle, replacement)
+      else
+        original.gsub(Regexp.new(needle), replacement)
+      end
+    end
+  end
+end
+

data/lib/fluent/plugin_mixin/mutate_event.rb

@@ -0,0 +1,78 @@
+module Fluent
+  module PluginMixin
+    class MutateEvent < SimpleDelegator
+      def initialize(record, expand_nesting = true)
+        super(record)
+        @record = record
+        @expand_nesting = expand_nesting
+      end
+
+      def to_record
+        @record
+      end
+
+      def dig(*keys)
+        item = @record
+
+        keys.each do |key|
+          break if item.nil?
+          item = item.is_a?(Hash) ? item[key] : nil
+        end
+
+        item
+      end
+
+      def prune
+        delete_proc = proc do |_,v|
+          v.delete_if(&delete_proc) if v.respond_to?(:delete_if)
+          v.nil? || v.respond_to?(:empty?) && v.empty?
+        end
+
+        @record.delete_if(&delete_proc)
+      end
+
+      def get(key_or_path, &block)
+        item = dig(*expand_key(key_or_path))
+        block_given? ? yield(item) : item
+      end
+
+      def parent(key_or_path, &block)
+        path = expand_key(key_or_path)
+        child = path.pop
+
+        item = dig(*path)
+        block_given? ? yield(item, child) : item
+      end
+
+      def set(key_or_path, value)
+        path = expand_key(key_or_path)
+        child = path.pop
+
+        item = @record
+        path.each do |key|
+          item = item[key] ||= {}
+        end
+        item[child] = value
+      end
+
+      def remove(key_or_path)
+        path = expand_key(key_or_path)
+        child = path.pop
+
+        parent(key_or_path) do |item, child|
+          item.delete(child) unless item.nil?
+        end
+      end
+
+      def include?(key_or_path)
+        !get(key_or_path).nil?
+      end
+
+      protected
+
+      def expand_key(key_or_path)
+        @expand_nesting ? key_or_path.split(".") : [key_or_path]
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,102 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-mutate_filter
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Jonathan Serafini
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-06-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.15.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.15.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: ''
+email:
+- jonathan@serafini.ca
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- CHANGELOG.md
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- fluent-plugin-mutate_filter.gemspec
+- lib/fluent/plugin/filter_mutate.rb
+- lib/fluent/plugin_mixin/mutate_event.rb
+homepage: http://fuck.off
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: A mutate filter for Fluent which functions like Logstash.
+test_files: []
+has_rdoc: