logstash-filter-dateparts 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 18f2c742ff7b1224e1332c29188a86b76e7d6873
+  data.tar.gz: fd2520f90581b63177f112b3f99b9d79db36b884
+SHA512:
+  metadata.gz: d7c58b44fcbe9711f23ba0e3813a8c544a5b846177bd987f50e11ae7dc47fa24ef150e3be55feba1f36eef1507d3b2c9d1cfebfa429bd2d6945f55ef5d81b339
+  data.tar.gz: b0c32973d8f267a68f547373d16701d5e1e8d8d47460b412b6183d4937ad41bd4a83f4fc2dc2d38dde4217b635539455400d7b5af24fd37ebaa7606491b48ff8

data/CHANGELOG.md

@@ -0,0 +1 @@
+

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2014–2015 Mike Baranski <http://www.mikeski.net>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,113 @@
+# Logstash Plugin
+
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
+
+## License ##
+
+Copyright (c) 2014–2015 Mike Baranski <http://www.mikeski.net>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+## About
+
+This plugin is useful if you want to easily query Logstash data on *day of week*, *hour of day*, or other parts of a date.  See the usage below for details on the output of the plugin.  The date parts that can be generated are:
+
+* day
+* wday
+* yday
+* month
+* year
+* hour
+* min
+* sec
+
+## Documentation
+
+### Installation
+
+To manually install the plugin, download the gem and run:
+
+`bin/plugin install --no-verify logstash-filter-dateparts-1.0.0.gem`
+
+### Usage
+
+To see the most basic usage, you can run the following (on Linux):
+
+`echo "HI" | bin/logstash -e 'input { stdin {} } filter {dateparts { }} output { stdout { codec=> rubydebug}}'`
+
+You could also use the logstash generator:
+
+`bin/logstash -e 'input {  generator { lines => ["HI"] count => 1  } } filter {dateparts { }} output { stdout { codec=> rubydebug}}'`
+
+Here is the sample output:
+
+	{
+		"message" => "HI",
+		"@version" => "1",
+		"@timestamp" => "2015-11-20T12:24:40.217Z",
+		"host" => "mike-VirtualBox",
+		"day" => 20,
+		"wday" => 5,
+		"yday" => 324,
+		"month" => 11,
+		"year" => 2015,
+		"hour" => 12,
+		"min" => 24,
+		"sec" => 40
+	}
+
+
+This uses the default configuration, which generates the following fields from the `@timestamp` field of the event:
+
+* day
+* wday
+* yday
+* month
+* year
+* hour
+* min
+* sec
+
+### Configuration
+
+#### Fields
+
+The generated fields are based on the date functions available in the [Ruby time class](http://ruby-doc.org/core-2.2.0/Time.html).  You can specify any valid function and it will be added to the event.
+
+For example, this will add 2 fields, *sec* corresponding to `time.sec()` and *hour* corresponding to `time.hour()`:
+
+    filter {
+    	   dateparts {
+	   	     "fields" => ["sec", "hour"]
+	   }
+    }
+
+#### Time Field
+
+By default, the plugin will use the *@timestamp* field, but you can specify a different one:
+
+    filter {
+    	   dateparts {
+	   	     "time_field" => "some_other_field"
+	   }
+    }
+
+#### Error Tags
+
+By default, the tag *_dateparts_error* is added on exception.  You can specify different tag(s) like so:
+
+    filter {
+    	   dateparts {
+	   	     "error_tags" => ["bad_dates", "xyz"]
+	   }
+    }

data/lib/logstash/filters/dateparts.rb

@@ -0,0 +1,68 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# This filter will add date parts to your record based on
+# the timestamp field.
+# 
+class LogStash::Filters::DateParts < LogStash::Filters::Base
+  # Setting the config_name here is required. This is how you
+  # configure this filter from your Logstash config.
+  #
+  # filter {
+  #   dateparts {
+  #     
+  #   }
+  # }
+  #
+  config_name "dateparts"
+  config :fields, :validate => :array, :default => ["day", "wday", "yday", "month", "year", "hour", "min", "sec"], :required => true
+  config :time_field, :validate => :string, :default => "@timestamp", :required => true
+  config :error_tags, :validate => :array, :default => ["_dateparts_error"], :required => true
+  
+  public
+  def register
+    logger.debug? and logger.debug("DateParts filter registered")
+  end
+
+  def plugin_error(message, event)
+    logger.error("DatePart filter error: " + message)
+    LogStash::Util::Decorators.add_tags(@error_tags, event, "filters/#{self.class.name}")
+  end
+
+  def get_time_from_field(f)
+    if f.class == Time
+      return f
+    elsif f.respond_to?("time")
+      logger.info("Class is #{f.class}")
+      return f.time()
+    else
+      return nil
+    end
+  end
+  
+  public
+  def filter(event)
+    if @fields.respond_to?("each") and @fields.respond_to?("join")
+      logger.debug? and logger.debug("DateParts plugin filtering #{@time_field} time_field and adding fields: " + @fields.join(", "))
+      t = get_time_from_field(event[@time_field])
+      if t == nil
+        plugin_error("Invalid time field #{@time_field}; Time field must be an instance of Time or provide a time method that returns one", event)
+        return
+      end
+      @fields.each do |field|
+        begin
+          event[field] = t.send(field)
+        rescue
+          plugin_error("No such method: #{field}\n", event)
+        end
+      end
+    else
+      plugin_error("DateParts plugin fields invalid, should be an array of function names")
+      return
+    end
+      
+    filter_matched(event)
+  end # def filter
+  
+end # class LogStash::Filters::DateParts

data/logstash-filter-dateparts.gemspec

@@ -0,0 +1,23 @@
+Gem::Specification.new do |s|
+  s.name = 'logstash-filter-dateparts'
+  s.version         = '1.0.0'
+  s.licenses = ['Apache License (2.0)']
+  s.summary = 'This dateparts fileter adds date information to your event based on your timestamp'
+  s.description = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program'
+  s.authors = ['Mike Baranski']
+  s.email = 'mike.baranski@gmail.com'
+  s.homepage = 'http://mikeski.net'
+  s.require_paths = ['lib']
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','Gemfile','LICENSE']
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'filter' }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash-core', '>= 2.0.0.beta2', '< 3.0.0'
+  s.add_development_dependency 'logstash-devutils'
+end

data/spec/filters/dateparts_spec.rb

@@ -0,0 +1,82 @@
+require 'spec_helper'
+require "logstash/filters/dateparts"
+require "logstash/timestamp"
+require "logstash/event"
+
+def get_event(contents = {})
+  contents["@timestamp"] = LogStash::Timestamp.new
+  event = LogStash::Event.new(contents)
+  return event
+end
+
+describe LogStash::Filters::DateParts do
+  default_ts = "@timestamp"
+  alt_ts_field = "zxlk"
+  
+  it "Default config should result in filter with 8 functions, one error tag and @timestamp as the time field" do
+    f = LogStash::Filters::DateParts.new({})
+    
+    expect(f.class).to eq(LogStash::Filters::DateParts)
+    expect(f.fields.length).to eq(8)
+    expect(f.time_field).to eq(default_ts)
+    expect(f.error_tags.length).to eq(1)
+  end
+
+  it "Config should result in filter with 2 functions and the alt timestamp field" do
+    f = LogStash::Filters::DateParts.new({
+                                           "fields" => ["sec", "hour"],
+                                           "time_field" => alt_ts_field 
+                                         })
+    
+    expect(f.class).to eq(LogStash::Filters::DateParts)
+    expect(f.fields.length).to eq(2)
+    expect(f.fields[0]).to eq("sec")
+    expect(f.time_field).to eq(alt_ts_field)
+  end
+
+  it "Should generate the default fields (8 of them)" do
+    event = get_event()
+    count = event.to_hash().count
+    f = LogStash::Filters::DateParts.new({})
+    f.filter(event)
+    
+    expect(event.to_hash().count).to eq(count + 8)
+    expect(event['sec']).to be_truthy
+    expect(event['hour']).to be_truthy
+    expect(event['min']).to be_truthy
+    expect(event['month']).to be_truthy
+    expect(event['year']).to be_truthy
+    expect(event['day']).to be_truthy
+    expect(event['wday']).to be_truthy
+    expect(event['yday']).to be_truthy
+    expect(event['tags']).to be_nil
+  end
+
+  it "Should generate only the specified fields" do
+    event = get_event()
+    count = event.to_hash.count
+    f = LogStash::Filters::DateParts.new({
+                                           "fields" => ["sec", "hour"]
+                                         })
+    f.filter(event)
+    expect(event.to_hash().count).to eq(count + 2)
+    expect(event['sec']).to be_truthy
+    expect(event['hour']).to be_truthy
+    expect(event['min']).to be_nil
+    expect(event['month']).to be_nil
+    expect(event['year']).to be_nil
+    expect(event['day']).to be_nil
+    expect(event['wday']).to be_nil
+    expect(event['yday']).to be_nil
+    expect(event['tags']).to be_nil
+  end
+
+  it "Should set the error tag on an invalid time field" do
+    event = get_event()
+    count = event.to_hash().count
+    f = LogStash::Filters::DateParts.new({ "time_field" => alt_ts_field })
+    
+    f.filter(event)
+    expect(event['tags'].include? '_dateparts_error').to eq(true)
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1 @@
+require "logstash/devutils/rspec/spec_helper"

metadata

@@ -0,0 +1,92 @@
+--- !ruby/object:Gem::Specification
+name: logstash-filter-dateparts
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Mike Baranski
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-11-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0.beta2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0.beta2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not
+  a stand-alone program
+email: mike.baranski@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/filters/dateparts.rb
+- logstash-filter-dateparts.gemspec
+- spec/filters/dateparts_spec.rb
+- spec/spec_helper.rb
+homepage: http://mikeski.net
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: filter
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: This dateparts fileter adds date information to your event based on your
+  timestamp
+test_files:
+- spec/filters/dateparts_spec.rb
+- spec/spec_helper.rb