logstash-filter-collect 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8bba7ec257f540842aa9d9f509d52ad982401ae6
+  data.tar.gz: a14a1f5b055e437c0d769123cfe586159485c542
+SHA512:
+  metadata.gz: c9181a4cfa77a6dc954e9b90e0b05e12e39ff4d636b249c487a4a341cd420fd3eb80c40a450b56e9513e4fc7e101f50307cecb0ec90e797ae2abb81ddeefd9e2
+  data.tar.gz: 61ceb2a840cba9c529f24136197143ac88f2889cda9156fcc736d36ce8b31445db2ffc73a89c52406c016237aa8419104f1cd63233a1eba4b555230b42ccc3ea

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+## 0.1.0
+  - Plugin created with the logstash plugin generator

data/CONTRIBUTORS

@@ -0,0 +1,10 @@
+The following is a list of people who have contributed ideas, code, bug
+reports, or in general have helped logstash along its way.
+
+Contributors:
+* Mike Baranski - mike.baranski@gmail.com
+
+Note: If you've sent us patches, bug reports, or otherwise contributed to
+Logstash, and you aren't on the list above and want to be, please let us know
+and we'll make sure you're here. Contributions from folks like you are what make
+open source awesome.

data/DEVELOPER.md

@@ -0,0 +1,2 @@
+# logstash-filter-collect
+Example filter plugin. This should help bootstrap your effort to write your own filter plugin!

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gemspec
+

data/LICENSE

@@ -0,0 +1,11 @@
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,44 @@
+# Logstash Collect Plugin
+
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
+
+It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
+
+Author: Mike Baranski (mike.baranski@gmail.com).  Contributions are welcome.
+
+## License ##
+
+Copyright (c) 2014–2017 Mike Baranski <http://www.mikeski.net>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+## Installation
+
+## Usage
+
+This plugin takes a list of objects in your event and turns it into a 
+list of strings.  The original purpose of this was to take something like this:
+
+     lop => [{ 'person' => {'id' => 12, 'name' => 'Mike'} }, {'person' => {'id' => 13, 'name' => 'Sam'}}]
+
+and turn it into a list like this:
+
+    names => ['Mike', 'Sam']
+    
+With this filter plugin, the following configuration will do that:
+    
+    collect => {
+        'field' => 'people', 
+        'property' => ['person' 'name'], 
+        'collection' => 'names'
+    }

data/lib/logstash/filters/collect.rb

@@ -0,0 +1,68 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# This  filter will replace the contents of the default 
+# message field with whatever you specify in the configuration.
+#
+# It is only intended to be used as an .
+class LogStash::Filters::Collect < LogStash::Filters::Base
+
+  # This is how you
+  # configure this filter from your Logstash config.
+  #
+  # filter {
+  #    collect {
+  #     field => 'object_array_field',
+  #     property => ['property', 'of', 'each', 'object'],
+  #     collection => 'output_list_name',
+  #   }
+  # }
+  #
+  config_name 'collect'
+
+  # Replace the message with this value.
+  config :field, :validate => :string, :required => true
+  config :property, :validate => :array, :required => true
+  config :collection, :validate => :string, :required => true, :default => 'collection'
+  config :error_tags, :validate => :array, :required => true, :default => ['_collect_error']
+
+  public
+  def register
+    # Add instance variables 
+  end
+
+  private
+  def plugin_error(message, event)
+    @logger.error("Collect filter error: #{message}")
+    LogStash::Util::Decorators.add_tags(@error_tags, event, "filters/#{self.class.name}")
+  end
+
+  private
+  def get_nested_property_from_object(property_array, source)
+    interim_object = source
+    @property.each { |x|
+      interim_object = interim_object[x]
+    }
+    return interim_object
+  end
+
+  public
+  def filter(event)
+    object_array = event.get(@field)
+    if not object_array.class == Array
+      plugin_error("Error, #{@field} is not an array", event)
+      return
+    end
+
+    new_collection = []
+    object_array.each { |obj|
+      # Add the value to the new collection
+      new_collection.push(get_nested_property_from_object(@property, obj))
+    }
+
+    event.set(@collection, new_collection)
+    filter_matched(event)
+  end # def filter
+
+end # class LogStash::Filters::Collect

data/logstash-filter-collect.gemspec

@@ -0,0 +1,22 @@
+Gem::Specification.new do |s|
+  s.name          = 'logstash-filter-collect'
+  s.version       = '1.0.0'
+  s.licenses      = ['Apache License (2.0)']
+  s.summary       = 'Collect list of properties from list of objects'
+  s.homepage      = 'http://mikeski.net'
+  s.authors       = ['Mike Baranski']
+  s.email         = 'mike.baranski@gmail.com'
+  s.require_paths = ['lib']
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
+  s.add_development_dependency 'logstash-devutils'
+end

data/spec/filters/collect_spec.rb

@@ -0,0 +1,71 @@
+# encoding: utf-8
+require_relative '../spec_helper'
+require "logstash/filters/collect"
+
+describe LogStash::Filters::Collect do
+  #config.expect_with(:rspec) { |c| c.syntax = :should }
+
+  def get_event(contents = {})
+    contents['@timestamp'] = LogStash::Timestamp.new
+    contents['host'] = 'somehost'
+    contents['message'] = 'some message for event'
+    LogStash::Event.new(contents)
+  end
+
+  it 'Test get_event method' do
+    e = get_event({'field1': 'val1'})
+    e.set('test', 1234)
+    e.to_hash
+  end
+
+  it 'Test new filter no config' do
+    expect { LogStash::Filters::Collect.new({}) }.to raise_error(LogStash::ConfigurationError)
+  end
+
+  it 'Property can be a list' do
+    config = {'field' => 'object_array', 'property' => %w(property to get)}
+    LogStash::Filters::Collect.new(config)
+  end
+
+  it 'Property can be a single string' do
+    config = {'field' => 'object_array', 'property' => 'prop'}
+    LogStash::Filters::Collect.new(config)
+  end
+
+  it 'Override collection default' do
+    config = {'field' => 'object_array', 'property' => 'prop', 'collection' => 'some_collection'}
+    LogStash::Filters::Collect.new(config)
+  end
+
+  it 'Passing in invalid collection adds error tag' do
+    config = {'field' => 'people', 'property' => 'name', 'collection' => 'names'}
+    f = LogStash::Filters::Collect.new(config)
+    e = get_event({'people' => 'scalar value'})
+    f.filter(e)
+    e.get('tags').should include('_collect_error')
+  end
+
+  it 'Test list of simple objects (only 1 level to de-reference)' do
+    ppl = [{'id' => 12, 'name' => 'Mike'}, {'id' => 13, 'name' => 'Sam'}]
+    config = {'field' => 'people', 'property' => 'name', 'collection' => 'names'}
+    f = LogStash::Filters::Collect.new(config)
+    e = get_event({'people' => ppl})
+    f.filter(e)
+
+    expect(e.get('names').class).to be(Array)
+    expect(e.get('names').include?('Mike')).to be(true)
+    expect(e.get('names').include?('Sam')).to be(true)
+  end
+
+  it 'Test list of not-so-simple objects (> 1 level to de-reference)' do
+    ppl = [{ 'person' => {'id' => 12, 'name' => 'Mike'} }, {'person' => {'id' => 13, 'name' => 'Sam'}}]
+    config = {'field' => 'people', 'property' => %w(person name), 'collection' => 'names'}
+    f = LogStash::Filters::Collect.new(config)
+    e = get_event({'people' => ppl})
+    f.filter(e)
+    expect(e.get('names').class).to be(Array)
+    expect(e.get('names').include?('Mike')).to be(true)
+    expect(e.get('names').include?('Sam')).to be(true)
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,3 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+

metadata

@@ -0,0 +1,85 @@
+--- !ruby/object:Gem::Specification
+name: logstash-filter-collect
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Mike Baranski
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2017-01-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description:
+email: mike.baranski@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- DEVELOPER.md
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/filters/collect.rb
+- logstash-filter-collect.gemspec
+- spec/filters/collect_spec.rb
+- spec/spec_helper.rb
+homepage: http://mikeski.net
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: filter
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.8
+signing_key:
+specification_version: 4
+summary: Collect list of properties from list of objects
+test_files:
+- spec/filters/collect_spec.rb
+- spec/spec_helper.rb