logstash-filter-categoriser 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: b8ed1e2988fc25a200ab05ab26d11c2f27ce20a4afe757341383e36a3d33c2a2
+  data.tar.gz: 1e7e44e58d9f586024ca96fc2a69c08b1195563e15bf6e37695d79e89a1a6928
+SHA512:
+  metadata.gz: e38601bcbd3abb4be52ab6a2e4dd7ea3aea88a8fa56136e1f3d41115cbe5f5e8032f696ebf55ac8337e27ce571ff0e637b5ba641e37787f25993286d7ce9db16
+  data.tar.gz: 6e61ecac55f46662375303d5650175864a19bcbd477ec573758d5069b85480da3b8ce01392f69c8cf1e3a5b1079e6842fc8ffa8d7c43ef25f4c1574bf00b8653

data/CHANGELOG.md

@@ -0,0 +1,7 @@
+## 3.0.2
+ - Docs: Add documentation template
+## 2.0.0
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
+   instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
+ - Dependency on logstash-core update to 2.0
+

data/CONTRIBUTORS

@@ -0,0 +1,11 @@
+The following is a list of people who have contributed ideas, code, bug
+reports, or in general have helped logstash along its way.
+
+Contributors:
+* Aaron Mildenstein (untergeek)
+* Pier-Hugues Pellerin (ph)
+
+Note: If you've sent us patches, bug reports, or otherwise contributed to
+Logstash, and you aren't on the list above and want to be, please let us know
+and we'll make sure you're here. Contributions from folks like you are what make
+open source awesome.

data/DEVELOPER.md

@@ -0,0 +1,2 @@
+# logstash-filter-example
+Example filter plugin. This should help bootstrap your effort to write your own filter plugin!

data/Gemfile

@@ -0,0 +1,11 @@
+source 'https://rubygems.org'
+
+gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/NOTICE.TXT

@@ -0,0 +1,5 @@
+Elasticsearch
+Copyright 2012-2015 Elasticsearch
+
+This product includes software developed by The Apache Software
+Foundation (http://www.apache.org/).

data/README.md

@@ -0,0 +1,98 @@
+# Logstash Plugin
+
+[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-filter-example.svg)](https://travis-ci.org/logstash-plugins/logstash-filter-example)
+
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
+
+It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
+
+## Documentation
+
+Logstash provides infrastructure to automatically build documentation for this plugin. We provide a template file, index.asciidoc, where you can add documentation. The contents of this file will be converted into html and then placed with other plugin documentation in a [central location](http://www.elastic.co/guide/en/logstash/current/).
+
+- For formatting config examples, you can use the asciidoc `[source,json]` directive
+- For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide
+
+## Need Help?
+
+Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/logstash discussion forum.
+
+## Developing
+
+### 1. Plugin Developement and Testing
+
+#### Code
+- To get started, you'll need JRuby with the Bundler gem installed.
+
+- Create a new plugin or clone and existing from the GitHub [logstash-plugins](https://github.com/logstash-plugins) organization. We also provide [example plugins](https://github.com/logstash-plugins?query=example).
+
+- Install dependencies
+```sh
+bundle install
+```
+
+#### Test
+
+- Update your dependencies
+
+```sh
+bundle install
+```
+
+- Run tests
+
+```sh
+bundle exec rspec
+```
+
+### 2. Running your unpublished Plugin in Logstash
+
+#### 2.1 Run in a local Logstash clone
+
+- Edit Logstash `Gemfile` and add the local plugin path, for example:
+```ruby
+gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
+```
+- Install plugin
+```sh
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
+bin/plugin install --no-verify
+
+```
+- Run Logstash with your plugin
+```sh
+bin/logstash -e 'filter {awesome {}}'
+```
+At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
+
+#### 2.2 Run in an installed Logstash
+
+You can use the same **2.1** method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory or you can build the gem and install it using:
+
+- Build your plugin gem
+```sh
+gem build logstash-filter-awesome.gemspec
+```
+- Install the plugin from the Logstash home
+```sh
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
+bin/plugin install --no-verify
+
+```
+- Start Logstash and proceed to test the plugin
+
+## Contributing
+
+All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
+
+Programming is not a required skill. Whatever you've seen about open source and maintainers or community members  saying "send patches or die" - you will not see that here.
+
+It is more important to the community that you are able to contribute.
+
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.

data/lib/logstash/filters/categoriser.rb

@@ -0,0 +1,80 @@
+# encoding: utf-8
+# frozen_string_literal: true
+require "logstash/filters/base"
+require "logstash/namespace"
+
+#
+# A way to categorise devices based on existing fields.
+#
+# In my scenario I have multiple types of devices all
+# sending syslog logs.  I use this to quickly separate
+# them in order to run a pipeline for each device type.
+#
+# Example config:
+#
+#   filter {
+#     categoriser {
+#       rules_file => "/etc/logstash/device_type.rules.json"
+#       target => "device_type"
+#       default_category => "unknown"
+#     }
+#   }
+#
+# .. with an example rules file:
+#
+# {
+#   "cisco_asa_firewall": ["hostname", "contains", "-asa-"],
+#   "cisco_pix_fwsm_firewall": [
+#     "or", [
+#       ["hostname", "contains", "-pix-"],
+#       ["hostname", "contains", "-fwsm-"]]],
+#   "web_servers": ["hostname", "starts_with", "web"]
+# }
+#
+# This would replace the contents of the "device_type" field
+# with the category in the rules file, ie "cisco_asa_firewall".
+# If we don't match any rules then "device_type" will be set
+# to "unknown".
+#
+class LogStash::Filters::Categoriser < LogStash::Filters::Base
+
+  config_name "categoriser"
+
+  require 'logstash/filters/categoriser/rules'
+
+  # The rules filename, ie:
+  #   filter {
+  #     categoriser {
+  #       rules_file => "/etc/logstash/device_type.rules.json"
+  #       target => "device_type"
+  #     }
+  #   }
+  config :rules_file, :validate => :string
+  config :target, :validate => :string, :default => "category"
+  config :default_category, :validate => :string, :default => "unknown"
+
+  public def register
+    # Add instance variables
+    filter_config = LogStash::Filters::Categoriser::Rules.new(@logger)
+    @rules = filter_config.read_config(@rules_file)
+  end
+
+  public def filter(event)
+    device_type = find_device_type(event)
+    @logger.debug? && @logger.debug("Device type: #{device_type}")
+    event.set(@target, device_type)
+
+    # filter_matched should go in the last line of our successful code
+    filter_matched(event)
+  end
+
+  private def find_device_type(event)
+    matched_type = @rules.find do |type, matcher|
+      matcher.call(event)
+    end || [@default_category, nil]
+
+    matched_type.first
+  end
+
+end
+

data/lib/logstash/filters/categoriser/config.rb

@@ -0,0 +1,153 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'json'
+
+class LogStash::Filters::Categoriser::RulesError < StandardError; end
+
+class LogStash::Filters::Categoriser::Rules
+
+  def initialize(logger)
+    @logger = logger
+  end
+
+  def default_config
+    default =<<-JSONCONFIG.gsub(/^ {6}/,'')
+      {
+        "cisco_asa_firewall": ["hostname", "contains", "-asa-"],
+        "cisco_pix_firewall": [
+          "or", [
+            ["hostname", "contains", "-pix-"],
+            ["hostname", "contains", "-fwsm-"]]],
+        "f5_bigip": ["hostname", "contains", "-bigip-"],
+        "checkpoint_ipso_firewall": ["hostname", "contains", "-nok-"],
+        "checkpoint_gaia_firewall": ["hostname", "contains", "-cpg-"],
+        "bluecoat_proxysg": ["hostname", "contains", "-bcsg-"],
+        "bluecoat_proxyav": ["hostname", "contains", "-bcav-"]
+      }
+    JSONCONFIG
+    JSON.load(default)
+  end
+
+  def read_config(filename)
+    conf = JSON.load(File.read(filename))
+    load_config(conf)
+  end
+
+  # Returns a hash:
+  #   {"cisco_asa_firewall" => Proc}
+  #
+  # So
+  def load_config(config)
+    config.keys.reduce({}) do |acc, key|
+      acc.merge({key => parse_checks(config[key])})
+    end
+  end
+
+  # +checks+::
+  #   ["and", [
+  #     ["hostname", "contains", "-pix-"],
+  #     ["hostname", "contains", "-fwsm-"]
+  #   ]]
+  # Returns a proc that you can pass 'event' into
+  # The proc will return true or false.
+  def parse_checks(checks)
+    if !checks.is_a?(Array)
+      raise ConfigError, "checks should be an array"
+    end
+
+    case checks.first
+    when "and"
+      if checks.last.is_a?(Array)
+        Proc.new do |event|
+          child_procs = checks.last.map {|check| parse_checks(check)}
+          child_procs.all? {|child| child.call(event)}
+        end
+      else
+        @logger.warn('Invalid config: "and" should be followed by an array, ie: ["and", [["hostname", "contains", "a"], ["hostname ", "contains", "b"]]]')
+      end
+
+    when "or"
+      if checks.last.is_a?(Array)
+        Proc.new do |event|
+          child_procs = checks.last.map {|check| parse_checks(check)}
+          child_procs.any? {|child| child.call(event)}
+        end
+      else
+        @logger.warn('Invalid config: "or" should be followed by an array, ie: ["or", [["hostname", "contains", "a"], ["hostname", "contains", "b"]]]')
+      end
+
+    when "not"
+      if checks.last.is_a?(Array)
+        Proc.new do |event|
+          not parse_checks(checks.last).call(event)
+        end
+      end
+    else
+      Proc.new { |event| load_check(checks).call(event) }
+    end
+  end
+
+
+  # Loads a check, ie
+  #   ["hostname", "contains", "-asa-"]
+  #
+  # .. where the format is:
+  #
+  #   [<field name>, <contains | regex>, <argument>]
+  #
+  # Returns a proc that can be used later to determine
+  # if the check is true or false, ie:
+  #
+  # asa_check = load_check(["hostname", "contains", "-asa-"])
+  # asa_check.(event)
+  #
+  def load_check(check)
+    if check.length == 3
+      if check.all? {|x| x.is_a?(String) }
+        (check_field, check_command, check_argument) = check
+
+        case check_command
+        when "contains"
+          Proc.new do |event|
+            field_value = event.get(check_field)
+            generic_check_ok?(check, check_field, field_value) && field_value.include?(check_argument)
+          end
+        when "regex"
+          regex = Regexp.new(check_argument, Regexp::IGNORECASE)
+          Proc.new do |event|
+            field_value = event.get(check_field)
+            generic_check_ok?(check, check_field, field_value) && regex.match?(field_value)
+          end
+        else
+          @logger.warn("Invalid config: #{check}, the second argument must be one of: \"contains\", \"regex\"")
+# TODO - see if it's ok to raise an error here?
+          raise ConfigError, "Invalid config: #{check}, the second argument must be one of: \"contains\", \"regex\""
+        end
+
+      else
+        @logger.warn("Invalid config: #{check}, all elements must be a string, ie: " + '["hostname", "contains", "-asa-"]')
+        Proc.new{|event| false}
+      end
+    else
+      @logger.warn("Invalid config: #{check}, expected three elements, ie: " + '["hostname", "contains", "-asa-"]')
+      Proc.new{|event| false}
+    end
+  end
+
+  private def generic_check_ok?(check, check_field, field_value)
+    if field_value.nil?
+      @logger.warn("vf_device_type - Could not read missing field, check #{check} returns false.", :field => check_field)
+      false
+    else
+      if !field_value.is_a?(String)
+        @logger.warn("vf_device_type - Field must be a string, check #{check} returns false (field was a(n) #{field_value.class}).", :field => check_field)
+        false
+      else
+        true
+      end
+    end
+  end
+
+end
+

data/lib/logstash/filters/categoriser/rules.rb

@@ -0,0 +1,185 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'json'
+
+class LogStash::Filters::Categoriser::RulesError < StandardError; end
+
+# This class loads the rules file.
+#
+#
+class LogStash::Filters::Categoriser::Rules
+
+  def initialize(logger)
+    @logger = logger
+  end
+
+  # Returns a hash of Procs that can be used to determine
+  # the device's category.
+  #
+  # For example, given a rules file something like this:
+  #
+  # {
+  #   "one": ["hostname", "contains", "one"],
+  #   "two": ["or", [
+  #     ["hostname", "eq", "a"],
+  #     ["message", "eq", "b"]
+  #   ]]
+  # }
+  #
+  # This method would return a hash:
+  #
+  #   {"one" => Proc, "two" => Proc}
+  #
+  # We can then pass "event" into one of the Procs
+  # and they'll return true or false, for example:
+  #
+  #  matched_type = @rules.find do |key, matcher|
+  #    matcher.call(event)
+  #  end
+  #
+  # @rules is the "key" => Proc hash above.  "matcher"
+  # is the Proc.
+  #
+  #
+  def read_config(filename)
+    conf = if filename.respond_to?(:read)
+             JSON.load(filename)
+           else
+             JSON.load(File.read(filename))
+           end
+
+    load_config(conf)
+  end
+
+  # Returns a hash, ie:
+  #   {"cisco_asa_firewall" => Proc}
+  #
+  def load_config(config)
+    config.keys.reduce({}) do |acc, key|
+      acc.merge({key => parse_checks(config[key])})
+    end
+  end
+
+  # +checks+::
+  #   Example:
+  #
+  #   ["and", [
+  #     ["hostname", "contains", "-pix-"],
+  #     ["hostname", "contains", "-fwsm-"]
+  #   ]]
+  #
+  # Returns a proc that you can pass 'event' into
+  # The proc will return true or false.
+  private def parse_checks(checks)
+    if !checks.is_a?(Array)
+      raise LogStash::Filters::Categoriser::RulesError, "Invalid config: #{checks.dump} Checks should be an array, " + 'ie: ["hostname", "starts_with", "web"]'
+    end
+
+    case checks.first
+    when "and"
+      if checks.last.is_a?(Array)
+        Proc.new do |event|
+          child_procs = checks.last.map {|check| parse_checks(check)}
+          child_procs.all? {|child| child.call(event)}
+        end
+      else
+        @logger.warn('Invalid config: "and" should be followed by an array, ie: ["and", [["hostname", "contains", "a"], ["hostname ", "contains", "b"]]]')
+      end
+
+    when "or"
+      if checks.last.is_a?(Array)
+        Proc.new do |event|
+          child_procs = checks.last.map {|check| parse_checks(check)}
+          child_procs.any? {|child| child.call(event)}
+        end
+      else
+        @logger.warn('Invalid config: "or" should be followed by an array, ie: ["or", [["hostname", "contains", "a"], ["hostname", "contains", "b"]]]')
+      end
+
+    when "not"
+      if checks.last.is_a?(Array)
+        Proc.new do |event|
+          not parse_checks(checks.last).call(event)
+        end
+      end
+    else
+      Proc.new { |event| load_check(checks).call(event) }
+    end
+  end
+
+
+  # Loads a check, ie
+  #   ["hostname", "contains", "-asa-"]
+  #
+  # .. where the format is:
+  #
+  #   [<field name>, <contains | eq etc>, <argument>]
+  #
+  # Returns a proc that can be used later to determine
+  # if the check is true or false, ie:
+  #
+  # asa_check = load_check(["hostname", "contains", "-asa-"])
+  # asa_check.call(event)
+  #
+  private def load_check(check)
+    tag = "#{self.class}\##{__method__}"
+    if check.length == 3
+      if check.all? {|x| x.is_a?(String) }
+        (check_field, check_command, check_argument) = check
+
+
+        case check_command
+        when "equals", "eql", "eq"
+          Proc.new do |event|
+            field_value = event.get(check_field)
+            generic_check_ok?(check, check_field, field_value) && field_value.eql?(check_argument)
+          end
+        when "include", "contains"
+          Proc.new do |event|
+            field_value = event.get(check_field)
+            generic_check_ok?(check, check_field, field_value) && field_value.include?(check_argument)
+          end
+        when "start_with", "starts_with"
+          Proc.new do |event|
+            field_value = event.get(check_field)
+            generic_check_ok?(check, check_field, field_value) && field_value.start_with?(check_argument)
+          end
+        when "end_with", "ends_with"
+          Proc.new do |event|
+            field_value = event.get(check_field)
+            generic_check_ok?(check, check_field, field_value) && field_value.end_with?(check_argument)
+          end
+        else
+          valid_commands = ["contains", "starts_with", "ends_with", "eq"]
+          @logger.warn(tag) { "Invalid config: #{check}, the second argument must be one of: #{valid_commands}" }
+          raise LogStash::Filters::Categoriser::RulesError, "Invalid config: #{check}, the second argument must be one of: #{valid_commands} "
+        end
+
+      else
+        @logger.warn(tag) { "Invalid config: #{check}, all elements must be a string, ie: " + '["hostname", "contains", "-asa-"]' }
+        Proc.new{|event| false}
+      end
+    else
+      @logger.warn(tag) { "Invalid config: #{check}, expected three elements, ie: " + '["hostname", "contains", "-asa-"]' }
+      Proc.new{|event| false}
+    end
+  end
+
+  private def generic_check_ok?(check, check_field, field_value)
+    tag = "#{self.class}\##{__method__}"
+    if field_value.nil?
+      @logger.info(tag) { "Could not read missing field \"#{check_field}\", check #{check.dump} returns false." }
+      false
+    else
+      if !field_value.is_a?(String)
+        @logger.info(tag) { "Field must be a string, check #{check} returns false (field was a(n) #{field_value.class})." }
+        false
+      else
+        true
+      end
+    end
+  end
+
+end
+

data/lib/logstash/filters/example.rb

@@ -0,0 +1,48 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# This example filter will replace the contents of the default
+# message field with whatever you specify in the configuration.
+#
+# It is only intended to be used as an example.
+class LogStash::Filters::Example < LogStash::Filters::Base
+
+  # Setting the config_name here is required. This is how you
+  # configure this filter from your Logstash config.
+  #
+  # filter {
+  #   example {
+  #     message => "My message..."
+  #   }
+  # }
+  #
+  config_name "example"
+
+  # Replace the message with this value.
+  config :message, :validate => :string, :default => "Hello World!"
+
+
+  public
+  def register
+    # Add instance variables
+  end # def register
+
+  public
+  def filter(event)
+
+    if @message
+      # Replace the event message with our message as configured in the
+      # config file.
+
+      # using the event.set API
+      event.set("message", @message)
+      # correct debugging log statement for reference
+      # using the event.get API
+      @logger.debug? && @logger.debug("Message is now: #{event.get("message")}")
+    end
+
+    # filter_matched should go in the last line of our successful code
+    filter_matched(event)
+  end # def filter
+end # class LogStash::Filters::Example

data/logstash-filter-categoriser.gemspec

@@ -0,0 +1,24 @@
+Gem::Specification.new do |s|
+  s.name = 'logstash-filter-categoriser'
+  s.version         = '1.0.0'
+  s.licenses = ['Apache License (2.0)']
+  s.summary = "Allows quick categorisation of incoming logs based on other fields."
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+  s.authors = ["Elastic", "Phil Helliwell"]
+  s.email = 'phil.helliwell@gmail.com'
+  s.homepage = "https://github.com/kill9zombie/logstash-filter-categoriser"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "json", "~> 1.8"
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
+  s.add_development_dependency 'logstash-devutils'
+end

data/spec/filters/categoriser_rules_spec.rb

@@ -0,0 +1,80 @@
+# encoding: utf-8
+require "logger"
+require "stringio"
+require "logstash/devutils/rspec/spec_helper"
+
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# Dummy filter just so that we can load LogStash::Filters::Categoriser::Rules
+class LogStash::Filters::Categoriser < LogStash::Filters::Base
+  def register
+  end
+
+  def filter(event)
+    filter_matched(event)
+  end
+end
+
+require "logstash/filters/categoriser/rules"
+
+
+
+describe LogStash::Filters::Categoriser::Rules do
+
+  before(:each) do
+    @logger = Logger.new(STDOUT)
+    @logger.level = Logger::ERROR
+  end
+
+  it "raises an error for a missing rules file" do
+    missing_rules_file = File.expand_path("../../fixtures/missing.rules.json", __FILE__)
+
+    filter_config = LogStash::Filters::Categoriser::Rules.new(@logger)
+    expect{ filter_config.read_config(missing_rules_file) }.to raise_error(Errno::ENOENT)
+  end
+
+  it "won't load invalid JSON" do
+    missing_rules_file = File.expand_path("../../fixtures/invalid_json.rules.json", __FILE__)
+
+    filter_config = LogStash::Filters::Categoriser::Rules.new(@logger)
+    expect{ filter_config.read_config(missing_rules_file) }.to raise_error(JSON::ParserError)
+  end
+
+  it "won't run invalid rules" do
+
+    event = {"hostname" => "foo-asa-01"}
+    def event.get(field)
+      self[field]
+    end
+
+    filter_config = LogStash::Filters::Categoriser::Rules.new(@logger)
+
+    invalid_rule = StringIO.new(%q({"cisco_asa": "invalid_rule"}))
+    expect do
+      rules = filter_config.read_config(invalid_rule)
+      matched = rules.find {|type, matcher| matcher.call(event)}
+    end.to raise_error(LogStash::Filters::Categoriser::RulesError)
+
+    invalid_command = StringIO.new(%q({"cisco_asa": ["hostname", "invalid_command", "-asa-"]}))
+    expect do
+      rules = filter_config.read_config(invalid_command)
+      matched = rules.find {|type, matcher| matcher.call(event)}
+    end.to raise_error(LogStash::Filters::Categoriser::RulesError)
+
+  end
+
+  it "returns false for missing fields" do
+    event = {"hostname" => "foo-asa-01"}
+    def event.get(field)
+      self[field]
+    end
+
+    filter_config = LogStash::Filters::Categoriser::Rules.new(@logger)
+
+    missing_field_rule = StringIO.new(%q({"cisco_asa": ["alice", "contains", "-asa-"]}))
+    expect(
+      filter_config.read_config(missing_field_rule).any? {|type, matcher| matcher.call(event)}
+    ).to be false
+  end
+end

data/spec/filters/categoriser_spec.rb

@@ -0,0 +1,80 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/filters/categoriser"
+
+describe LogStash::Filters::Categoriser do
+  describe "basic rules" do
+    test_rules = File.expand_path("../../fixtures/test.rules.json", __FILE__)
+
+    config <<-CONFIG
+      filter {
+        categoriser {
+          rules_file => "#{test_rules}"
+          target => "device_type"
+        }
+      }
+    CONFIG
+
+    # Simple match
+    sample({"hostname" => "foo-asa-01"}) do
+      expect(subject.get("device_type")).to eq('cisco_asa')
+    end
+
+    # "or"
+    sample({"hostname" => "foo-dc-01"}) do
+      expect(subject.get("device_type")).to eq('windows')
+    end
+    sample({"hostname" => "foo-sql-01"}) do
+      expect(subject.get("device_type")).to eq('windows')
+    end
+
+    # "and"
+    sample({"hostname" => "web01", "program" => "httpd"}) do
+      expect(subject.get("device_type")).to eq('web_servers')
+    end
+
+    # "and" "not"
+    sample({"hostname" => "foo-bigip-01", "message" => "test"}) do
+      expect(subject.get("device_type")).to eq('bigip')
+    end
+    sample({"hostname" => "foo-bigip-01", "message" => "elephant test"}) do
+      expect(subject.get("device_type")).to eq('unknown')
+    end
+  end
+
+  describe "config defaults" do
+    test_rules = File.expand_path("../../fixtures/test.rules.json", __FILE__)
+
+    config <<-CONFIG
+      filter {
+        categoriser {
+          rules_file => "#{test_rules}"
+        }
+      }
+    CONFIG
+
+    sample({"hostname" => "unmatched"}) do
+      expect(subject.get("category")).to eq('unknown')
+    end
+  end
+
+  describe "config" do
+    test_rules = File.expand_path("../../fixtures/test.rules.json", __FILE__)
+
+    # Once more with definition
+    config <<-CONFIG
+      filter {
+        categoriser {
+          rules_file => "#{test_rules}"
+          target => "alice"
+          default_category => "bob"
+        }
+      }
+    CONFIG
+
+    sample({"hostname" => "unmatched"}) do
+      expect(subject.get("alice")).to eq('bob')
+    end
+  end
+
+end

data/spec/filters/example_spec.rb

@@ -0,0 +1,20 @@
+# encoding: utf-8
+require 'spec_helper'
+require "logstash/filters/example"
+
+describe LogStash::Filters::Example do
+  describe "Set to Hello World" do
+    let(:config) do <<-CONFIG
+      filter {
+        example {
+          message => "Hello World"
+        }
+      }
+    CONFIG
+    end
+
+    sample("message" => "some text") do
+      expect(subject.get("message")).to eq('Hello World')
+    end
+  end
+end

data/spec/fixtures/invalid_json.rules.json

@@ -0,0 +1,3 @@
+{
+  "cisco_asa": ["hostname", "contains", "-asa-"],
+}

data/spec/fixtures/test.rules.json

@@ -0,0 +1,15 @@
+{
+  "cisco_asa": ["hostname", "contains", "-asa-"],
+  "windows": [
+    "or", [
+      ["hostname", "contains", "-sql-"],
+      ["hostname", "contains", "-dc-"]]],
+  "bigip": [
+    "and", [
+      ["hostname", "contains", "-bigip-"],
+      ["not", ["message", "contains", "elephant"]]]],
+  "web_servers": [
+    "and", [
+      ["hostname", "starts_with", "web"],
+      ["program", "eq", "httpd"]]]
+}

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"

metadata

@@ -0,0 +1,114 @@
+--- !ruby/object:Gem::Specification
+name: logstash-filter-categoriser
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Elastic
+- Phil Helliwell
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2019-01-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  name: json
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
+email: phil.helliwell@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- DEVELOPER.md
+- Gemfile
+- LICENSE
+- NOTICE.TXT
+- README.md
+- lib/logstash/filters/categoriser.rb
+- lib/logstash/filters/categoriser/config.rb
+- lib/logstash/filters/categoriser/rules.rb
+- lib/logstash/filters/example.rb
+- logstash-filter-categoriser.gemspec
+- spec/filters/categoriser_rules_spec.rb
+- spec/filters/categoriser_spec.rb
+- spec/filters/example_spec.rb
+- spec/fixtures/invalid_json.rules.json
+- spec/fixtures/test.rules.json
+- spec/spec_helper.rb
+homepage: https://github.com/kill9zombie/logstash-filter-categoriser
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: filter
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.11
+signing_key:
+specification_version: 4
+summary: Allows quick categorisation of incoming logs based on other fields.
+test_files:
+- spec/filters/categoriser_rules_spec.rb
+- spec/filters/categoriser_spec.rb
+- spec/filters/example_spec.rb
+- spec/fixtures/invalid_json.rules.json
+- spec/fixtures/test.rules.json
+- spec/spec_helper.rb