logstash-filter-schema_check 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: df4d2d0e171441ff522ca54efa4ae55cfedd9bf722e1c35a2b89282e1a5138e0
+  data.tar.gz: 1247588c62c88bccc3075ff5c5396e9b9397719d5a796dc2bb19e1711c0ebddb
+SHA512:
+  metadata.gz: 6ab3ef261ed8e59097a6824616e73e394ce798f50f36aeb7ec8e8249b63af5b1444b8d9d822e1f182549ddc9ede72fe374bf2cdc7a48d74996a81e5818b763d7
+  data.tar.gz: 639675dfe6fc2250dc8102f13bada82683c8187a0f9ee7504200f16a52a5287efc00a970e849ffdcc5c1ac1110cda3b83f59e8eb837f603c0bb656b559e4f62c

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+## 0.1.0
+  - Initial Generation of logstash-filter-schema_check

data/CONTRIBUTORS

@@ -0,0 +1,24 @@
+The following is a list of people who have contributed ideas, code, bug
+reports, or in general have helped logstash along its way.
+
+Contributors and maintainers:
+* Aaron Mildenstein (untergeek)
+* Darren Foo (stonith)
+* Jordan Sissel (jordansissel)
+* Kurt Hurtado (kurtado)
+* Paul Czarkowski (paulczar)
+* Pier-Hugues Pellerin (ph)
+* Richard Pijnenburg (electrical)
+* Suyog Rao (suyograo)
+* piavlo
+
+
+Target Team:
+* Justin Bovee (thehybridtechnician)
+* Andrew Orr (andreworr)
+* Thao Vo (tnvo)
+
+Note: If you've sent us patches, bug reports, or otherwise contributed to
+Logstash, and you aren't on the list above and want to be, please let us know
+and we'll make sure you're here. Contributions from folks like you are what make
+open source awesome.

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gemspec
+

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (C) 2018 Target Brands, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,116 @@
+# Schema Check Filter for Logstash
+
+* This is a plugin for [Logstash](https://github.com/elastic/logstash).
+* This plugin have to be installed on top of the Logstash core pipeline. It is not a stand-alone program.
+
+[![Target’s CFC-Open-Source Slack](https://cfc-slack-inv.herokuapp.com/badge.svg?colorA=155799&colorB=159953)](https://cfc-slack-inv.herokuapp.com/)
+
+
+## Installing
+
+This plugin is to be used with the Logstash system.  It is used to check whether an event is compliant with a provided schema.
+
+## Installing plugin
+
+```sh
+bin/logstash-plugin install logstash-filter-schema_check
+```
+
+### Example Configuration
+```ruby
+input {
+  stdin {}
+}
+
+filter {
+  schema_check {
+      schema_path => "test.json"
+
+      }
+    }
+  }
+}
+
+```
+
+### JSON Schema Example
+
+```json
+{
+  "type": "object",
+  "properties": {
+    "message": {
+      "oneOf":[
+        {
+          "format":"ipv4",
+          "type":"string"
+        },
+        {
+          "format":"ipv6",
+          "type":"string"
+        }
+      ]
+    }
+  }
+}
+```
+
+### Example run and output
+`logstash -e 'input { stdin {} } filter { schema_check { schema_path => "test.json" } } output { stdout {} }'`
+
+```ruby
+ping
+{
+       "message" => "ping",
+      "@version" => "1",
+          "tags" => [
+        [0] "schema_invalid"
+    ],
+    "@timestamp" => 2018-08-06T15:49:19.021Z
+}
+192.168.10.1
+{
+       "message" => "192.168.10.1",
+      "@version" => "1",
+          "tags" => [
+        [0] "schema_valid"
+    ],
+    "@timestamp" => 2018-08-06T15:49:19.021Z
+}
+
+```
+
+### Example run with debug and output
+`logstash -e 'input { stdin {} } filter { schema_check { schema_path => "test.json" debug_output => true } } output { stdout {} }'`
+
+```ruby
+ping
+{
+          "message" => "ping",
+    "schema_errors" => [
+        [0] "The property '#/message' of type String did not match any of the required schemas. The schema specific errors were:\n\n- oneOf #0:\n    - The property '#/message' must be a valid IPv4 address\n- oneOf #1:\n    - The property '#/message' must be a valid IPv6 address"
+    ],
+       "@timestamp" => 2018-08-06T15:53:07.835Z,
+         "@version" => "1"
+}
+```
+
+## Configuration Parameters
+
+### schema
+This is schema provided by hash in the configuration instead of a separate file.
+This isn't as recommended as schemas can get very large, very quickly.
+
+### schema_path
+Path to file with schema
+
+### refresh_interval
+Set refresh interval for reading json schema file for updates
+
+### debug_output
+Enable debug output.  This prints validation failures in an array under the
+field schema_errors.
+
+
+Logstash is a registered trademark of Elasticsearch BV.
+

data/lib/logstash/filters/schema_check.rb

@@ -0,0 +1,161 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+require "json"
+require "json-schema"
+
+java_import 'java.util.concurrent.locks.ReentrantReadWriteLock'
+
+class LogStash::Filters::SchemaCheck < LogStash::Filters::Base
+  # configure this filter from your Logstash config.
+  #
+  # filter {
+  #   schema_check {
+  #     schema_path => "test.json"
+  #   }
+  # }
+  #
+  # {
+  #   "type": "object",
+  #   "required": ["message","@version","@timestamp","host"],
+  #   "properties": {
+  #     "message": {
+  #       "oneOf":[
+  #         {
+  #           "format":"ipv4",
+  #           "type":"string"
+  #         },
+  #         {
+  #           "format":"ipv6",
+  #           "type":"string"
+  #         }
+  #       ]
+  #     }
+  #   }
+  # }
+  config_name "schema_check"
+
+  # Setup json schema in ruby hash inline logstash config
+  config :schema, :validate => :string, :default => ""
+  # Provide path to the json schema to use
+  config :schema_path, :validate => :path
+  # Set refresh interval for reading json schema file for updates
+  config :refresh_interval, :validate => :number, :default => 300
+  # Enable json-schema strict checking
+  config :strict, :validate => :boolean, :default => false
+  # JSON-Schema fragment option
+  config :fragment, :validate => :string
+  # JSON-Schema validate schema option
+  config :validate_schema, :validate => :boolean, :default => false
+  # Enable debug
+  config :debug_output, :validate => :boolean, :default => false
+  # Schema failures output field
+  config :failures_field, :validate => :string, :default => "schema_errors"
+  # Enable Schema Error Message
+  config :tag_on_failure, :validate => :array, :default => ["_schemacheckfailure"]
+
+  public
+  def register
+    rw_lock = java.util.concurrent.locks.ReentrantReadWriteLock.new
+    @read_lock = rw_lock.readLock
+    @write_lock = rw_lock.writeLock
+
+    if @schema_path && !@schema.empty?
+      raise LogStash::ConfigurationError, I18n.t(
+        "logstash.agent.configuration.invalid_plugin_register",
+        :plugin => "filter",
+        :type => "schema_check",
+        :error => "The configuration options 'schema' and 'schema_path' are mutually exclusive"
+      )
+    end
+
+   if @schema_path
+     @next_refresh = Time.now + @refresh_interval
+     raise_exception = true
+     lock_for_write { load_schema(raise_exception) }
+   end
+
+    @logger.debug? and @logger.debug("#{self.class.name}: schema - ", :schema => @schema)
+  end # def register
+
+  public
+  def filter(event)
+    if @schema_path
+      if needs_refresh?
+        lock_for_write do
+          if needs_refresh?
+            load_schema
+            @next_refresh = Time.now + @refresh_interval
+            @logger.info("refreshing schema file")
+          end
+        end
+      end
+    end
+
+    begin
+      event_obj = event.to_hash
+      if @debug_output
+        output = JSON::Validator.fully_validate(@schema, event_obj, :strict => @strict, :fragment => @fragment, :validate_schema => validate_schema)
+        unless output.empty?
+          @tag_on_failure.each {|tag| event.tag(tag)}
+          event.set(@failures_field, output)
+        end
+      else
+        unless JSON::Validator.validate(@schema, event_obj, :strict => @strict, :fragment => @fragment, :validate_schema => validate_schema)
+          @tag_on_failure.each {|tag| event.tag(tag)}
+        end
+      end
+      filter_matched(event)
+    rescue Exception => e
+      @logger.error("Something went wrong when checking schema", :exception => e, :event => event)
+    end
+  end # def filter
+
+  private
+  def lock_for_read
+    @read_lock.lock
+    begin
+      yield
+    ensure
+      @read_lock.unlock
+    end
+  end
+
+  private
+  def lock_for_write
+    @write_lock.lock
+    begin
+      yield
+    ensure
+      @write_lock.unlock
+    end
+  end
+
+  private
+  def load_schema(raise_exception=false)
+    if !File.exists?(@schema_path)
+      @logger.warn("schema file read failure", :path => @schema_path)
+      return
+    end
+    refresh_schema!(JSON.parse(File.read(@schema_path)))
+  end
+
+  private
+  def refresh_schema!(data)
+    @schema = data
+  end
+
+  def loading_exception(e, raise_exception=false)
+    msg = "#{self.class.name}: #{e.message} when loading schema file at #{@schema_path}"
+    if raise_exception
+      raise RuntimeError.new(msg)
+    else
+      @logger.warn("#{msg}, continuing with old schema", :schema_path => @schema_path)
+    end
+  end
+
+  private
+  def needs_refresh?
+    @next_refresh < Time.now
+  end
+end # class LogStash::Filters::JsonSchema

data/logstash-filter-schema_check.gemspec

@@ -0,0 +1,26 @@
+Gem::Specification.new do |s|
+  s.name          = "logstash-filter-schema_check"
+  s.version       = "0.1.0"
+  s.licenses        = ['Apache License (2.0)']
+  s.summary       = 'A Logstash plugin to validate JSON schemas'
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+  s.homepage      = "http://www.elastic.co/guide/en/logstash/current/index.html"
+  s.authors       = ["Elastic", "Justin Bovee"]
+  s.email         = 'jbovee@thehybridtech.com'
+  s.require_paths = ['lib']
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "json-schema"
+  s.add_runtime_dependency 'logstash-core-plugin-api', '>= 2.1.1', '<= 2.99'
+
+  s.add_development_dependency 'rspec'
+  s.add_development_dependency 'logstash-devutils', "~> 1.3", ">= 1.3.1"
+end

data/spec/filters/schema_check_spec.rb

@@ -0,0 +1,64 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/filters/schema_check"
+
+describe LogStash::Filters::SchemaCheck do
+
+  context 'Inline Schema' do
+    describe "Test schema provided inline" do
+      config <<-CONFIG
+        filter {
+          schema_check {
+            schema => '{"type":"object","properties":{"ip_address":{"oneOf":[{"format":"ipv4","type":"string"},{"format":"ipv6","type":"string"}]}}}'
+          }
+        }
+      CONFIG
+
+      sample("ip_address" => "I_AM_NOT_AN_IP") do
+        insist { subject.get("tags") } == ["_schemacheckfailure"]
+      end
+      sample("ip_address" => "10.0.0.1") do
+        insist { subject.get("tags") } != ["_schemacheckfailure"]
+      end
+    end
+
+    describe "Test schema debug functionality inline" do
+      config <<-CONFIG
+        filter {
+          schema_check {
+            schema => '{"type":"object","properties":{"ip_address":{"oneOf":[{"format":"ipv4","type":"string"},{"format":"ipv6","type":"string"}]}}}'
+            debug_output => true
+          }
+        }
+      CONFIG
+
+      sample("ip_address" => "I_AM_NOT_AN_IP") do
+        insist { subject.get("tags") } == ["_schemacheckfailure"]
+        insist { subject.get("schema_errors").to be(Array) }
+      end
+      sample("ip_address" => "10.0.0.1") do
+        insist { subject.get("tags") } != ["_schemacheckfailure"]
+      end
+    end
+  end
+
+  context  "File Schema" do
+    schema_file = File.join(File.dirname(__FILE__), "..", "fixtures", "test.json")
+    describe "Test schema provided from file" do
+      config <<-CONFIG
+        filter {
+          schema_check {
+            "schema_path" => "#{schema_file}"
+          }
+        }
+      CONFIG
+
+      sample("ip_address" => "I_AM_NOT_AN_IP") do
+        insist { subject.get("tags") } == ["_schemacheckfailure"]
+      end
+      sample("ip_address" => "10.0.0.1") do
+        insist { subject.get("tags") } != ["_schemacheckfailure"]
+      end
+    end
+  end
+end

data/spec/fixtures/test.json

@@ -0,0 +1,17 @@
+{
+  "type": "object",
+  "properties": {
+    "ip_address": {
+      "oneOf":[
+        {
+          "format":"ipv4",
+          "type":"string"
+        },
+        {
+          "format":"ipv6",
+          "type":"string"
+        }
+      ]
+    }
+  }
+}

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"

metadata

@@ -0,0 +1,129 @@
+--- !ruby/object:Gem::Specification
+name: logstash-filter-schema_check
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Elastic
+- Justin Bovee
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2018-10-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: json-schema
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.1
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.1
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: rspec
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
+email: jbovee@thehybridtech.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/filters/schema_check.rb
+- logstash-filter-schema_check.gemspec
+- spec/filters/schema_check_spec.rb
+- spec/fixtures/test.json
+- spec/spec_helper.rb
+homepage: http://www.elastic.co/guide/en/logstash/current/index.html
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: filter
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.7.6
+signing_key:
+specification_version: 4
+summary: A Logstash plugin to validate JSON schemas
+test_files:
+- spec/filters/schema_check_spec.rb
+- spec/fixtures/test.json
+- spec/spec_helper.rb