logstash-filter-bytes 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: fadc30e8ea226b625ef2bfe6e7ad5ba0c661ad1ca183d949ca9c008a19924606
+  data.tar.gz: 5fcdc6c8d58ddedceab3266a9b0ce338a93849f8b3541386532c36dd6c7913cf
+SHA512:
+  metadata.gz: 4ea531657bcf341a9b9eded3c5eb98551b1212e43a1a3b6e41085eb5a515d5e7c0b4d3c8f4c29dd329f6acdbdc1db5bdd0135d744119a3340649daaf29fe6ab2
+  data.tar.gz: fa00c9bf52bd727098b3e5a1c8d54c82c7b9276e4907564af67d2ae400d62aed37aaa76c4d2633666160bebc75319f20e884a2d8e085665a31c9ecddc7241e47

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+## 0.1.0
+  - Plugin created with the logstash plugin generator

data/CONTRIBUTORS

@@ -0,0 +1,10 @@
+The following is a list of people who have contributed ideas, code, bug
+reports, or in general have helped logstash along its way.
+
+Contributors:
+* Shaunak Kashyap (ycombinator)
+
+Note: If you've sent us patches, bug reports, or otherwise contributed to
+Logstash, and you aren't on the list above and want to be, please let us know
+and we'll make sure you're here. Contributions from folks like you are what make
+open source awesome.

data/Gemfile

@@ -0,0 +1,10 @@
+source 'https://rubygems.org'
+gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/LICENSE

@@ -0,0 +1,12 @@
+Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and

data/README.md

@@ -0,0 +1,86 @@
+# Logstash Plugin
+
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
+
+It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
+
+## Documentation
+
+Logstash provides infrastructure to automatically build documentation for this plugin. We provide a template file, index.asciidoc, where you can add documentation. The contents of this file will be converted into html and then placed with other plugin documentation in a [central location](http://www.elastic.co/guide/en/logstash/current/).
+
+- For formatting config examples, you can use the asciidoc `[source,json]` directive
+- For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide
+
+## Need Help?
+
+Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/logstash discussion forum.
+
+## Developing
+
+### 1. Plugin Developement and Testing
+
+#### Code
+- To get started, you'll need JRuby with the Bundler gem installed.
+
+- Create a new plugin or clone and existing from the GitHub [logstash-plugins](https://github.com/logstash-plugins) organization. We also provide [example plugins](https://github.com/logstash-plugins?query=example).
+
+- Install dependencies
+```sh
+bundle install
+```
+
+#### Test
+
+- Update your dependencies
+
+```sh
+bundle install
+```
+
+- Run tests
+
+```sh
+bundle exec rspec
+```
+
+### 2. Running your unpublished Plugin in Logstash
+
+#### 2.1 Run in a local Logstash clone
+
+- Edit Logstash `Gemfile` and add the local plugin path, for example:
+```ruby
+gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
+```
+- Install plugin
+```sh
+bin/logstash-plugin install --no-verify
+```
+- Run Logstash with your plugin
+```sh
+bin/logstash -e 'filter {awesome {}}'
+```
+At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
+
+#### 2.2 Run in an installed Logstash
+
+You can use the same **2.1** method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory or you can build the gem and install it using:
+
+- Build your plugin gem
+```sh
+gem build logstash-filter-awesome.gemspec
+```
+- Install the plugin from the Logstash home
+```sh
+bin/logstash-plugin install /your/local/plugin/logstash-filter-awesome.gem
+```
+- Start Logstash and proceed to test the plugin
+
+## Contributing
+
+All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
+
+Programming is not a required skill. Whatever you've seen about open source and maintainers or community members  saying "send patches or die" - you will not see that here.
+
+It is more important to the community that you are able to contribute.
+
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.

data/lib/logstash/filters/bytes.rb

@@ -0,0 +1,116 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# This filter will parse a given string as a computer storage
+# value (e.g. "123 MB" or "5.3GB") and add a new numeric field with
+# size in bytes
+#
+class LogStash::Filters::Bytes < LogStash::Filters::Base
+
+  PREFIX_POWERS = {
+    'k' => 1, # 1 kilobyte = 1024 ^ 1 bytes
+    'm' => 2, # 1 megabyte = 1024 ^ 2 bytes
+    'g' => 3, # 1 gigabyte = 1024 ^ 3 bytes
+    't' => 4, # 1 terabyte = 1024 ^ 4 bytes
+    'p' => 5, # 1 petabyte = 1024 ^ 5 bytes
+    'e' => 6, # 1 exabyte = 1024 ^ 6 bytes
+  }.freeze
+
+  DIGIT_GROUP_SEPARATORS = " _,."
+
+  # Setting the config_name here is required. This is how you
+  # configure this filter from your Logstash config.
+  #
+  # filter {
+  #   bytes {
+  #     source => "123 MB"
+  #     target => "num_bytes"
+  #   }
+  # }
+  #
+  config_name "bytes"
+  
+  # Source string, e.g. "123 MB", "5.3GB", etc.
+  config :source, :validate => :string, :default => "message"
+
+  # Target field name
+  config :target, :validate => :string
+
+  # Conversion method, either "binary" (1K = 1024B) or "metric" (1K = 1000B)
+  config :conversion_method, :validate => [ "binary", "metric" ], :default => "binary"
+
+  # Decimal separator
+  config :decimal_separator, :validate => [ ".", "," ] , :default => "."
+
+  # Append values to the `tags` field when there has been no
+  # successful match
+  config :tag_on_failure, :validate => :array, :default => ["_bytesparsefailure"]
+
+  private
+  def normalize_number(number)
+    number
+      .tr("^0-9#{@decimal_separator}", '')
+      .tr(@decimal_separator, '.')
+  end
+
+  public
+  def register
+    # Add instance variables 
+  end # def register
+
+  public
+  def filter(event)
+
+    source = event.get(@source)
+
+    if !source
+      @tag_on_failure.each{|tag| event.tag(tag)}
+      return
+    end
+    source.strip!
+
+    # Parse the source into the number part (e.g. 123),
+    # the unit prefix part (e.g. M), and the unit suffix part (e.g. B)
+    match = source.match(/^([0-9#{DIGIT_GROUP_SEPARATORS}#{@decimal_separator}]*)\s*([kKmMgGtTpPeE]?)([bB]?)$/)
+    if !match
+      @tag_on_failure.each{|tag| event.tag(tag)}
+      return
+    end
+
+    number, prefix, suffix = match.captures
+
+    # Flag error if more than one decimal separator is found
+    num_decimals = number.count(@decimal_separator)
+    if num_decimals > 1
+      @tag_on_failure.each{|tag| event.tag(tag)}
+      return
+    end
+
+    number = normalize_number(number.strip)
+    if number == ''
+      @tag_on_failure.each{|tag| event.tag(tag)}
+      return
+    end
+
+    if suffix == ''
+      suffix = 'B'
+    end
+
+    # Convert the number to bytes
+    result = number.to_f
+    if prefix != ''
+      if @conversion_method == 'binary'
+        result *= (1024 ** PREFIX_POWERS[prefix.downcase])
+      else
+        result *= (1000 ** PREFIX_POWERS[prefix.downcase])
+      end
+    end
+    result = result.round
+
+    event.set(@target, result)
+
+    # filter_matched should go in the last line of our successful code
+    filter_matched(event)
+  end # def filter
+end # class LogStash::Filters::Bytes

data/logstash-filter-bytes.gemspec

@@ -0,0 +1,23 @@
+Gem::Specification.new do |s|
+  s.name          = 'logstash-filter-bytes'
+  s.version       = '1.0.0'
+  s.licenses      = ['Apache-2.0']
+  s.summary       = "This filter parses strings representing computer storage sizes into numeric bytes"
+  s.description   = 'This filter parses strings representing computer storage sizes (e.g. "123 MB" or "6.3GB") into numeric bytes (12'
+  s.homepage      = 'https://github.com/logstash-plugins/logstash-filter-bytes'
+  s.authors       = ['Elastic']
+  s.email         = 'info@elastic.co'
+  s.require_paths = ['lib']
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
+  s.add_development_dependency 'logstash-devutils'
+end

data/spec/filters/bytes_spec.rb

@@ -0,0 +1,276 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/filters/bytes"
+
+describe LogStash::Filters::Bytes do
+  let(:config) do <<-CONFIG
+    filter {
+      bytes {
+        target => dest
+        tag_on_failure => [ "boom", "_bytesparsefailure" ]
+      }
+    }
+  CONFIG
+  end
+
+  describe "empty" do
+    sample("") do
+      expect(subject.get('tags')).to eq(["boom", "_bytesparsefailure"])
+    end
+  end
+
+  describe "garbage" do
+    sample("abcdef") do
+      expect(subject.get('tags')).to eq(["boom", "_bytesparsefailure"])
+    end
+  end
+
+  describe "no number, only units" do
+    sample("mb") do
+      expect(subject.get('tags')).to eq(["boom", "_bytesparsefailure"])
+    end
+  end
+
+  describe "no units" do
+    sample("0") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq(0)
+    end
+
+    sample("123") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq(123)
+    end
+
+    sample("12.3") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq(12)
+    end
+  end
+
+  describe "from bytes" do
+    sample("32.8B") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq(33)
+    end
+  end
+
+  describe "from kilobytes" do
+    sample("32kB") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq(32 * 1024)
+    end
+
+    sample("32KB") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq(32 * 1024)
+    end
+  end
+
+  describe "from megabytes" do
+    sample("32mB") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq(32 * 1024 * 1024)
+    end
+
+    sample("32mb") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq(32 * 1024 * 1024)
+    end
+
+    sample("32MB") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq(32 * 1024 * 1024)
+    end
+  end
+
+  describe "from gigabytes" do
+    sample("32gB") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq(32 * 1024 * 1024 * 1024)
+    end
+
+    sample("32gb") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq(32 * 1024 * 1024 * 1024)
+    end
+
+    sample("32GB") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq(32 * 1024 * 1024 * 1024)
+    end
+  end
+
+  describe "from terabytes" do
+    sample("32tB") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq(32 * 1024 * 1024 * 1024 * 1024)
+    end
+
+    sample("32tb") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq(32 * 1024 * 1024 * 1024 * 1024)
+    end
+
+    sample("32TB") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq(32 * 1024 * 1024 * 1024 * 1024)
+    end
+  end
+
+  describe "from petabytes" do
+    sample("32pB") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq(32 * 1024 * 1024 * 1024 * 1024 * 1024)
+    end
+
+    sample("32pb") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq(32 * 1024 * 1024 * 1024 * 1024 * 1024)
+    end
+
+    sample("32PB") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq(32 * 1024 * 1024 * 1024 * 1024 * 1024)
+    end
+  end
+
+  describe "from exabytes" do
+    sample("32eB") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq(32 * 1024 * 1024 * 1024 * 1024 * 1024 * 1024)
+    end
+
+    sample("32eb") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq(32 * 1024 * 1024 * 1024 * 1024 * 1024 * 1024)
+    end
+
+    sample("32EB") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq(32 * 1024 * 1024 * 1024 * 1024 * 1024 * 1024)
+    end
+  end
+
+  describe "with spaces" do
+    sample("32 mb") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq(32 * 1024 * 1024)
+    end
+
+    sample("32\tmb") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq(32 * 1024 * 1024)
+    end
+  end
+
+  describe "using metric conversion system" do
+    let(:config) do <<-CONFIG
+      filter {
+        bytes {
+          target => dest
+          conversion_method => metric
+        }
+      }
+    CONFIG
+    end
+
+    sample("32 mb") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq(32 * 1000 * 1000)
+    end
+  end
+
+  describe "no decimal separator" do
+    sample("3 mb") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq(3 * 1024 * 1024)
+    end      
+
+    sample("3,124 mb") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq(3124 * 1024 * 1024)
+    end      
+  end
+
+  describe "digits only to right of decimal separator" do
+    sample(".3124 mb") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq((0.3124 * 1024 * 1024).round)
+    end
+  end
+
+  describe "digits only to left of decimal separator" do
+    sample("3. mb") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq(3 * 1024 * 1024)
+    end      
+
+    sample("3,124. mb") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq(3124 * 1024 * 1024)
+    end      
+  end
+
+  describe "digits on both sides of decimal separator" do
+    sample("3.56 mb") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq((3.56 * 1024 * 1024).round)
+    end      
+
+    sample("3,124.56 mb") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq((3124.56 * 1024 * 1024).round)
+    end      
+  end
+
+  describe "non-default decimal separator" do
+    let(:config) do <<-CONFIG
+      filter {
+        bytes {
+          target => dest
+          decimal_separator => ','
+        }
+      }
+    CONFIG
+    end
+
+    sample("3,9 mb") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq((3.9 * 1024 * 1024).round)
+    end
+
+    sample("1.9 kb") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq((19 * 1024).round)
+    end
+
+    sample("3.124,56 mb") do
+      expect(subject).to include("dest")
+      expect(subject.get('dest')).to eq((3124.56 * 1024 * 1024).round)
+    end
+  end
+
+  describe "with two decimal separators" do
+    sample("1.000.000 mb") do
+      expect(subject.get('tags')).to eq(["boom", "_bytesparsefailure"])
+    end
+
+    context "non-default decimal separator" do
+      let(:config) do <<-CONFIG
+        filter {
+          bytes {
+            target => dest
+            decimal_separator => ','
+            tag_on_failure => [ "boom", "_bytesparsefailure" ]
+          }
+        }
+      CONFIG
+      end
+
+      sample("1,000,000 mb") do
+        expect(subject.get('tags')).to eq(["boom", "_bytesparsefailure"])
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,84 @@
+--- !ruby/object:Gem::Specification
+name: logstash-filter-bytes
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Elastic
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2018-04-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This filter parses strings representing computer storage sizes (e.g.
+  "123 MB" or "6.3GB") into numeric bytes (12
+email: info@elastic.co
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/filters/bytes.rb
+- logstash-filter-bytes.gemspec
+- spec/filters/bytes_spec.rb
+homepage: https://github.com/logstash-plugins/logstash-filter-bytes
+licenses:
+- Apache-2.0
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: filter
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.13
+signing_key:
+specification_version: 4
+summary: This filter parses strings representing computer storage sizes into numeric
+  bytes
+test_files:
+- spec/filters/bytes_spec.rb