logstash-filter-tld 0.1.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 91de2c319b63ec14eb09f213d34891b0714000b6
4
+ data.tar.gz: cbec7901e34db15005ae32a5655ebdca16ba9503
5
+ SHA512:
6
+ metadata.gz: e042323730698fb53f299aefdce93cf936614a8eca472b0caf96dc2d1d02718504dd607e77abf465c75baef85c65816f9697cf677eac7ce9c956324a23f6c643
7
+ data.tar.gz: 38aa85f696e3807593130d18ce65f622da761c48c5c4b8961647a9b04c5d87d5f6d20d27aa71cf8e50d716b8b0173aebb28c85e305aa079458bdd9b240f8a095
data/DEVELOPER.md ADDED
@@ -0,0 +1,2 @@
1
+ # logstash-filter-tld
2
+ Example filter plugin. This should help bootstrap your effort to write your own filter plugin!
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source 'https://rubygems.org'
2
+ gemspec
3
+ gem "logstash", :github => "elasticsearch/logstash", :branch => "1.5"
data/LICENSE ADDED
@@ -0,0 +1,13 @@
1
+ Copyright (c) 2012-2015 Elasticsearch <http://www.elasticsearch.org>
2
+
3
+ Licensed under the Apache License, Version 2.0 (the "License");
4
+ you may not use this file except in compliance with the License.
5
+ You may obtain a copy of the License at
6
+
7
+ http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+ Unless required by applicable law or agreed to in writing, software
10
+ distributed under the License is distributed on an "AS IS" BASIS,
11
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ See the License for the specific language governing permissions and
13
+ limitations under the License.
data/README.md ADDED
@@ -0,0 +1,95 @@
1
+ # Logstash Plugin
2
+
3
+ This is a plugin for [Logstash](https://github.com/elasticsearch/logstash).
4
+
5
+ It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
6
+
7
+ ## Documentation
8
+
9
+ Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elasticsearch.org/guide/en/logstash/current/).
10
+
11
+ - For formatting code or config example, you can use the asciidoc `[source,ruby]` directive
12
+ - For more asciidoc formatting tips, see the excellent reference here https://github.com/elasticsearch/docs#asciidoc-guide
13
+
14
+ ## Need Help?
15
+
16
+ Need help? Try #logstash on freenode IRC or the logstash-users@googlegroups.com mailing list.
17
+
18
+ ## Developing
19
+
20
+ ### 1. Plugin Developement and Testing
21
+
22
+ #### Code
23
+ - To get started, you'll need JRuby with the Bundler gem installed.
24
+
25
+ - Create a new plugin or clone and existing from the GitHub [logstash-plugins](https://github.com/logstash-plugins) organization.
26
+
27
+ - Install dependencies
28
+ ```sh
29
+ bundle install
30
+ ```
31
+
32
+ #### Test
33
+
34
+ ```sh
35
+ bundle exec rspec
36
+ ```
37
+
38
+ The Logstash code required to run the tests/specs is specified in the `Gemfile` by the line similar to:
39
+ ```ruby
40
+ gem "logstash", :github => "elasticsearch/logstash", :branch => "1.5"
41
+ ```
42
+ To test against another version or a local Logstash, edit the `Gemfile` to specify an alternative location, for example:
43
+ ```ruby
44
+ gem "logstash", :github => "elasticsearch/logstash", :ref => "master"
45
+ ```
46
+ ```ruby
47
+ gem "logstash", :path => "/your/local/logstash"
48
+ ```
49
+
50
+ Then update your dependencies and run your tests:
51
+
52
+ ```sh
53
+ bundle install
54
+ bundle exec rspec
55
+ ```
56
+
57
+ ### 2. Running your unpublished Plugin in Logstash
58
+
59
+ #### 2.1 Run in a local Logstash clone
60
+
61
+ - Edit Logstash `tools/Gemfile` and add the local plugin path, for example:
62
+ ```ruby
63
+ gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
64
+ ```
65
+ - Update Logstash dependencies
66
+ ```sh
67
+ rake vendor:gems
68
+ ```
69
+ - Run Logstash with your plugin
70
+ ```sh
71
+ bin/logstash -e 'filter {awesome {}}'
72
+ ```
73
+ At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
74
+
75
+ #### 2.2 Run in an installed Logstash
76
+
77
+ - Build your plugin gem
78
+ ```sh
79
+ gem build logstash-filter-awesome.gemspec
80
+ ```
81
+ - Install the plugin from the Logstash home
82
+ ```sh
83
+ bin/plugin install /your/local/plugin/logstash-filter-awesome.gem
84
+ ```
85
+ - Start Logstash and proceed to test the plugin
86
+
87
+ ## Contributing
88
+
89
+ All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
90
+
91
+ Programming is not a required skill. Whatever you've seen about open source and maintainers or community members saying "send patches or die" - you will not see that here.
92
+
93
+ It is more important to me that you are able to contribute.
94
+
95
+ For more information about contributing, see the [CONTRIBUTING](https://github.com/elasticsearch/logstash/blob/master/CONTRIBUTING.md) file.
data/Rakefile ADDED
@@ -0,0 +1 @@
1
+ require "logstash/devutils/rake"
data/example.conf ADDED
@@ -0,0 +1,42 @@
1
+ input {
2
+
3
+ # This works
4
+ generator {
5
+ message => "www.google.com"
6
+ count => 1
7
+ type => "test1"
8
+ }
9
+
10
+ # This will fail
11
+ generator {
12
+ message => "google.co.ukf"
13
+ count => 1
14
+ type => "test1"
15
+ }
16
+
17
+ # Shows using the source option
18
+ generator {
19
+ message => "The domain is www.google.com"
20
+ count => 1
21
+ type => "test2"
22
+ }
23
+
24
+ }
25
+
26
+ filter {
27
+ if [type] == "test1" {
28
+ tld {}
29
+ }
30
+ if [type] == "test2" {
31
+ grok {
32
+ match => [ "message", "The domain is %{HOST:domain}" ]
33
+ }
34
+ tld {
35
+ source => "domain"
36
+ }
37
+ }
38
+ }
39
+
40
+ output {
41
+ stdout { codec => rubydebug }
42
+ }
@@ -0,0 +1,54 @@
1
+ # encoding: utf-8
2
+ require "logstash/filters/base"
3
+ require "logstash/namespace"
4
+
5
+ # This example filter will replace the contents of the default
6
+ # message field with whatever you specify in the configuration.
7
+ #
8
+ # It is only intended to be used as an example.
9
+ class LogStash::Filters::Tld < LogStash::Filters::Base
10
+
11
+ # Setting the config_name here is required. This is how you
12
+ # configure this filter from your Logstash config.
13
+ #
14
+ # filter {
15
+ # example {
16
+ # message => "My message..."
17
+ # }
18
+ # }
19
+ #
20
+ config_name "tld"
21
+ milestone 1
22
+
23
+ # The source field to parse
24
+ config :source, :validate => :string, :default => "message"
25
+
26
+ # The target field to place all the data
27
+ config :target, :validate => :string, :default => "tld"
28
+
29
+ public
30
+ def register
31
+ # Add instance variables
32
+ require 'public_suffix'
33
+ end # def register
34
+
35
+ public
36
+ def filter(event)
37
+
38
+ if @source and PublicSuffix.valid?(event[@source])
39
+ domain = PublicSuffix.parse(event[@source])
40
+ # Replace the event message with our message as configured in the
41
+ # config file.
42
+ event[@target] = Hash.new
43
+ event[@target]['tld'] = domain.tld
44
+ event[@target]['sld'] = domain.sld
45
+ event[@target]['trd'] = domain.trd
46
+ event[@target]['domain'] = domain.domain
47
+ event[@target]['subdomain'] = domain.subdomain
48
+
49
+ # filter_matched should go in the last line of our successful code
50
+ filter_matched(event)
51
+
52
+ end
53
+ end # def filter
54
+ end # class LogStash::Filters::Example
@@ -0,0 +1,25 @@
1
+ Gem::Specification.new do |s|
2
+ s.name = 'logstash-filter-tld'
3
+ s.version = '0.1.1'
4
+ s.licenses = ['Apache License (2.0)']
5
+ s.summary = "This example filter replaces the contents of the message field with the specified value."
6
+ s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
7
+ s.authors = ["Elasticsearch"]
8
+ s.email = 'info@elasticsearch.com'
9
+ s.homepage = "http://www.elasticsearch.org/guide/en/logstash/current/index.html"
10
+ s.require_paths = ["lib"]
11
+
12
+ # Files
13
+ s.files = `git ls-files`.split($\)
14
+ # Tests
15
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
16
+
17
+ # Special flag to let us know this is actually a logstash plugin
18
+ s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
19
+
20
+ # Gem dependencies
21
+ s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
22
+ s.add_runtime_dependency 'public_suffix', '=1.4.6'
23
+
24
+ s.add_development_dependency 'logstash-devutils'
25
+ end
@@ -0,0 +1,53 @@
1
+ require "logstash/devutils/rspec/spec_helper"
2
+ require "logstash/filters/tld"
3
+
4
+ describe LogStash::Filters::Tld do
5
+ describe "Set to TLD" do
6
+ config <<-CONFIG
7
+ filter {
8
+ tld {
9
+ }
10
+ }
11
+ CONFIG
12
+
13
+ #{
14
+ # "message" => "google.com",
15
+ # "@version" => "1",
16
+ # "@timestamp" => "2015-01-22T17:33:19.669Z",
17
+ # "host" => "homer",
18
+ # "sequence" => 0,
19
+ # "tld" => {
20
+ # "tld" => "com",
21
+ # "sld" => "google",
22
+ # "trd" => nil,
23
+ # "domain" => "google.com",
24
+ # "subdomain" => nil
25
+ # }
26
+
27
+
28
+ sample("message" => "google.com") do
29
+ insist { subject["tld"]["tld"] } == "com"
30
+ insist { subject["tld"]["sld"] } == "google"
31
+ insist { subject["tld"]["trd"] } == nil
32
+ insist { subject["tld"]["domain"] } == "google.com"
33
+ insist { subject["tld"]["subdomain"] } == nil
34
+ end
35
+
36
+ sample("message" => "google.co.uk") do
37
+ insist { subject["tld"]["tld"] } == "co.uk"
38
+ insist { subject["tld"]["sld"] } == "google"
39
+ insist { subject["tld"]["trd"] } == nil
40
+ insist { subject["tld"]["domain"] } == "google.co.uk"
41
+ insist { subject["tld"]["subdomain"] } == nil
42
+ end
43
+
44
+ sample("message" => "www.google.com") do
45
+ insist { subject["tld"]["tld"] } == "com"
46
+ insist { subject["tld"]["sld"] } == "google"
47
+ insist { subject["tld"]["trd"] } == "www"
48
+ insist { subject["tld"]["domain"] } == "google.com"
49
+ insist { subject["tld"]["subdomain"] } == "www.google.com"
50
+ end
51
+
52
+ end
53
+ end
metadata ADDED
@@ -0,0 +1,103 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: logstash-filter-tld
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.1
5
+ platform: ruby
6
+ authors:
7
+ - Elasticsearch
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2015-01-22 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: logstash
15
+ version_requirements: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - '>='
18
+ - !ruby/object:Gem::Version
19
+ version: 1.4.0
20
+ - - <
21
+ - !ruby/object:Gem::Version
22
+ version: 2.0.0
23
+ requirement: !ruby/object:Gem::Requirement
24
+ requirements:
25
+ - - '>='
26
+ - !ruby/object:Gem::Version
27
+ version: 1.4.0
28
+ - - <
29
+ - !ruby/object:Gem::Version
30
+ version: 2.0.0
31
+ prerelease: false
32
+ type: :runtime
33
+ - !ruby/object:Gem::Dependency
34
+ name: public_suffix
35
+ version_requirements: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - '='
38
+ - !ruby/object:Gem::Version
39
+ version: 1.4.6
40
+ requirement: !ruby/object:Gem::Requirement
41
+ requirements:
42
+ - - '='
43
+ - !ruby/object:Gem::Version
44
+ version: 1.4.6
45
+ prerelease: false
46
+ type: :runtime
47
+ - !ruby/object:Gem::Dependency
48
+ name: logstash-devutils
49
+ version_requirements: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - '>='
52
+ - !ruby/object:Gem::Version
53
+ version: '0'
54
+ requirement: !ruby/object:Gem::Requirement
55
+ requirements:
56
+ - - '>='
57
+ - !ruby/object:Gem::Version
58
+ version: '0'
59
+ prerelease: false
60
+ type: :development
61
+ description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
62
+ email: info@elasticsearch.com
63
+ executables: []
64
+ extensions: []
65
+ extra_rdoc_files: []
66
+ files:
67
+ - DEVELOPER.md
68
+ - Gemfile
69
+ - LICENSE
70
+ - README.md
71
+ - Rakefile
72
+ - example.conf
73
+ - lib/logstash/filters/tld.rb
74
+ - logstash-filter-tld.gemspec
75
+ - spec/filters/tld_spec.rb
76
+ homepage: http://www.elasticsearch.org/guide/en/logstash/current/index.html
77
+ licenses:
78
+ - Apache License (2.0)
79
+ metadata:
80
+ logstash_plugin: 'true'
81
+ logstash_group: filter
82
+ post_install_message:
83
+ rdoc_options: []
84
+ require_paths:
85
+ - lib
86
+ required_ruby_version: !ruby/object:Gem::Requirement
87
+ requirements:
88
+ - - '>='
89
+ - !ruby/object:Gem::Version
90
+ version: '0'
91
+ required_rubygems_version: !ruby/object:Gem::Requirement
92
+ requirements:
93
+ - - '>='
94
+ - !ruby/object:Gem::Version
95
+ version: '0'
96
+ requirements: []
97
+ rubyforge_project:
98
+ rubygems_version: 2.1.9
99
+ signing_key:
100
+ specification_version: 4
101
+ summary: This example filter replaces the contents of the message field with the specified value.
102
+ test_files:
103
+ - spec/filters/tld_spec.rb