logstash-filter-hashid 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
 - data/Gemfile +11 -0
 - data/LICENSE +13 -0
 - data/NOTICE.TXT +5 -0
 - data/README.md +90 -0
 - data/docs/index.asciidoc +110 -0
 - data/lib/logstash/filters/hashid.rb +113 -0
 - data/logstash-filter-hashid.gemspec +25 -0
 - data/spec/filters/hashid_spec.rb +258 -0
 - metadata +88 -0
 
    
        checksums.yaml
    ADDED
    
    | 
         @@ -0,0 +1,7 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            ---
         
     | 
| 
      
 2 
     | 
    
         
            +
            SHA1:
         
     | 
| 
      
 3 
     | 
    
         
            +
              metadata.gz: a1abd6841318a26deeb00351c2050707b607be49
         
     | 
| 
      
 4 
     | 
    
         
            +
              data.tar.gz: 2e2257a2af30a646f75617a1fd3fb647dc3b3253
         
     | 
| 
      
 5 
     | 
    
         
            +
            SHA512:
         
     | 
| 
      
 6 
     | 
    
         
            +
              metadata.gz: fc8fbf19c41a01c52c8b90b5c4ea2ea3d57863e66aa7ed1995268bdf82ca7b8f7167099d5112b021dc2da7704e1a86e8f8e4f7b8817ffc0e7a2e137d40975661
         
     | 
| 
      
 7 
     | 
    
         
            +
              data.tar.gz: fd1e85f8949fb0f69d7646cb71a39ba52dd8ee8aab2b207a45366d014f71c78ab110c6a2dd17c42f13f814a09d85410b3424d895240d7cdb96254f17e38e3388
         
     | 
    
        data/Gemfile
    ADDED
    
    | 
         @@ -0,0 +1,11 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            source 'https://rubygems.org'
         
     | 
| 
      
 2 
     | 
    
         
            +
             
     | 
| 
      
 3 
     | 
    
         
            +
            gemspec
         
     | 
| 
      
 4 
     | 
    
         
            +
             
     | 
| 
      
 5 
     | 
    
         
            +
            logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
         
     | 
| 
      
 6 
     | 
    
         
            +
            use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
         
     | 
| 
      
 7 
     | 
    
         
            +
             
     | 
| 
      
 8 
     | 
    
         
            +
            if Dir.exist?(logstash_path) && use_logstash_source
         
     | 
| 
      
 9 
     | 
    
         
            +
              gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
         
     | 
| 
      
 10 
     | 
    
         
            +
              gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
         
     | 
| 
      
 11 
     | 
    
         
            +
            end
         
     | 
    
        data/LICENSE
    ADDED
    
    | 
         @@ -0,0 +1,13 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
         
     | 
| 
      
 2 
     | 
    
         
            +
             
     | 
| 
      
 3 
     | 
    
         
            +
            Licensed under the Apache License, Version 2.0 (the "License");
         
     | 
| 
      
 4 
     | 
    
         
            +
            you may not use this file except in compliance with the License.
         
     | 
| 
      
 5 
     | 
    
         
            +
            You may obtain a copy of the License at
         
     | 
| 
      
 6 
     | 
    
         
            +
             
     | 
| 
      
 7 
     | 
    
         
            +
                http://www.apache.org/licenses/LICENSE-2.0
         
     | 
| 
      
 8 
     | 
    
         
            +
             
     | 
| 
      
 9 
     | 
    
         
            +
            Unless required by applicable law or agreed to in writing, software
         
     | 
| 
      
 10 
     | 
    
         
            +
            distributed under the License is distributed on an "AS IS" BASIS,
         
     | 
| 
      
 11 
     | 
    
         
            +
            WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
         
     | 
| 
      
 12 
     | 
    
         
            +
            See the License for the specific language governing permissions and
         
     | 
| 
      
 13 
     | 
    
         
            +
            limitations under the License.
         
     | 
    
        data/NOTICE.TXT
    ADDED
    
    
    
        data/README.md
    ADDED
    
    | 
         @@ -0,0 +1,90 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            # Logstash Plugin
         
     | 
| 
      
 2 
     | 
    
         
            +
             
     | 
| 
      
 3 
     | 
    
         
            +
            [](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-fingerprint-unit/)
         
     | 
| 
      
 5 
     | 
    
         
            +
             
     | 
| 
      
 6 
     | 
    
         
            +
            This is a plugin for [Logstash](https://github.com/elastic/logstash).
         
     | 
| 
      
 7 
     | 
    
         
            +
             
     | 
| 
      
 8 
     | 
    
         
            +
            It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
         
     | 
| 
      
 9 
     | 
    
         
            +
             
     | 
| 
      
 10 
     | 
    
         
            +
            ## Documentation
         
     | 
| 
      
 11 
     | 
    
         
            +
             
     | 
| 
      
 12 
     | 
    
         
            +
            Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/).
         
     | 
| 
      
 13 
     | 
    
         
            +
             
     | 
| 
      
 14 
     | 
    
         
            +
            - For formatting code or config example, you can use the asciidoc `[source,ruby]` directive
         
     | 
| 
      
 15 
     | 
    
         
            +
            - For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide
         
     | 
| 
      
 16 
     | 
    
         
            +
             
     | 
| 
      
 17 
     | 
    
         
            +
            ## Need Help?
         
     | 
| 
      
 18 
     | 
    
         
            +
             
     | 
| 
      
 19 
     | 
    
         
            +
            Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/logstash discussion forum.
         
     | 
| 
      
 20 
     | 
    
         
            +
             
     | 
| 
      
 21 
     | 
    
         
            +
            ## Developing
         
     | 
| 
      
 22 
     | 
    
         
            +
             
     | 
| 
      
 23 
     | 
    
         
            +
            ### 1. Plugin Developement and Testing
         
     | 
| 
      
 24 
     | 
    
         
            +
             
     | 
| 
      
 25 
     | 
    
         
            +
            #### Code
         
     | 
| 
      
 26 
     | 
    
         
            +
            - To get started, you'll need JRuby with the Bundler gem installed.
         
     | 
| 
      
 27 
     | 
    
         
            +
             
     | 
| 
      
 28 
     | 
    
         
            +
            - Create a new plugin or clone and existing from the GitHub [logstash-plugins](https://github.com/logstash-plugins) organization. We also provide [example plugins](https://github.com/logstash-plugins?query=example).
         
     | 
| 
      
 29 
     | 
    
         
            +
             
     | 
| 
      
 30 
     | 
    
         
            +
            - Install dependencies
         
     | 
| 
      
 31 
     | 
    
         
            +
            ```sh
         
     | 
| 
      
 32 
     | 
    
         
            +
            bundle install
         
     | 
| 
      
 33 
     | 
    
         
            +
            ```
         
     | 
| 
      
 34 
     | 
    
         
            +
             
     | 
| 
      
 35 
     | 
    
         
            +
            #### Test
         
     | 
| 
      
 36 
     | 
    
         
            +
             
     | 
| 
      
 37 
     | 
    
         
            +
            - Update your dependencies
         
     | 
| 
      
 38 
     | 
    
         
            +
             
     | 
| 
      
 39 
     | 
    
         
            +
            ```sh
         
     | 
| 
      
 40 
     | 
    
         
            +
            bundle install
         
     | 
| 
      
 41 
     | 
    
         
            +
            ```
         
     | 
| 
      
 42 
     | 
    
         
            +
             
     | 
| 
      
 43 
     | 
    
         
            +
            - Run tests
         
     | 
| 
      
 44 
     | 
    
         
            +
             
     | 
| 
      
 45 
     | 
    
         
            +
            ```sh
         
     | 
| 
      
 46 
     | 
    
         
            +
            bundle exec rspec
         
     | 
| 
      
 47 
     | 
    
         
            +
            ```
         
     | 
| 
      
 48 
     | 
    
         
            +
             
     | 
| 
      
 49 
     | 
    
         
            +
            ### 2. Running your unpublished Plugin in Logstash
         
     | 
| 
      
 50 
     | 
    
         
            +
             
     | 
| 
      
 51 
     | 
    
         
            +
            #### 2.1 Run in a local Logstash clone
         
     | 
| 
      
 52 
     | 
    
         
            +
             
     | 
| 
      
 53 
     | 
    
         
            +
            - Edit Logstash `Gemfile` and add the local plugin path, for example:
         
     | 
| 
      
 54 
     | 
    
         
            +
            ```ruby
         
     | 
| 
      
 55 
     | 
    
         
            +
            gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
         
     | 
| 
      
 56 
     | 
    
         
            +
            ```
         
     | 
| 
      
 57 
     | 
    
         
            +
            - Install plugin
         
     | 
| 
      
 58 
     | 
    
         
            +
            ```sh
         
     | 
| 
      
 59 
     | 
    
         
            +
            bin/plugin install --no-verify
         
     | 
| 
      
 60 
     | 
    
         
            +
            ```
         
     | 
| 
      
 61 
     | 
    
         
            +
            - Run Logstash with your plugin
         
     | 
| 
      
 62 
     | 
    
         
            +
            ```sh
         
     | 
| 
      
 63 
     | 
    
         
            +
            bin/logstash -e 'filter {awesome {}}'
         
     | 
| 
      
 64 
     | 
    
         
            +
            ```
         
     | 
| 
      
 65 
     | 
    
         
            +
            At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
         
     | 
| 
      
 66 
     | 
    
         
            +
             
     | 
| 
      
 67 
     | 
    
         
            +
            #### 2.2 Run in an installed Logstash
         
     | 
| 
      
 68 
     | 
    
         
            +
             
     | 
| 
      
 69 
     | 
    
         
            +
            You can use the same **2.1** method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory or you can build the gem and install it using:
         
     | 
| 
      
 70 
     | 
    
         
            +
             
     | 
| 
      
 71 
     | 
    
         
            +
            - Build your plugin gem
         
     | 
| 
      
 72 
     | 
    
         
            +
            ```sh
         
     | 
| 
      
 73 
     | 
    
         
            +
            gem build logstash-filter-awesome.gemspec
         
     | 
| 
      
 74 
     | 
    
         
            +
            ```
         
     | 
| 
      
 75 
     | 
    
         
            +
            - Install the plugin from the Logstash home
         
     | 
| 
      
 76 
     | 
    
         
            +
            ```sh
         
     | 
| 
      
 77 
     | 
    
         
            +
            bin/plugin install /your/local/plugin/logstash-filter-awesome.gem
         
     | 
| 
      
 78 
     | 
    
         
            +
            ```
         
     | 
| 
      
 79 
     | 
    
         
            +
            - Start Logstash and proceed to test the plugin
         
     | 
| 
      
 80 
     | 
    
         
            +
             
     | 
| 
      
 81 
     | 
    
         
            +
            ## Contributing
         
     | 
| 
      
 82 
     | 
    
         
            +
             
     | 
| 
      
 83 
     | 
    
         
            +
            All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
         
     | 
| 
      
 84 
     | 
    
         
            +
             
     | 
| 
      
 85 
     | 
    
         
            +
            Programming is not a required skill. Whatever you've seen about open source and maintainers or community members  saying "send patches or die" - you will not see that here.
         
     | 
| 
      
 86 
     | 
    
         
            +
             
     | 
| 
      
 87 
     | 
    
         
            +
            It is more important to the community that you are able to contribute.
         
     | 
| 
      
 88 
     | 
    
         
            +
             
     | 
| 
      
 89 
     | 
    
         
            +
            For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.
         
     | 
| 
      
 90 
     | 
    
         
            +
             
     | 
    
        data/docs/index.asciidoc
    ADDED
    
    | 
         @@ -0,0 +1,110 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            :plugin: hashid
         
     | 
| 
      
 2 
     | 
    
         
            +
            :type: filter
         
     | 
| 
      
 3 
     | 
    
         
            +
             
     | 
| 
      
 4 
     | 
    
         
            +
            ///////////////////////////////////////////
         
     | 
| 
      
 5 
     | 
    
         
            +
            START - GENERATED VARIABLES, DO NOT EDIT!
         
     | 
| 
      
 6 
     | 
    
         
            +
            ///////////////////////////////////////////
         
     | 
| 
      
 7 
     | 
    
         
            +
            :version: %VERSION%
         
     | 
| 
      
 8 
     | 
    
         
            +
            :release_date: %RELEASE_DATE%
         
     | 
| 
      
 9 
     | 
    
         
            +
            :changelog_url: %CHANGELOG_URL%
         
     | 
| 
      
 10 
     | 
    
         
            +
            :include_path: ../../../../logstash/docs/include
         
     | 
| 
      
 11 
     | 
    
         
            +
            ///////////////////////////////////////////
         
     | 
| 
      
 12 
     | 
    
         
            +
            END - GENERATED VARIABLES, DO NOT EDIT!
         
     | 
| 
      
 13 
     | 
    
         
            +
            ///////////////////////////////////////////
         
     | 
| 
      
 14 
     | 
    
         
            +
             
     | 
| 
      
 15 
     | 
    
         
            +
            [id="plugins-{type}-{plugin}"]
         
     | 
| 
      
 16 
     | 
    
         
            +
             
     | 
| 
      
 17 
     | 
    
         
            +
            === Hashid filter plugin
         
     | 
| 
      
 18 
     | 
    
         
            +
             
     | 
| 
      
 19 
     | 
    
         
            +
            include::{include_path}/plugin_header.asciidoc[]
         
     | 
| 
      
 20 
     | 
    
         
            +
             
     | 
| 
      
 21 
     | 
    
         
            +
            ==== Description
         
     | 
| 
      
 22 
     | 
    
         
            +
             
     | 
| 
      
 23 
     | 
    
         
            +
            This filter allow you to generate predictable, string encoded hashed keys 
         
     | 
| 
      
 24 
     | 
    
         
            +
            based om event contents and timestamp. This can be used to avoid getting 
         
     | 
| 
      
 25 
     | 
    
         
            +
            duplicate records indexed into Elasticsearch.
         
     | 
| 
      
 26 
     | 
    
         
            +
             
     | 
| 
      
 27 
     | 
    
         
            +
            Hashed keys to be generated based on full or partial hashes and
         
     | 
| 
      
 28 
     | 
    
         
            +
            has the ability to prefix these keys based on the event timestamp in order
         
     | 
| 
      
 29 
     | 
    
         
            +
            to make then largely ordered by timestamp, which tend to lead to increased
         
     | 
| 
      
 30 
     | 
    
         
            +
            indexing performance for event based use cases where data is being indexed
         
     | 
| 
      
 31 
     | 
    
         
            +
            in near real time.
         
     | 
| 
      
 32 
     | 
    
         
            +
             
     | 
| 
      
 33 
     | 
    
         
            +
            When used with the timestamp prefix enabled, it should ideally be run after 
         
     | 
| 
      
 34 
     | 
    
         
            +
            the date filter has run and populated the @timestamp field.
         
     | 
| 
      
 35 
     | 
    
         
            +
             
     | 
| 
      
 36 
     | 
    
         
            +
            [id="plugins-{type}s-{plugin}-options"]
         
     | 
| 
      
 37 
     | 
    
         
            +
            ==== Hashid Filter Configuration Options
         
     | 
| 
      
 38 
     | 
    
         
            +
             
     | 
| 
      
 39 
     | 
    
         
            +
            This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
         
     | 
| 
      
 40 
     | 
    
         
            +
             
     | 
| 
      
 41 
     | 
    
         
            +
            [cols="<,<,<",options="header",]
         
     | 
| 
      
 42 
     | 
    
         
            +
            |=======================================================================
         
     | 
| 
      
 43 
     | 
    
         
            +
            |Setting |Input type|Required
         
     | 
| 
      
 44 
     | 
    
         
            +
            | <<plugins-{type}s-{plugin}-add_timestamp_prefix>> |<<boolean,boolean>>|No
         
     | 
| 
      
 45 
     | 
    
         
            +
            | <<plugins-{type}s-{plugin}-hash_bytes_used>> |<<number,number>>|No
         
     | 
| 
      
 46 
     | 
    
         
            +
            | <<plugins-{type}s-{plugin}-key>> |<<string,string>>|No
         
     | 
| 
      
 47 
     | 
    
         
            +
            | <<plugins-{type}s-{plugin}-method>> |<<string,string>>, one of `["SHA1", "SHA256", "SHA384", "SHA512", "MD5"]`|No
         
     | 
| 
      
 48 
     | 
    
         
            +
            | <<plugins-{type}s-{plugin}-source>> |<<array,array>>|No
         
     | 
| 
      
 49 
     | 
    
         
            +
            | <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
         
     | 
| 
      
 50 
     | 
    
         
            +
            |=======================================================================
         
     | 
| 
      
 51 
     | 
    
         
            +
             
     | 
| 
      
 52 
     | 
    
         
            +
            Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
         
     | 
| 
      
 53 
     | 
    
         
            +
            filter plugins.
         
     | 
| 
      
 54 
     | 
    
         
            +
             
     | 
| 
      
 55 
     | 
    
         
            +
             
         
     | 
| 
      
 56 
     | 
    
         
            +
             
     | 
| 
      
 57 
     | 
    
         
            +
            [id="plugins-{type}s-{plugin}-add_timestamp_prefix"]
         
     | 
| 
      
 58 
     | 
    
         
            +
            ===== `add_timestamp_prefix` 
         
     | 
| 
      
 59 
     | 
    
         
            +
             
     | 
| 
      
 60 
     | 
    
         
            +
              * Value type is <<boolean,boolean>>
         
     | 
| 
      
 61 
     | 
    
         
            +
              * Default value is `true`
         
     | 
| 
      
 62 
     | 
    
         
            +
             
     | 
| 
      
 63 
     | 
    
         
            +
            Use the timestamp to generate an ID prefix
         
     | 
| 
      
 64 
     | 
    
         
            +
             
     | 
| 
      
 65 
     | 
    
         
            +
            [id="plugins-{type}s-{plugin}-hash_bytes_used"]
         
     | 
| 
      
 66 
     | 
    
         
            +
            ===== `hash_bytes_used` 
         
     | 
| 
      
 67 
     | 
    
         
            +
             
     | 
| 
      
 68 
     | 
    
         
            +
              * Value type is <<number,number>>
         
     | 
| 
      
 69 
     | 
    
         
            +
              * There is no default value for this setting.
         
     | 
| 
      
 70 
     | 
    
         
            +
             
     | 
| 
      
 71 
     | 
    
         
            +
            If full hash generated is not to be used, this parameter specifies how many bytes that should be used
         
     | 
| 
      
 72 
     | 
    
         
            +
            If not specified, the full hash will be used
         
     | 
| 
      
 73 
     | 
    
         
            +
             
     | 
| 
      
 74 
     | 
    
         
            +
            [id="plugins-{type}s-{plugin}-key"]
         
     | 
| 
      
 75 
     | 
    
         
            +
            ===== `key` 
         
     | 
| 
      
 76 
     | 
    
         
            +
             
     | 
| 
      
 77 
     | 
    
         
            +
              * Value type is <<string,string>>
         
     | 
| 
      
 78 
     | 
    
         
            +
              * Default value is `"hashid"`
         
     | 
| 
      
 79 
     | 
    
         
            +
             
     | 
| 
      
 80 
     | 
    
         
            +
            Encryption key to be used when generating cryptographic hashes
         
     | 
| 
      
 81 
     | 
    
         
            +
             
     | 
| 
      
 82 
     | 
    
         
            +
            [id="plugins-{type}s-{plugin}-method"]
         
     | 
| 
      
 83 
     | 
    
         
            +
            ===== `method` 
         
     | 
| 
      
 84 
     | 
    
         
            +
             
     | 
| 
      
 85 
     | 
    
         
            +
              * Value can be any of: `SHA1`, `SHA256`, `SHA384`, `SHA512`, `MD5`
         
     | 
| 
      
 86 
     | 
    
         
            +
              * Default value is `"MD5"`
         
     | 
| 
      
 87 
     | 
    
         
            +
             
     | 
| 
      
 88 
     | 
    
         
            +
            Hash function to use
         
     | 
| 
      
 89 
     | 
    
         
            +
             
     | 
| 
      
 90 
     | 
    
         
            +
            [id="plugins-{type}s-{plugin}-source"]
         
     | 
| 
      
 91 
     | 
    
         
            +
            ===== `source` 
         
     | 
| 
      
 92 
     | 
    
         
            +
             
     | 
| 
      
 93 
     | 
    
         
            +
              * Value type is <<array,array>>
         
     | 
| 
      
 94 
     | 
    
         
            +
              * Default value is `["message"]`
         
     | 
| 
      
 95 
     | 
    
         
            +
             
     | 
| 
      
 96 
     | 
    
         
            +
            Source field(s) to base the hash calculation on
         
     | 
| 
      
 97 
     | 
    
         
            +
             
     | 
| 
      
 98 
     | 
    
         
            +
            [id="plugins-{type}s-{plugin}-target"]
         
     | 
| 
      
 99 
     | 
    
         
            +
            ===== `target` 
         
     | 
| 
      
 100 
     | 
    
         
            +
             
     | 
| 
      
 101 
     | 
    
         
            +
              * Value type is <<string,string>>
         
     | 
| 
      
 102 
     | 
    
         
            +
              * Default value is `"hashid"`
         
     | 
| 
      
 103 
     | 
    
         
            +
             
     | 
| 
      
 104 
     | 
    
         
            +
            Target field.
         
     | 
| 
      
 105 
     | 
    
         
            +
            Will overwrite current value of a field if it exists.
         
     | 
| 
      
 106 
     | 
    
         
            +
             
     | 
| 
      
 107 
     | 
    
         
            +
             
     | 
| 
      
 108 
     | 
    
         
            +
             
     | 
| 
      
 109 
     | 
    
         
            +
            [id="plugins-{type}s-{plugin}-common-options"]
         
     | 
| 
      
 110 
     | 
    
         
            +
            include::{include_path}/{type}.asciidoc[]
         
     | 
| 
         @@ -0,0 +1,113 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            # encoding: utf-8
         
     | 
| 
      
 2 
     | 
    
         
            +
            require "logstash/filters/base"
         
     | 
| 
      
 3 
     | 
    
         
            +
            require "logstash/namespace"
         
     | 
| 
      
 4 
     | 
    
         
            +
            require "openssl"
         
     | 
| 
      
 5 
     | 
    
         
            +
             
     | 
| 
      
 6 
     | 
    
         
            +
            # This filter allow you to generate predictable, string encoded hashed keys 
         
     | 
| 
      
 7 
     | 
    
         
            +
            # based om event contents and timestamp. This can be used to avoid getting 
         
     | 
| 
      
 8 
     | 
    
         
            +
            # duplicate records indexed into Elasticsearch.
         
     | 
| 
      
 9 
     | 
    
         
            +
            #
         
     | 
| 
      
 10 
     | 
    
         
            +
            # Hashed keys to be generated based on full or partial hashes and
         
     | 
| 
      
 11 
     | 
    
         
            +
            # has the ability to prefix these keys based on the event timestamp in order
         
     | 
| 
      
 12 
     | 
    
         
            +
            # to make then largely ordered by timestamp, which tend to lead to increased
         
     | 
| 
      
 13 
     | 
    
         
            +
            # indexing performance for event based use cases where data is being indexed
         
     | 
| 
      
 14 
     | 
    
         
            +
            # in near real time.
         
     | 
| 
      
 15 
     | 
    
         
            +
            #
         
     | 
| 
      
 16 
     | 
    
         
            +
            # When used with the timestamp prefix enabled, it should ideally be run after 
         
     | 
| 
      
 17 
     | 
    
         
            +
            # the date filter has run and populated the @timestamp field.
         
     | 
| 
      
 18 
     | 
    
         
            +
            class LogStash::Filters::Hashid < LogStash::Filters::Base
         
     | 
| 
      
 19 
     | 
    
         
            +
              config_name "hashid"
         
     | 
| 
      
 20 
     | 
    
         
            +
             
     | 
| 
      
 21 
     | 
    
         
            +
              # Source field(s) to base the hash calculation on
         
     | 
| 
      
 22 
     | 
    
         
            +
              config :source, :validate => :array, :default => ['message']
         
     | 
| 
      
 23 
     | 
    
         
            +
             
     | 
| 
      
 24 
     | 
    
         
            +
              # Target field.
         
     | 
| 
      
 25 
     | 
    
         
            +
              # Will overwrite current value of a field if it exists.
         
     | 
| 
      
 26 
     | 
    
         
            +
              config :target, :validate => :string, :default => 'hashid'
         
     | 
| 
      
 27 
     | 
    
         
            +
             
     | 
| 
      
 28 
     | 
    
         
            +
              # Encryption key to be used when generating cryptographic hashes
         
     | 
| 
      
 29 
     | 
    
         
            +
              config :key, :validate => :string, :default => 'hashid'
         
     | 
| 
      
 30 
     | 
    
         
            +
             
     | 
| 
      
 31 
     | 
    
         
            +
              # Hash function to use
         
     | 
| 
      
 32 
     | 
    
         
            +
              config :method, :validate => ['SHA1', 'SHA256', 'SHA384', 'SHA512', 'MD5'], :default => 'MD5'
         
     | 
| 
      
 33 
     | 
    
         
            +
             
     | 
| 
      
 34 
     | 
    
         
            +
              # If full hash generated is not to be used, this parameter specifies how many bytes that should be used
         
     | 
| 
      
 35 
     | 
    
         
            +
              # If not specified, the full hash will be used
         
     | 
| 
      
 36 
     | 
    
         
            +
              config :hash_bytes_used, :validate => :number
         
     | 
| 
      
 37 
     | 
    
         
            +
             
     | 
| 
      
 38 
     | 
    
         
            +
              # Use the timestamp to generate an ID prefix
         
     | 
| 
      
 39 
     | 
    
         
            +
              config :add_timestamp_prefix, :validate => :boolean, :default => true
         
     | 
| 
      
 40 
     | 
    
         
            +
             
     | 
| 
      
 41 
     | 
    
         
            +
              CHARS = '-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz'.chars.to_a.freeze
         
     | 
| 
      
 42 
     | 
    
         
            +
              SHIFTS = [18, 12, 6, 0].freeze
         
     | 
| 
      
 43 
     | 
    
         
            +
             
     | 
| 
      
 44 
     | 
    
         
            +
              def register
         
     | 
| 
      
 45 
     | 
    
         
            +
                # convert to symbol for faster comparisons
         
     | 
| 
      
 46 
     | 
    
         
            +
                @method = @method.to_sym
         
     | 
| 
      
 47 
     | 
    
         
            +
                @digest = select_digest(@method)
         
     | 
| 
      
 48 
     | 
    
         
            +
              end
         
     | 
| 
      
 49 
     | 
    
         
            +
             
     | 
| 
      
 50 
     | 
    
         
            +
              def filter(event)
         
     | 
| 
      
 51 
     | 
    
         
            +
                hmac = OpenSSL::HMAC.new(@key, @digest.new)
         
     | 
| 
      
 52 
     | 
    
         
            +
             
     | 
| 
      
 53 
     | 
    
         
            +
                @source.sort.each do |k|
         
     | 
| 
      
 54 
     | 
    
         
            +
                  hmac.update("|#{k}|#{event.get(k)}") 
         
     | 
| 
      
 55 
     | 
    
         
            +
                end
         
     | 
| 
      
 56 
     | 
    
         
            +
             
     | 
| 
      
 57 
     | 
    
         
            +
                hash = hmac.digest
         
     | 
| 
      
 58 
     | 
    
         
            +
             
     | 
| 
      
 59 
     | 
    
         
            +
                if !@hash_bytes_used.nil? && @hash_bytes_used > 0 && hash.length > @hash_bytes_used
         
     | 
| 
      
 60 
     | 
    
         
            +
                  hash = hash[(-1 * @hash_bytes_used), @hash_bytes_used]
         
     | 
| 
      
 61 
     | 
    
         
            +
                end
         
     | 
| 
      
 62 
     | 
    
         
            +
             
     | 
| 
      
 63 
     | 
    
         
            +
                epoch_array = []
         
     | 
| 
      
 64 
     | 
    
         
            +
                if @add_timestamp_prefix
         
     | 
| 
      
 65 
     | 
    
         
            +
                  epoch = event.get('@timestamp').to_i
         
     | 
| 
      
 66 
     | 
    
         
            +
                  epoch_array.push(epoch >> 24)
         
     | 
| 
      
 67 
     | 
    
         
            +
                  epoch_array.push((epoch >> 16) % 256)
         
     | 
| 
      
 68 
     | 
    
         
            +
                  epoch_array.push((epoch >> 8) % 256)
         
     | 
| 
      
 69 
     | 
    
         
            +
                  epoch_array.push(epoch % 256)
         
     | 
| 
      
 70 
     | 
    
         
            +
                end
         
     | 
| 
      
 71 
     | 
    
         
            +
             
     | 
| 
      
 72 
     | 
    
         
            +
                binary_array = epoch_array + hash.unpack('C*')
         
     | 
| 
      
 73 
     | 
    
         
            +
             
     | 
| 
      
 74 
     | 
    
         
            +
                event.set(@target, encode_to_sortable_string(binary_array).force_encoding(Encoding::UTF_8))
         
     | 
| 
      
 75 
     | 
    
         
            +
              end
         
     | 
| 
      
 76 
     | 
    
         
            +
             
     | 
| 
      
 77 
     | 
    
         
            +
              def select_digest(method)
         
     | 
| 
      
 78 
     | 
    
         
            +
                case method
         
     | 
| 
      
 79 
     | 
    
         
            +
                when :SHA1
         
     | 
| 
      
 80 
     | 
    
         
            +
                  OpenSSL::Digest::SHA1
         
     | 
| 
      
 81 
     | 
    
         
            +
                when :SHA256
         
     | 
| 
      
 82 
     | 
    
         
            +
                  OpenSSL::Digest::SHA256
         
     | 
| 
      
 83 
     | 
    
         
            +
                when :SHA384
         
     | 
| 
      
 84 
     | 
    
         
            +
                  OpenSSL::Digest::SHA384
         
     | 
| 
      
 85 
     | 
    
         
            +
                when :SHA512
         
     | 
| 
      
 86 
     | 
    
         
            +
                  OpenSSL::Digest::SHA512
         
     | 
| 
      
 87 
     | 
    
         
            +
                when :MD5
         
     | 
| 
      
 88 
     | 
    
         
            +
                  OpenSSL::Digest::MD5
         
     | 
| 
      
 89 
     | 
    
         
            +
                else
         
     | 
| 
      
 90 
     | 
    
         
            +
                  # we really should never get here
         
     | 
| 
      
 91 
     | 
    
         
            +
                  raise(LogStash::ConfigurationError, "Unknown digest for method=#{method.to_s}")
         
     | 
| 
      
 92 
     | 
    
         
            +
                end
         
     | 
| 
      
 93 
     | 
    
         
            +
              end
         
     | 
| 
      
 94 
     | 
    
         
            +
             
     | 
| 
      
 95 
     | 
    
         
            +
              def encode_to_sortable_string(data)
         
     | 
| 
      
 96 
     | 
    
         
            +
                idxes = []
         
     | 
| 
      
 97 
     | 
    
         
            +
                to_take = 0
         
     | 
| 
      
 98 
     | 
    
         
            +
                data.each_slice(3) do |part0, part1, part2|
         
     | 
| 
      
 99 
     | 
    
         
            +
                  to_take = 0
         
     | 
| 
      
 100 
     | 
    
         
            +
                  if part1.nil?
         
     | 
| 
      
 101 
     | 
    
         
            +
                    part1 = part2 = 0
         
     | 
| 
      
 102 
     | 
    
         
            +
                    to_take = 2
         
     | 
| 
      
 103 
     | 
    
         
            +
                  end
         
     | 
| 
      
 104 
     | 
    
         
            +
                  if part2.nil?
         
     | 
| 
      
 105 
     | 
    
         
            +
                    part2 = 0
         
     | 
| 
      
 106 
     | 
    
         
            +
                    to_take = 1
         
     | 
| 
      
 107 
     | 
    
         
            +
                  end
         
     | 
| 
      
 108 
     | 
    
         
            +
                  group24 = (part0 << 16) | (part1 << 8) | part2
         
     | 
| 
      
 109 
     | 
    
         
            +
                  idxes.concat(SHIFTS.map{|n| (group24 >> n) & 0x3f })
         
     | 
| 
      
 110 
     | 
    
         
            +
                end
         
     | 
| 
      
 111 
     | 
    
         
            +
                CHARS.values_at(*idxes.take(idxes.size - to_take)).join
         
     | 
| 
      
 112 
     | 
    
         
            +
              end
         
     | 
| 
      
 113 
     | 
    
         
            +
            end
         
     | 
| 
         @@ -0,0 +1,25 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            Gem::Specification.new do |s|
         
     | 
| 
      
 2 
     | 
    
         
            +
             
     | 
| 
      
 3 
     | 
    
         
            +
              s.name            = 'logstash-filter-hashid'
         
     | 
| 
      
 4 
     | 
    
         
            +
              s.version         = '0.1.2'
         
     | 
| 
      
 5 
     | 
    
         
            +
              s.licenses        = ['Apache License (2.0)']
         
     | 
| 
      
 6 
     | 
    
         
            +
              s.summary         = "This filter lets you create base64 encoded event IDs based on the event contents and timestamp using a number of hash functions."
         
     | 
| 
      
 7 
     | 
    
         
            +
              s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
         
     | 
| 
      
 8 
     | 
    
         
            +
              s.authors         = ["Christian Dahlqvist"]
         
     | 
| 
      
 9 
     | 
    
         
            +
              s.email           = 'christian@elastic.co'
         
     | 
| 
      
 10 
     | 
    
         
            +
              s.homepage        = "https://github.com/cdahlqvist/logstash-filter-hashid/blob/master/README.md"
         
     | 
| 
      
 11 
     | 
    
         
            +
              s.require_paths = ["lib"]
         
     | 
| 
      
 12 
     | 
    
         
            +
             
     | 
| 
      
 13 
     | 
    
         
            +
              # Files
         
     | 
| 
      
 14 
     | 
    
         
            +
              s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
         
     | 
| 
      
 15 
     | 
    
         
            +
             
     | 
| 
      
 16 
     | 
    
         
            +
              # Tests
         
     | 
| 
      
 17 
     | 
    
         
            +
              s.test_files = s.files.grep(%r{^(test|spec|features)/})
         
     | 
| 
      
 18 
     | 
    
         
            +
             
     | 
| 
      
 19 
     | 
    
         
            +
              # Special flag to let us know this is actually a logstash plugin
         
     | 
| 
      
 20 
     | 
    
         
            +
              s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
         
     | 
| 
      
 21 
     | 
    
         
            +
             
     | 
| 
      
 22 
     | 
    
         
            +
              # Gem dependencies
         
     | 
| 
      
 23 
     | 
    
         
            +
              s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
         
     | 
| 
      
 24 
     | 
    
         
            +
              s.add_development_dependency 'logstash-devutils'
         
     | 
| 
      
 25 
     | 
    
         
            +
            end
         
     | 
| 
         @@ -0,0 +1,258 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            # encoding: utf-8
         
     | 
| 
      
 2 
     | 
    
         
            +
            require "logstash/devutils/rspec/spec_helper"
         
     | 
| 
      
 3 
     | 
    
         
            +
            require "logstash/filters/hashid"
         
     | 
| 
      
 4 
     | 
    
         
            +
             
     | 
| 
      
 5 
     | 
    
         
            +
            describe LogStash::Filters::Hashid do
         
     | 
| 
      
 6 
     | 
    
         
            +
             
     | 
| 
      
 7 
     | 
    
         
            +
              describe 'Full MD5, no timestamp prefix' do
         
     | 
| 
      
 8 
     | 
    
         
            +
                config <<-CONFIG
         
     | 
| 
      
 9 
     | 
    
         
            +
                  filter {
         
     | 
| 
      
 10 
     | 
    
         
            +
                    hashid {
         
     | 
| 
      
 11 
     | 
    
         
            +
                      source => ['message']
         
     | 
| 
      
 12 
     | 
    
         
            +
                      method => 'MD5'
         
     | 
| 
      
 13 
     | 
    
         
            +
                      add_timestamp_prefix => false
         
     | 
| 
      
 14 
     | 
    
         
            +
                    }
         
     | 
| 
      
 15 
     | 
    
         
            +
                  }
         
     | 
| 
      
 16 
     | 
    
         
            +
                CONFIG
         
     | 
| 
      
 17 
     | 
    
         
            +
             
     | 
| 
      
 18 
     | 
    
         
            +
                sample("message" => "testmessage") do
         
     | 
| 
      
 19 
     | 
    
         
            +
                  insist { subject.get("hashid") } == 'Fpbg8CbSbOQ81JSd3HmPFk'
         
     | 
| 
      
 20 
     | 
    
         
            +
                end
         
     | 
| 
      
 21 
     | 
    
         
            +
              end
         
     | 
| 
      
 22 
     | 
    
         
            +
             
     | 
| 
      
 23 
     | 
    
         
            +
              describe '12 byte MD5, no timestamp prefix' do
         
     | 
| 
      
 24 
     | 
    
         
            +
                config <<-CONFIG
         
     | 
| 
      
 25 
     | 
    
         
            +
                  filter {
         
     | 
| 
      
 26 
     | 
    
         
            +
                    hashid {
         
     | 
| 
      
 27 
     | 
    
         
            +
                      source => ['message']
         
     | 
| 
      
 28 
     | 
    
         
            +
                      method => 'MD5'
         
     | 
| 
      
 29 
     | 
    
         
            +
                      add_timestamp_prefix => false
         
     | 
| 
      
 30 
     | 
    
         
            +
                      hash_bytes_used => 12
         
     | 
| 
      
 31 
     | 
    
         
            +
                    }
         
     | 
| 
      
 32 
     | 
    
         
            +
                  }
         
     | 
| 
      
 33 
     | 
    
         
            +
                CONFIG
         
     | 
| 
      
 34 
     | 
    
         
            +
             
     | 
| 
      
 35 
     | 
    
         
            +
                sample("message" => "testmessage") do
         
     | 
| 
      
 36 
     | 
    
         
            +
                  insist { subject.get("hashid") } == 'qSqS_gZ8GqZGA8d2'
         
     | 
| 
      
 37 
     | 
    
         
            +
                end
         
     | 
| 
      
 38 
     | 
    
         
            +
              end
         
     | 
| 
      
 39 
     | 
    
         
            +
             
     | 
| 
      
 40 
     | 
    
         
            +
              describe 'Full SHA1, no timestamp prefix' do
         
     | 
| 
      
 41 
     | 
    
         
            +
                config <<-CONFIG
         
     | 
| 
      
 42 
     | 
    
         
            +
                  filter {
         
     | 
| 
      
 43 
     | 
    
         
            +
                    hashid {
         
     | 
| 
      
 44 
     | 
    
         
            +
                      source => ['message']
         
     | 
| 
      
 45 
     | 
    
         
            +
                      method => 'SHA1'
         
     | 
| 
      
 46 
     | 
    
         
            +
                      add_timestamp_prefix => false
         
     | 
| 
      
 47 
     | 
    
         
            +
                    }
         
     | 
| 
      
 48 
     | 
    
         
            +
                  }
         
     | 
| 
      
 49 
     | 
    
         
            +
                CONFIG
         
     | 
| 
      
 50 
     | 
    
         
            +
             
     | 
| 
      
 51 
     | 
    
         
            +
                sample("message" => "testmessage") do
         
     | 
| 
      
 52 
     | 
    
         
            +
                  insist { subject.get("hashid") } == 'sOoRSauukymQT3a8q4C8FZyDncw'
         
     | 
| 
      
 53 
     | 
    
         
            +
                end
         
     | 
| 
      
 54 
     | 
    
         
            +
              end
         
     | 
| 
      
 55 
     | 
    
         
            +
             
     | 
| 
      
 56 
     | 
    
         
            +
              describe '12 byte SHA1, no timestamp prefix' do
         
     | 
| 
      
 57 
     | 
    
         
            +
                config <<-CONFIG
         
     | 
| 
      
 58 
     | 
    
         
            +
                  filter {
         
     | 
| 
      
 59 
     | 
    
         
            +
                    hashid {
         
     | 
| 
      
 60 
     | 
    
         
            +
                      source => ['message']
         
     | 
| 
      
 61 
     | 
    
         
            +
                      method => 'SHA1'
         
     | 
| 
      
 62 
     | 
    
         
            +
                      add_timestamp_prefix => false
         
     | 
| 
      
 63 
     | 
    
         
            +
                      hash_bytes_used => 12
         
     | 
| 
      
 64 
     | 
    
         
            +
                    }
         
     | 
| 
      
 65 
     | 
    
         
            +
                  }
         
     | 
| 
      
 66 
     | 
    
         
            +
                CONFIG
         
     | 
| 
      
 67 
     | 
    
         
            +
             
     | 
| 
      
 68 
     | 
    
         
            +
                sample("message" => "testmessage") do
         
     | 
| 
      
 69 
     | 
    
         
            +
                  insist { subject.get("hashid") } == 'arW8XSWIHJ8EYguE'
         
     | 
| 
      
 70 
     | 
    
         
            +
                end
         
     | 
| 
      
 71 
     | 
    
         
            +
              end
         
     | 
| 
      
 72 
     | 
    
         
            +
             
     | 
| 
      
 73 
     | 
    
         
            +
              describe 'Full SHA256, no timestamp prefix' do
         
     | 
| 
      
 74 
     | 
    
         
            +
                config <<-CONFIG
         
     | 
| 
      
 75 
     | 
    
         
            +
                  filter {
         
     | 
| 
      
 76 
     | 
    
         
            +
                    hashid {
         
     | 
| 
      
 77 
     | 
    
         
            +
                      source => ['message']
         
     | 
| 
      
 78 
     | 
    
         
            +
                      method => 'SHA256'
         
     | 
| 
      
 79 
     | 
    
         
            +
                      add_timestamp_prefix => false
         
     | 
| 
      
 80 
     | 
    
         
            +
                    }
         
     | 
| 
      
 81 
     | 
    
         
            +
                  }
         
     | 
| 
      
 82 
     | 
    
         
            +
                CONFIG
         
     | 
| 
      
 83 
     | 
    
         
            +
             
     | 
| 
      
 84 
     | 
    
         
            +
                sample("message" => "testmessage") do
         
     | 
| 
      
 85 
     | 
    
         
            +
                  insist { subject.get("hashid") } == 'UZ8NNsx-LsLjV7S6ElKKyMG_Xv274XQE_-nRv-UNgm3'
         
     | 
| 
      
 86 
     | 
    
         
            +
                end
         
     | 
| 
      
 87 
     | 
    
         
            +
              end
         
     | 
| 
      
 88 
     | 
    
         
            +
             
     | 
| 
      
 89 
     | 
    
         
            +
              describe '12 byte SHA256, no timestamp prefix' do
         
     | 
| 
      
 90 
     | 
    
         
            +
                config <<-CONFIG
         
     | 
| 
      
 91 
     | 
    
         
            +
                  filter {
         
     | 
| 
      
 92 
     | 
    
         
            +
                    hashid {
         
     | 
| 
      
 93 
     | 
    
         
            +
                      source => ['message']
         
     | 
| 
      
 94 
     | 
    
         
            +
                      method => 'SHA256'
         
     | 
| 
      
 95 
     | 
    
         
            +
                      add_timestamp_prefix => false
         
     | 
| 
      
 96 
     | 
    
         
            +
                      hash_bytes_used => 12
         
     | 
| 
      
 97 
     | 
    
         
            +
                    }
         
     | 
| 
      
 98 
     | 
    
         
            +
                  }
         
     | 
| 
      
 99 
     | 
    
         
            +
                CONFIG
         
     | 
| 
      
 100 
     | 
    
         
            +
             
     | 
| 
      
 101 
     | 
    
         
            +
                sample("message" => "testmessage") do
         
     | 
| 
      
 102 
     | 
    
         
            +
                  insist { subject.get("hashid") } == 'm0NantFBrDk6qABW'
         
     | 
| 
      
 103 
     | 
    
         
            +
                end
         
     | 
| 
      
 104 
     | 
    
         
            +
              end
         
     | 
| 
      
 105 
     | 
    
         
            +
             
     | 
| 
      
 106 
     | 
    
         
            +
              describe 'Full SHA384, no timestamp prefix' do
         
     | 
| 
      
 107 
     | 
    
         
            +
                config <<-CONFIG
         
     | 
| 
      
 108 
     | 
    
         
            +
                  filter {
         
     | 
| 
      
 109 
     | 
    
         
            +
                    hashid {
         
     | 
| 
      
 110 
     | 
    
         
            +
                      source => ['message']
         
     | 
| 
      
 111 
     | 
    
         
            +
                      method => 'SHA384'
         
     | 
| 
      
 112 
     | 
    
         
            +
                      add_timestamp_prefix => false
         
     | 
| 
      
 113 
     | 
    
         
            +
                    }
         
     | 
| 
      
 114 
     | 
    
         
            +
                  }
         
     | 
| 
      
 115 
     | 
    
         
            +
                CONFIG
         
     | 
| 
      
 116 
     | 
    
         
            +
             
     | 
| 
      
 117 
     | 
    
         
            +
                sample("message" => "testmessage") do
         
     | 
| 
      
 118 
     | 
    
         
            +
                  insist { subject.get("hashid") } == 'uvdBF5FfTA-ns_Ou6QWTTKsmOCH8T7j6691tr3N7DtZlAeDSvawhWfTAKwwJq9c2'
         
     | 
| 
      
 119 
     | 
    
         
            +
                end
         
     | 
| 
      
 120 
     | 
    
         
            +
              end
         
     | 
| 
      
 121 
     | 
    
         
            +
             
     | 
| 
      
 122 
     | 
    
         
            +
              describe '12 byte SHA384, no timestamp prefix' do
         
     | 
| 
      
 123 
     | 
    
         
            +
                config <<-CONFIG
         
     | 
| 
      
 124 
     | 
    
         
            +
                  filter {
         
     | 
| 
      
 125 
     | 
    
         
            +
                    hashid {
         
     | 
| 
      
 126 
     | 
    
         
            +
                      source => ['message']
         
     | 
| 
      
 127 
     | 
    
         
            +
                      method => 'SHA384'
         
     | 
| 
      
 128 
     | 
    
         
            +
                      add_timestamp_prefix => false
         
     | 
| 
      
 129 
     | 
    
         
            +
                      hash_bytes_used => 12
         
     | 
| 
      
 130 
     | 
    
         
            +
                    }
         
     | 
| 
      
 131 
     | 
    
         
            +
                  }
         
     | 
| 
      
 132 
     | 
    
         
            +
                CONFIG
         
     | 
| 
      
 133 
     | 
    
         
            +
             
     | 
| 
      
 134 
     | 
    
         
            +
                sample("message" => "testmessage") do
         
     | 
| 
      
 135 
     | 
    
         
            +
                  insist { subject.get("hashid") } == 'vawhWfTAKwwJq9c2'
         
     | 
| 
      
 136 
     | 
    
         
            +
                end
         
     | 
| 
      
 137 
     | 
    
         
            +
              end
         
     | 
| 
      
 138 
     | 
    
         
            +
             
     | 
| 
      
 139 
     | 
    
         
            +
              describe 'Full SHA512, no timestamp prefix' do
         
     | 
| 
      
 140 
     | 
    
         
            +
                config <<-CONFIG
         
     | 
| 
      
 141 
     | 
    
         
            +
                  filter {
         
     | 
| 
      
 142 
     | 
    
         
            +
                    hashid {
         
     | 
| 
      
 143 
     | 
    
         
            +
                      source => ['message']
         
     | 
| 
      
 144 
     | 
    
         
            +
                      method => 'SHA512'
         
     | 
| 
      
 145 
     | 
    
         
            +
                      add_timestamp_prefix => false
         
     | 
| 
      
 146 
     | 
    
         
            +
                    }
         
     | 
| 
      
 147 
     | 
    
         
            +
                  }
         
     | 
| 
      
 148 
     | 
    
         
            +
                CONFIG
         
     | 
| 
      
 149 
     | 
    
         
            +
             
     | 
| 
      
 150 
     | 
    
         
            +
                sample("message" => "testmessage") do
         
     | 
| 
      
 151 
     | 
    
         
            +
                  insist { subject.get("hashid") } == 'ODUuPJaePwJupNWp1exdZJWRrz1aLATtrD0yzOrVBce9hUkhtW266C4djjkp4kqLVU9LtlPh0IermOgJpJx9VV'
         
     | 
| 
      
 152 
     | 
    
         
            +
                end
         
     | 
| 
      
 153 
     | 
    
         
            +
              end
         
     | 
| 
      
 154 
     | 
    
         
            +
             
     | 
| 
      
 155 
     | 
    
         
            +
              describe '12 byte SHA512, no timestamp prefix' do
         
     | 
| 
      
 156 
     | 
    
         
            +
                config <<-CONFIG
         
     | 
| 
      
 157 
     | 
    
         
            +
                  filter {
         
     | 
| 
      
 158 
     | 
    
         
            +
                    hashid {
         
     | 
| 
      
 159 
     | 
    
         
            +
                      source => ['message']
         
     | 
| 
      
 160 
     | 
    
         
            +
                      method => 'SHA512'
         
     | 
| 
      
 161 
     | 
    
         
            +
                      add_timestamp_prefix => false
         
     | 
| 
      
 162 
     | 
    
         
            +
                      hash_bytes_used => 12
         
     | 
| 
      
 163 
     | 
    
         
            +
                    }
         
     | 
| 
      
 164 
     | 
    
         
            +
                  }
         
     | 
| 
      
 165 
     | 
    
         
            +
                CONFIG
         
     | 
| 
      
 166 
     | 
    
         
            +
             
     | 
| 
      
 167 
     | 
    
         
            +
                sample("message" => "testmessage") do
         
     | 
| 
      
 168 
     | 
    
         
            +
                  insist { subject.get("hashid") } == '4eo4DfU8alIKIoe1'
         
     | 
| 
      
 169 
     | 
    
         
            +
                end
         
     | 
| 
      
 170 
     | 
    
         
            +
              end
         
     | 
| 
      
 171 
     | 
    
         
            +
             
     | 
| 
      
 172 
     | 
    
         
            +
              context 'Timestamps' do
         
     | 
| 
      
 173 
     | 
    
         
            +
                epoch_time = Time.at(1451613600).gmtime
         
     | 
| 
      
 174 
     | 
    
         
            +
             
     | 
| 
      
 175 
     | 
    
         
            +
                describe 'Full MD5 with timestamp prefix' do
         
     | 
| 
      
 176 
     | 
    
         
            +
                  config <<-CONFIG
         
     | 
| 
      
 177 
     | 
    
         
            +
                    filter {
         
     | 
| 
      
 178 
     | 
    
         
            +
                      hashid {
         
     | 
| 
      
 179 
     | 
    
         
            +
                        source => ['message']
         
     | 
| 
      
 180 
     | 
    
         
            +
                        method => 'MD5'
         
     | 
| 
      
 181 
     | 
    
         
            +
                        add_timestamp_prefix => true
         
     | 
| 
      
 182 
     | 
    
         
            +
                      }
         
     | 
| 
      
 183 
     | 
    
         
            +
                    }
         
     | 
| 
      
 184 
     | 
    
         
            +
                  CONFIG
         
     | 
| 
      
 185 
     | 
    
         
            +
             
     | 
| 
      
 186 
     | 
    
         
            +
                  sample("@timestamp" => epoch_time, "message" => "testmessage") do
         
     | 
| 
      
 187 
     | 
    
         
            +
                    insist { subject.get("hashid") } == 'KcMSc3COv1IOrOqLmF_6PG3gaZB'
         
     | 
| 
      
 188 
     | 
    
         
            +
                  end
         
     | 
| 
      
 189 
     | 
    
         
            +
                end
         
     | 
| 
      
 190 
     | 
    
         
            +
             
     | 
| 
      
 191 
     | 
    
         
            +
                describe 'Full MD5 with timestamp prefix and concatenated source part 1' do
         
     | 
| 
      
 192 
     | 
    
         
            +
                  config <<-CONFIG
         
     | 
| 
      
 193 
     | 
    
         
            +
                    filter {
         
     | 
| 
      
 194 
     | 
    
         
            +
                      hashid {
         
     | 
| 
      
 195 
     | 
    
         
            +
                        source => ['part1','part2']
         
     | 
| 
      
 196 
     | 
    
         
            +
                        method => 'MD5'
         
     | 
| 
      
 197 
     | 
    
         
            +
                        add_timestamp_prefix => true
         
     | 
| 
      
 198 
     | 
    
         
            +
                      }
         
     | 
| 
      
 199 
     | 
    
         
            +
                    }
         
     | 
| 
      
 200 
     | 
    
         
            +
                  CONFIG
         
     | 
| 
      
 201 
     | 
    
         
            +
             
     | 
| 
      
 202 
     | 
    
         
            +
                  sample("@timestamp" => epoch_time, "part1" => "test", "part2" => "message") do
         
     | 
| 
      
 203 
     | 
    
         
            +
                    insist { subject.get("hashid") } == 'KcMSc-8mTz750RP7I-pCuae4U6o'
         
     | 
| 
      
 204 
     | 
    
         
            +
                  end
         
     | 
| 
      
 205 
     | 
    
         
            +
                end
         
     | 
| 
      
 206 
     | 
    
         
            +
             
     | 
| 
      
 207 
     | 
    
         
            +
                describe 'Full MD5 with timestamp prefix and concatenated source part 2' do
         
     | 
| 
      
 208 
     | 
    
         
            +
                  config <<-CONFIG
         
     | 
| 
      
 209 
     | 
    
         
            +
                    filter {
         
     | 
| 
      
 210 
     | 
    
         
            +
                      hashid {
         
     | 
| 
      
 211 
     | 
    
         
            +
                        source => ['part2','part1']
         
     | 
| 
      
 212 
     | 
    
         
            +
                        method => 'MD5'
         
     | 
| 
      
 213 
     | 
    
         
            +
                        add_timestamp_prefix => true
         
     | 
| 
      
 214 
     | 
    
         
            +
                      }
         
     | 
| 
      
 215 
     | 
    
         
            +
                    }
         
     | 
| 
      
 216 
     | 
    
         
            +
                  CONFIG
         
     | 
| 
      
 217 
     | 
    
         
            +
             
     | 
| 
      
 218 
     | 
    
         
            +
                  sample("@timestamp" => epoch_time, "part2" => "message", "part1" => "test") do
         
     | 
| 
      
 219 
     | 
    
         
            +
                    insist { subject.get("hashid") } == 'KcMSc-8mTz750RP7I-pCuae4U6o'
         
     | 
| 
      
 220 
     | 
    
         
            +
                  end
         
     | 
| 
      
 221 
     | 
    
         
            +
                end
         
     | 
| 
      
 222 
     | 
    
         
            +
             
     | 
| 
      
 223 
     | 
    
         
            +
                describe 'Full MD5 with timestamp prefix and concatenated timestamp' do
         
     | 
| 
      
 224 
     | 
    
         
            +
                  config <<-CONFIG
         
     | 
| 
      
 225 
     | 
    
         
            +
                    filter {
         
     | 
| 
      
 226 
     | 
    
         
            +
                      hashid {
         
     | 
| 
      
 227 
     | 
    
         
            +
                        source => ['@timestamp','message']
         
     | 
| 
      
 228 
     | 
    
         
            +
                        method => 'MD5'
         
     | 
| 
      
 229 
     | 
    
         
            +
                        add_timestamp_prefix => true
         
     | 
| 
      
 230 
     | 
    
         
            +
                      }
         
     | 
| 
      
 231 
     | 
    
         
            +
                    }
         
     | 
| 
      
 232 
     | 
    
         
            +
                  CONFIG
         
     | 
| 
      
 233 
     | 
    
         
            +
             
     | 
| 
      
 234 
     | 
    
         
            +
                  sample("@timestamp" => epoch_time, "message" => "testmessage") do
         
     | 
| 
      
 235 
     | 
    
         
            +
                    insist { subject.get("hashid") } == 'KcMSc6IRDIqJMU6VN1x0TKt2fIo'
         
     | 
| 
      
 236 
     | 
    
         
            +
                  end
         
     | 
| 
      
 237 
     | 
    
         
            +
                end
         
     | 
| 
      
 238 
     | 
    
         
            +
             
     | 
| 
      
 239 
     | 
    
         
            +
                describe '12 byte MD5 with timestamp prefix' do
         
     | 
| 
      
 240 
     | 
    
         
            +
                  config <<-CONFIG
         
     | 
| 
      
 241 
     | 
    
         
            +
                    filter {
         
     | 
| 
      
 242 
     | 
    
         
            +
                      hashid {
         
     | 
| 
      
 243 
     | 
    
         
            +
                        source => ['message']
         
     | 
| 
      
 244 
     | 
    
         
            +
                        method => 'MD5'
         
     | 
| 
      
 245 
     | 
    
         
            +
                        add_timestamp_prefix => true
         
     | 
| 
      
 246 
     | 
    
         
            +
                        hash_bytes_used => 12
         
     | 
| 
      
 247 
     | 
    
         
            +
                      }
         
     | 
| 
      
 248 
     | 
    
         
            +
                    }
         
     | 
| 
      
 249 
     | 
    
         
            +
                  CONFIG
         
     | 
| 
      
 250 
     | 
    
         
            +
             
     | 
| 
      
 251 
     | 
    
         
            +
                  sample("@timestamp" => epoch_time, "message" => "testmessage") do
         
     | 
| 
      
 252 
     | 
    
         
            +
                    insist { subject.get("hashid") } == 'KcMScCbSbOQ81JSd3HmPFk'
         
     | 
| 
      
 253 
     | 
    
         
            +
                  end
         
     | 
| 
      
 254 
     | 
    
         
            +
                end
         
     | 
| 
      
 255 
     | 
    
         
            +
             
     | 
| 
      
 256 
     | 
    
         
            +
              end
         
     | 
| 
      
 257 
     | 
    
         
            +
             
     | 
| 
      
 258 
     | 
    
         
            +
            end
         
     | 
    
        metadata
    ADDED
    
    | 
         @@ -0,0 +1,88 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            --- !ruby/object:Gem::Specification
         
     | 
| 
      
 2 
     | 
    
         
            +
            name: logstash-filter-hashid
         
     | 
| 
      
 3 
     | 
    
         
            +
            version: !ruby/object:Gem::Version
         
     | 
| 
      
 4 
     | 
    
         
            +
              version: 0.1.2
         
     | 
| 
      
 5 
     | 
    
         
            +
            platform: ruby
         
     | 
| 
      
 6 
     | 
    
         
            +
            authors:
         
     | 
| 
      
 7 
     | 
    
         
            +
            - Christian Dahlqvist
         
     | 
| 
      
 8 
     | 
    
         
            +
            autorequire:
         
     | 
| 
      
 9 
     | 
    
         
            +
            bindir: bin
         
     | 
| 
      
 10 
     | 
    
         
            +
            cert_chain: []
         
     | 
| 
      
 11 
     | 
    
         
            +
            date: 2017-06-23 00:00:00.000000000 Z
         
     | 
| 
      
 12 
     | 
    
         
            +
            dependencies:
         
     | 
| 
      
 13 
     | 
    
         
            +
            - !ruby/object:Gem::Dependency
         
     | 
| 
      
 14 
     | 
    
         
            +
              requirement: !ruby/object:Gem::Requirement
         
     | 
| 
      
 15 
     | 
    
         
            +
                requirements:
         
     | 
| 
      
 16 
     | 
    
         
            +
                - - ">="
         
     | 
| 
      
 17 
     | 
    
         
            +
                  - !ruby/object:Gem::Version
         
     | 
| 
      
 18 
     | 
    
         
            +
                    version: '1.60'
         
     | 
| 
      
 19 
     | 
    
         
            +
                - - "<="
         
     | 
| 
      
 20 
     | 
    
         
            +
                  - !ruby/object:Gem::Version
         
     | 
| 
      
 21 
     | 
    
         
            +
                    version: '2.99'
         
     | 
| 
      
 22 
     | 
    
         
            +
              name: logstash-core-plugin-api
         
     | 
| 
      
 23 
     | 
    
         
            +
              prerelease: false
         
     | 
| 
      
 24 
     | 
    
         
            +
              type: :runtime
         
     | 
| 
      
 25 
     | 
    
         
            +
              version_requirements: !ruby/object:Gem::Requirement
         
     | 
| 
      
 26 
     | 
    
         
            +
                requirements:
         
     | 
| 
      
 27 
     | 
    
         
            +
                - - ">="
         
     | 
| 
      
 28 
     | 
    
         
            +
                  - !ruby/object:Gem::Version
         
     | 
| 
      
 29 
     | 
    
         
            +
                    version: '1.60'
         
     | 
| 
      
 30 
     | 
    
         
            +
                - - "<="
         
     | 
| 
      
 31 
     | 
    
         
            +
                  - !ruby/object:Gem::Version
         
     | 
| 
      
 32 
     | 
    
         
            +
                    version: '2.99'
         
     | 
| 
      
 33 
     | 
    
         
            +
            - !ruby/object:Gem::Dependency
         
     | 
| 
      
 34 
     | 
    
         
            +
              requirement: !ruby/object:Gem::Requirement
         
     | 
| 
      
 35 
     | 
    
         
            +
                requirements:
         
     | 
| 
      
 36 
     | 
    
         
            +
                - - ">="
         
     | 
| 
      
 37 
     | 
    
         
            +
                  - !ruby/object:Gem::Version
         
     | 
| 
      
 38 
     | 
    
         
            +
                    version: '0'
         
     | 
| 
      
 39 
     | 
    
         
            +
              name: logstash-devutils
         
     | 
| 
      
 40 
     | 
    
         
            +
              prerelease: false
         
     | 
| 
      
 41 
     | 
    
         
            +
              type: :development
         
     | 
| 
      
 42 
     | 
    
         
            +
              version_requirements: !ruby/object:Gem::Requirement
         
     | 
| 
      
 43 
     | 
    
         
            +
                requirements:
         
     | 
| 
      
 44 
     | 
    
         
            +
                - - ">="
         
     | 
| 
      
 45 
     | 
    
         
            +
                  - !ruby/object:Gem::Version
         
     | 
| 
      
 46 
     | 
    
         
            +
                    version: '0'
         
     | 
| 
      
 47 
     | 
    
         
            +
            description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
         
     | 
| 
      
 48 
     | 
    
         
            +
            email: christian@elastic.co
         
     | 
| 
      
 49 
     | 
    
         
            +
            executables: []
         
     | 
| 
      
 50 
     | 
    
         
            +
            extensions: []
         
     | 
| 
      
 51 
     | 
    
         
            +
            extra_rdoc_files: []
         
     | 
| 
      
 52 
     | 
    
         
            +
            files:
         
     | 
| 
      
 53 
     | 
    
         
            +
            - Gemfile
         
     | 
| 
      
 54 
     | 
    
         
            +
            - LICENSE
         
     | 
| 
      
 55 
     | 
    
         
            +
            - NOTICE.TXT
         
     | 
| 
      
 56 
     | 
    
         
            +
            - README.md
         
     | 
| 
      
 57 
     | 
    
         
            +
            - docs/index.asciidoc
         
     | 
| 
      
 58 
     | 
    
         
            +
            - lib/logstash/filters/hashid.rb
         
     | 
| 
      
 59 
     | 
    
         
            +
            - logstash-filter-hashid.gemspec
         
     | 
| 
      
 60 
     | 
    
         
            +
            - spec/filters/hashid_spec.rb
         
     | 
| 
      
 61 
     | 
    
         
            +
            homepage: https://github.com/cdahlqvist/logstash-filter-hashid/blob/master/README.md
         
     | 
| 
      
 62 
     | 
    
         
            +
            licenses:
         
     | 
| 
      
 63 
     | 
    
         
            +
            - Apache License (2.0)
         
     | 
| 
      
 64 
     | 
    
         
            +
            metadata:
         
     | 
| 
      
 65 
     | 
    
         
            +
              logstash_plugin: 'true'
         
     | 
| 
      
 66 
     | 
    
         
            +
              logstash_group: filter
         
     | 
| 
      
 67 
     | 
    
         
            +
            post_install_message:
         
     | 
| 
      
 68 
     | 
    
         
            +
            rdoc_options: []
         
     | 
| 
      
 69 
     | 
    
         
            +
            require_paths:
         
     | 
| 
      
 70 
     | 
    
         
            +
            - lib
         
     | 
| 
      
 71 
     | 
    
         
            +
            required_ruby_version: !ruby/object:Gem::Requirement
         
     | 
| 
      
 72 
     | 
    
         
            +
              requirements:
         
     | 
| 
      
 73 
     | 
    
         
            +
              - - ">="
         
     | 
| 
      
 74 
     | 
    
         
            +
                - !ruby/object:Gem::Version
         
     | 
| 
      
 75 
     | 
    
         
            +
                  version: '0'
         
     | 
| 
      
 76 
     | 
    
         
            +
            required_rubygems_version: !ruby/object:Gem::Requirement
         
     | 
| 
      
 77 
     | 
    
         
            +
              requirements:
         
     | 
| 
      
 78 
     | 
    
         
            +
              - - ">="
         
     | 
| 
      
 79 
     | 
    
         
            +
                - !ruby/object:Gem::Version
         
     | 
| 
      
 80 
     | 
    
         
            +
                  version: '0'
         
     | 
| 
      
 81 
     | 
    
         
            +
            requirements: []
         
     | 
| 
      
 82 
     | 
    
         
            +
            rubyforge_project:
         
     | 
| 
      
 83 
     | 
    
         
            +
            rubygems_version: 2.4.8
         
     | 
| 
      
 84 
     | 
    
         
            +
            signing_key:
         
     | 
| 
      
 85 
     | 
    
         
            +
            specification_version: 4
         
     | 
| 
      
 86 
     | 
    
         
            +
            summary: This filter lets you create base64 encoded event IDs based on the event contents and timestamp using a number of hash functions.
         
     | 
| 
      
 87 
     | 
    
         
            +
            test_files:
         
     | 
| 
      
 88 
     | 
    
         
            +
            - spec/filters/hashid_spec.rb
         
     |