logstash-filter-useragent 1.0.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 98b359b332dded4870ad7811f750a28b60012efb
-  data.tar.gz: 99d3bf56ecc02e4c0e3cc2106b1d31ba94bc4cb5
+  metadata.gz: 635e0d9c99d16e89f80e2636ff9fd490df621b4b
+  data.tar.gz: a39b3167347dc9b389048584d6da3feb8b4658c3
 SHA512:
-  metadata.gz: 79fd51fe698a9fbaf3d86b77c845521c13e12e3574b0d9e969db7f3ae2478b3f369da94ca8ecbc161edf70726c6c8d34a90fa8122bb29d85c048fbe02fc13758
-  data.tar.gz: ce6b4f6c2a6c97dc9b642bd3589e52d79d0b7a7f6ddf9f435c70d72240a42a60cf2247f8541d03ee51b52ae6e3f41bb5e4efab8d4306be4af0dbc8e9cb9c7221
+  metadata.gz: 0667b4eb93b90c48c4466f1b2385907ecdeb21acec0fed7a3adaff3e2dd8d113ad9cfb857fc1957b0592202e9279e35f34d27832331125303369919965a21d37
+  data.tar.gz: 493a091f47bff9eb965af0fc46e58b3936558c61bf2a2f7c5c400225fbc71e7ed833ed75fa05848da941e9514cba7d33080e33a8b1d4bb2b26ab6b33d0910085

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+* 1.1.0
+  - Add LRU cache 

data/README.md

@@ -1,15 +1,15 @@
 # Logstash Plugin
 
-This is a plugin for [Logstash](https://github.com/elasticsearch/logstash).
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
 
 ## Documentation
 
-Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elasticsearch.org/guide/en/logstash/current/).
+Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/).
 
 - For formatting code or config example, you can use the asciidoc `[source,ruby]` directive
-- For more asciidoc formatting tips, see the excellent reference here https://github.com/elasticsearch/docs#asciidoc-guide
+- For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide
 
 ## Need Help?
 
@@ -83,4 +83,4 @@ Programming is not a required skill. Whatever you've seen about open source and
 
 It is more important to the community that you are able to contribute.
 
-For more information about contributing, see the [CONTRIBUTING](https://github.com/elasticsearch/logstash/blob/master/CONTRIBUTING.md) file.
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.

data/lib/logstash/filters/useragent.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 require "logstash/filters/base"
 require "logstash/namespace"
+require "lru_redux"
 require "tempfile"
 
 # Parse user agent strings into structured data based on BrowserScope data
@@ -12,6 +13,8 @@ require "tempfile"
 # ua-parser with an Apache 2.0 license. For more details on ua-parser, see
 # <https://github.com/tobie/ua-parser/>.
 class LogStash::Filters::UserAgent < LogStash::Filters::Base
+  LOOKUP_CACHE = LruRedux::ThreadSafeCache.new(1000)
+
   config_name "useragent"
 
   # The field containing the user agent string. If this field is an
@@ -35,6 +38,22 @@ class LogStash::Filters::UserAgent < LogStash::Filters::Base
   # A string to prepend to all of the extracted keys
   config :prefix, :validate => :string, :default => ''
 
+  # UA parsing is surprisingly expensive. This filter uses an LRU cache to take advantage of the fact that
+  # user agents are often found adjacent to one another in log files and rarely have a random distribution.
+  # The higher you set this the more likely an item is to be in the cache and the faster this filter will run.
+  # However, if you set this too high you can use more memory than desired.
+  #
+  # Experiment with different values for this option to find the best performance for your dataset.
+  #
+  # This MUST be set to a value > 0. There is really no reason to not want this behavior, the overhead is minimal
+  # and the speed gains are large.
+  #
+  # It is important to note that this config value is global. That is to say all instances of the user agent filter
+  # share the same cache. The last declared cache size will 'win'. The reason for this is that there would be no benefit
+  # to having multiple caches for different instances at different points in the pipeline, that would just increase the
+  # number of cache misses and waste memory.
+  config :lru_cache_size, :validate => :number, :default => 1000
+
   public
   def register
     require 'user_agent_parser'
@@ -53,55 +72,74 @@ class LogStash::Filters::UserAgent < LogStash::Filters::Base
       @logger.info("Using user agent regexes", :regexes => @regexes)
       @parser = UserAgentParser::Parser.new(:patterns_path => @regexes)
     end
+
+    LOOKUP_CACHE.max_size = @lru_cache_size
   end #def register
 
   public
   def filter(event)
     return unless filter?(event)
-    ua_data = nil
 
     useragent = event[@source]
     useragent = useragent.first if useragent.is_a? Array
 
     begin
-      ua_data = @parser.parse(useragent)
-    rescue Exception => e
+      ua_data = lookup_useragent(useragent)
+    rescue StandardError => e
       @logger.error("Uknown error while parsing user agent data", :exception => e, :field => @source, :event => event)
+      return
     end
 
-    if !ua_data.nil?
-      if @target.nil?
-        # default write to the root of the event
-        target = event
-      else
-        target = event[@target] ||= {}
-      end
+    return unless ua_data
 
-      # UserAgentParser outputs as US-ASCII.
+    target = @target.nil? ? event : (event[@target] ||= {})
+    write_to_target(target, ua_data)
 
-      target[@prefix + "name"] = ua_data.name.force_encoding(Encoding::UTF_8)
+    filter_matched(event)
+  end # def filter
 
-      #OSX, Andriod and maybe iOS parse correctly, ua-agent parsing for Windows does not provide this level of detail
-      unless ua_data.os.nil?
-        target[@prefix + "os"] = ua_data.os.to_s.force_encoding(Encoding::UTF_8)
-        target[@prefix + "os_name"] = ua_data.os.name.to_s.force_encoding(Encoding::UTF_8)
-        target[@prefix + "os_major"] = ua_data.os.version.major.to_s.force_encoding(Encoding::UTF_8) unless ua_data.os.version.nil?
-        target[@prefix + "os_minor"] = ua_data.os.version.minor.to_s.force_encoding(Encoding::UTF_8) unless ua_data.os.version.nil?
-      end
+  def lookup_useragent(useragent)
+    return unless useragent
 
-      target[@prefix + "device"] = ua_data.device.to_s.force_encoding(Encoding::UTF_8) if not ua_data.device.nil?
+    cached = LOOKUP_CACHE[useragent]
+    return cached if cached
 
-      if not ua_data.version.nil?
-        ua_version = ua_data.version
-        target[@prefix + "major"] = ua_version.major.force_encoding(Encoding::UTF_8) if ua_version.major
-        target[@prefix + "minor"] = ua_version.minor.force_encoding(Encoding::UTF_8) if ua_version.minor
-        target[@prefix + "patch"] = ua_version.patch.force_encoding(Encoding::UTF_8) if ua_version.patch
-        target[@prefix + "build"] = ua_version.patch_minor.force_encoding(Encoding::UTF_8) if ua_version.patch_minor
+    ua_data = @parser.parse(useragent)
+
+    LOOKUP_CACHE[useragent] = ua_data
+    ua_data
+  end
+
+  def write_to_target(target, ua_data)
+    # UserAgentParser outputs as US-ASCII.
+
+    target[@prefix + "name"] = ua_data.name.dup.force_encoding(Encoding::UTF_8)
+
+    #OSX, Andriod and maybe iOS parse correctly, ua-agent parsing for Windows does not provide this level of detail
+
+    # Calls in here use #dup because there's potential for later filters to modify these values
+    # and corrupt the cache. See uap source here for details https://github.com/ua-parser/uap-ruby/tree/master/lib/user_agent_parser
+    if (os = ua_data.os)
+      # The OS is a rich object
+      target[@prefix + "os"] = ua_data.os.to_s.dup.force_encoding(Encoding::UTF_8)
+      target[@prefix + "os_name"] = os.name.dup.force_encoding(Encoding::UTF_8) if os.name
+
+      # These are all strings
+      if (os_version = os.version)
+        target[@prefix + "os_major"] = os_version.major.dup.force_encoding(Encoding::UTF_8) if os_version.major
+        target[@prefix + "os_minor"] = os_version.minor.dup.force_encoding(Encoding::UTF_8) if os_version.minor
       end
+    end
 
-      filter_matched(event)
+    target[@prefix + "device"] = ua_data.device.to_s.dup.force_encoding(Encoding::UTF_8) if ua_data.device
+
+    if (ua_version = ua_data.version)
+      target[@prefix + "major"] = ua_version.major.dup.force_encoding(Encoding::UTF_8) if ua_version.major
+      target[@prefix + "minor"] = ua_version.minor.dup.force_encoding(Encoding::UTF_8) if ua_version.minor
+      target[@prefix + "patch"] = ua_version.patch.dup.force_encoding(Encoding::UTF_8) if ua_version.patch
+      target[@prefix + "build"] = ua_version.patch_minor.dup.force_encoding(Encoding::UTF_8) if ua_version.patch_minor
     end
+  end
 
-  end # def filter
 end # class LogStash::Filters::UserAgent
 

data/logstash-filter-useragent.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-useragent'
-  s.version         = '1.0.1'
+  s.version         = '1.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Parse user agent strings into structured data based on BrowserScope data"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = `git ls-files`.split($\)+::Dir.glob('vendor/*')
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
@@ -23,6 +23,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
 
   s.add_runtime_dependency 'user_agent_parser', ['>= 2.0.0']
+  s.add_runtime_dependency 'lru_redux', "~> 1.1.0"
   s.add_development_dependency 'logstash-devutils'
 end
 

data/spec/filters/useragent_spec.rb

@@ -40,4 +40,51 @@ describe LogStash::Filters::UserAgent do
       insist { subject["minor"] } == "0"
     end
   end
+
+  describe "LRU object identity" do
+    let(:uafilter) { LogStash::Filters::UserAgent.new("source" => "foo") }
+    let(:ua_data) {
+      uafilter.lookup_useragent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.85 Safari/537.36")
+    }
+    subject(:target) { {} }
+
+    before do
+      uafilter.register
+
+      # Stub this out because this UA doesn't have this field
+      allow(ua_data.version).to receive(:patch_minor).and_return("foo")
+
+      uafilter.write_to_target(target, ua_data)
+    end
+
+    {
+      "name" => lambda {|uad| uad.name},
+      "os" => lambda {|uad| uad.os.to_s},
+      "os_name" => lambda {|uad| uad.os.name},
+      "os_major" => lambda {|uad| uad.os.version.major},
+      "os_minor" => lambda {|uad| uad.os.version.minor},
+      "device" => lambda {|uad| uad.device.to_s},
+      "major" => lambda {|uad| uad.version.major},
+      "minor" => lambda {|uad| uad.version.minor},
+      "patch" => lambda {|uad| uad.version.patch},
+      "build" => lambda {|uad| uad.version.patch_minor}
+    }.each do |field, uad_getter|
+      context "for the #{field} field" do
+        let(:value) {uad_getter.call(ua_data)}
+        let(:target_field) { target[field]}
+
+        it "should not have a nil value" do
+          expect(target_field).to be_truthy
+        end
+
+        it "should have equivalent values" do
+          expect(target_field).to eql(value)
+        end
+
+        it "should dup/clone the field to prevent cache corruption" do
+          expect(target_field.object_id).not_to eql(value.object_id)
+        end
+      end
+    end
+  end
 end

metadata

@@ -1,82 +1,92 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-useragent
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.1.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-06-30 00:00:00.000000000 Z
+date: 2015-09-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
   name: logstash-core
+  prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.4.0
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: 2.0.0
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 1.4.0
-    - - <
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.0.0
+  name: user_agent_parser
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: user_agent_parser
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.0.0
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 1.1.0
+  name: lru_redux
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.1.0
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-devutils
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
 - CHANGELOG.md
 - CONTRIBUTORS
 - Gemfile
 - LICENSE
 - NOTICE.TXT
 - README.md
-- Rakefile
 - lib/logstash/filters/useragent.rb
 - logstash-filter-useragent.gemspec
 - spec/filters/useragent_spec.rb
-- vendor.json
-- vendor/regexes.yaml
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -89,19 +99,20 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.1.9
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: Parse user agent strings into structured data based on BrowserScope data
 test_files:
 - spec/filters/useragent_spec.rb
+has_rdoc:

data/.gitignore

@@ -1,4 +0,0 @@
-*.gem
-Gemfile.lock
-.bundle
-vendor

data/Rakefile

@@ -1,10 +0,0 @@
-require 'json'
-
-BASE_PATH = File.expand_path(File.dirname(__FILE__))
-@files = JSON.parse(File.read(File.join(BASE_PATH, 'vendor.json')))
-
-task :default do
-  system("rake -T")
-end
-
-require "logstash/devutils/rake"

data/vendor.json

@@ -1,4 +0,0 @@
-[{
-  "url": "https://raw.githubusercontent.com/tobie/ua-parser/master/regexes.yaml", 
-  "sha1": "0b5e6f5ce44e389106d0343e54afc5f62767867c"
-}]