logstash-filter-useragent 3.2.2-java → 3.3.2-java

data/lib/logstash/filters/useragent.rb

@@ -1,10 +1,8 @@
 # encoding: utf-8
-require "java"
 require "logstash-filter-useragent_jars"
 require "logstash/filters/base"
 require "logstash/namespace"
-require "tempfile"
-require "thread"
+require 'logstash/plugin_mixins/ecs_compatibility_support'
 
 # Parse user agent strings into structured data based on BrowserScope data
 #
@@ -16,6 +14,8 @@ require "thread"
 # <https://github.com/tobie/ua-parser/>.
 class LogStash::Filters::UserAgent < LogStash::Filters::Base
 
+  include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
+
   config_name "useragent"
 
   # The field containing the user agent string. If this field is an
@@ -25,7 +25,7 @@ class LogStash::Filters::UserAgent < LogStash::Filters::Base
   # The name of the field to assign user agent data into.
   #
   # If not specified user agent data will be stored in the root of the event.
-  config :target, :validate => :string
+  config :target, :validate => :string # default [user_agent] in ECS mode
 
   # `regexes.yaml` file to use
   #
@@ -37,7 +37,7 @@ class LogStash::Filters::UserAgent < LogStash::Filters::Base
   config :regexes, :validate => :string
 
   # A string to prepend to all of the extracted keys
-  config :prefix, :validate => :string, :default => ''
+  config :prefix, :validate => :string, :default => '' # not supported in ECS mode
 
   # UA parsing is surprisingly expensive. This filter uses an LRU cache to take advantage of the fact that
   # user agents are often found adjacent to one another in log files and rarely have a random distribution.
@@ -55,7 +55,46 @@ class LogStash::Filters::UserAgent < LogStash::Filters::Base
   # number of cache misses and waste memory.
   config :lru_cache_size, :validate => :number, :default => 100_000
 
+  def initialize(*params)
+    super
+
+    # make @target in the format [field name] if defined, i.e. surrounded by brackets
+    target = @target || ecs_select[disabled: '', v1: '[user_agent]']
+    target = "[#{@target}]" if !target.empty? && target !~ /^\[[^\[\]]+\]$/
+
+    @name_field = ecs_select[disabled: "[#{@prefix}name]", v1: '[name]']
+    @name_field = "#{target}#{@name_field}"
+
+    @device_name_field = ecs_select[disabled: "[#{@prefix}device]", v1: '[device][name]']
+    @device_name_field = "#{target}#{@device_name_field}"
+
+    @version_field = ecs_select[disabled: "[#{@prefix}version]", v1: '[version]']
+    @version_field = "#{target}#{@version_field}"
+    @major_field = ecs_select[disabled: "#{target}[#{@prefix}major]", v1: "[@metadata][filter][user_agent][version][major]"]
+    @minor_field = ecs_select[disabled: "#{target}[#{@prefix}minor]", v1: "[@metadata][filter][user_agent][version][minor]"]
+    @patch_field = ecs_select[disabled: "#{target}[#{@prefix}patch]", v1: "[@metadata][filter][user_agent][version][patch]"]
+
+    @os_full_name_field = ecs_select[disabled: "[#{@prefix}os_full]", v1: '[os][full]'] # did not exist in legacy prior to ECS-ification
+    @os_full_name_field = "#{target}#{@os_full_name_field}"
+
+    @os_name_field = ecs_select[disabled: "[#{@prefix}os_name]", v1: '[os][name]']
+    @os_name_field = "#{target}#{@os_name_field}"
+    @legacy_os_field = ecs_select[disabled: "#{target}[#{@prefix}os]", v1: nil] # same as [os_name] in legacy mode
+
+    @os_version_field = ecs_select[disabled: "[#{@prefix}os_version]", v1: '[os][version]']
+    @os_version_field = "#{target}#{@os_version_field}"
+    @os_major_field = ecs_select[disabled: "#{target}[#{@prefix}os_major]", v1: "[@metadata][filter][user_agent][os][version][major]"]
+    @os_minor_field = ecs_select[disabled: "#{target}[#{@prefix}os_minor]", v1: "[@metadata][filter][user_agent][os][version][minor]"]
+    @os_patch_field = ecs_select[disabled: "#{target}[#{@prefix}os_patch]", v1: "[@metadata][filter][user_agent][os][version][patch]"]
+
+    # NOTE: unfortunately we can not reliably provide `user_agent.original` since the patterns do not
+    # reliably give back the matched group and they support the UA string prefixed and/or suffixed
+  end
+
   def register
+    if ecs_compatibility != :disabled && @prefix && !@prefix.empty?
+      @logger.warn "Field prefix isn't supported in ECS compatibility mode, please remove `prefix => #{@prefix.inspect}`"
+    end
 
     if @regexes.nil?
       @parser = org.logstash.uaparser.CachingParser.new(lru_cache_size)
@@ -63,21 +102,6 @@ class LogStash::Filters::UserAgent < LogStash::Filters::Base
       @logger.debug("Using user agent regexes", :regexes => @regexes)
       @parser = org.logstash.uaparser.CachingParser.new(@regexes, lru_cache_size)
     end
-
-    # make @target in the format [field name] if defined, i.e. surrounded by brakets
-    normalized_target = (@target && @target !~ /^\[[^\[\]]+\]$/) ? "[#{@target}]" : ""
-
-    # predefine prefixed field names
-    @prefixed_name = "#{normalized_target}[#{@prefix}name]"
-    @prefixed_os = "#{normalized_target}[#{@prefix}os]"
-    @prefixed_os_name = "#{normalized_target}[#{@prefix}os_name]"
-    @prefixed_os_major = "#{normalized_target}[#{@prefix}os_major]"
-    @prefixed_os_minor = "#{normalized_target}[#{@prefix}os_minor]"
-    @prefixed_device = "#{normalized_target}[#{@prefix}device]"
-    @prefixed_major = "#{normalized_target}[#{@prefix}major]"
-    @prefixed_minor = "#{normalized_target}[#{@prefix}minor]"
-    @prefixed_patch = "#{normalized_target}[#{@prefix}patch]"
-    @prefixed_build = "#{normalized_target}[#{@prefix}build]"
   end
 
   def filter(event)
@@ -88,60 +112,127 @@ class LogStash::Filters::UserAgent < LogStash::Filters::Base
 
     begin
       ua_data = lookup_useragent(useragent)
-    rescue StandardError => e
-      @logger.error("Uknown error while parsing user agent data", :exception => e, :field => @source, :event => event)
+    rescue => e
+      @logger.error("Unknown error while parsing user agent data",
+                    :exception => e.class, :message => e.message, :backtrace => e.backtrace,
+                    :field => @source, :event => event.to_hash)
       return
     end
 
     return unless ua_data
 
     event.remove(@source) if @target == @source
-    set_fields(event, ua_data)
+    set_fields(event, useragent, ua_data)
 
     filter_matched(event)
   end
 
-  # should be private but need to stay public for specs
-  # TODO: (colin) the related specs should be refactored to not rely on private methods.
-  def lookup_useragent(useragent)
-    return unless useragent
+  private
 
-    # the UserAgentParser::Parser class is not thread safe, indications are that it is probably
-    # caused by the underlying JRuby regex code that is not thread safe.
-    # see https://github.com/logstash-plugins/logstash-filter-useragent/issues/25
+  def lookup_useragent(useragent)
     @parser.parse(useragent)
   end
 
-  private
-
-  def set_fields(event, ua_data)
-    # UserAgentParser outputs as US-ASCII.
-
-    event.set(@prefixed_name, ua_data.userAgent.family.dup.force_encoding(Encoding::UTF_8))
-
-    #OSX, Android and maybe iOS parse correctly, ua-agent parsing for Windows does not provide this level of detail
+  def set_fields(event, ua_source, ua_data)
+    # UserAgentParser strings are US-ASCII
+
+    ua = ua_data.userAgent
+    event.set(@name_field, duped_string(ua.family))
+    event.set(@device_name_field, duped_string(ua_data.device)) if ua_data.device
+
+    event.set(@major_field, duped_string(ua.major)) if ua.major
+    event.set(@minor_field, duped_string(ua.minor)) if ua.minor
+    event.set(@patch_field, duped_string(ua.patch)) if ua.patch
+    set_version(event, ua_source, ua) # UA version string e.g. "89.0.4389.90"
+
+    os = ua_data.os
+    if os
+      # os.major, os.minor, ... are all strings
+      event.set(@os_major_field, duped_string(os.major)) if os.major # e.g. 'Vista' or '10'
+      event.set(@os_minor_field, duped_string(os.minor)) if os.minor
+      event.set(@os_patch_field, duped_string(os.patch)) if os.patch
+      os_version = build_os_version(os)
+      event.set(@os_version_field, os_version) if os_version
+
+      os_name = os.family
+      if os_name
+        os_name = duped_string(os_name)
+        event.set(@os_name_field, os_name)
+        event.set(@legacy_os_field, os_name.dup) if @legacy_os_field
+        os_full_name = os_name.dup
+        os_full_name << ' ' << os_version if os_version
+        event.set(@os_full_name_field, os_full_name)
+      end
+    end
+  end
 
-    # Calls in here use #dup because there's potential for later filters to modify these values
-    # and corrupt the cache. See uap source here for details https://github.com/ua-parser/uap-ruby/tree/master/lib/user_agent_parser
-    if (os = ua_data.os)
-      # The OS is a rich object
-      event.set(@prefixed_os, ua_data.os.family.dup.force_encoding(Encoding::UTF_8))
-      event.set(@prefixed_os_name, os.family.dup.force_encoding(Encoding::UTF_8)) if os.family
-
-      # These are all strings
-      if os.minor && os.major
-        event.set(@prefixed_os_major, os.major.dup.force_encoding(Encoding::UTF_8)) if os.major
-        event.set(@prefixed_os_minor, os.minor.dup.force_encoding(Encoding::UTF_8)) if os.minor
+  # reconstruct and set the User-Agent version string
+  def set_version(event, ua_source, ua)
+    if @version_field && ua.major
+      # only Chrome has all 4 segments, while Firefox only uses major.minor
+      version = duped_string(ua.major)
+      if ua.minor
+        version << '.' << ua.minor
+        if ua.patch
+          version << '.' << ua.patch
+          if ua.patchMinor
+            version << '.' << ua.patchMinor
+          else
+            adjusted_version = check_and_adjust_version(ua_source, version)
+            version = adjusted_version if adjusted_version
+          end
+        end
       end
+      event.set(@version_field, version)
     end
+  end
 
-    event.set(@prefixed_device, ua_data.device.to_s.dup.force_encoding(Encoding::UTF_8)) if ua_data.device
+  def check_and_adjust_version(ua_source, version)
+    # only set OS version if it's not 'interpreted' (contained in UA string)
+    return nil if !version || (i = ua_source.index(version)).nil?
+    i += version.size
+    # complete version when patchMinor is not matched but still there
+    if ua_source[i] == '.' # we built the version with dots
+      if patch_minor = ua_source.index(' ', i + 1)
+        patch_minor = ua_source[i + 1...patch_minor]
+        if patch_minor.eql? patch_minor.to_i.to_s
+          version = "#{version}.#{patch_minor}"
+        end
+      end
+    end
+    version
+  end
 
-    if (ua_version = ua_data.userAgent)
-      event.set(@prefixed_major, ua_version.major.dup.force_encoding(Encoding::UTF_8)) if ua_version.major
-      event.set(@prefixed_minor, ua_version.minor.dup.force_encoding(Encoding::UTF_8)) if ua_version.minor
-      event.set(@prefixed_patch, ua_version.patch.dup.force_encoding(Encoding::UTF_8)) if ua_version.patch
-      event.set(@prefixed_build, ua_version.patchMinor.dup.force_encoding(Encoding::UTF_8)) if ua_version.patchMinor
+  # reconstructs the OS version string
+  def build_os_version(os)
+    # NOTE: UA regexes don't always give us the versions back
+    # they do get "corrected" for various OSes such as:
+    # - Windows (Windows NT 6.0 => 'Vista')
+    # - Windows ('Windows NT 6.3' => '8','1')
+    # - Windows ('Windows NT 10.0' => '10')
+    # - iOS ('Darwin/15.5' => '9','3','2')
+    return unless major = os.major
+    if major.to_i.to_s == major
+      version, sep = duped_string(major), '.'
+    else
+      version, sep = duped_string(major), ' '
     end
+    if os.minor
+      version << sep << os.minor
+      if os.patch
+        version << '.' << os.patch
+        if os.patchMinor
+          version << '.' << os.patchMinor
+        end
+      end
+    end
+    version
   end
+
+  def duped_string(str)
+    # Calls in here use #dup because there's potential for later filters to modify these values
+    # and corrupt the cache. See uap source here for details https://github.com/ua-parser/uap-ruby/tree/master/lib/user_agent_parser
+    str.dup.force_encoding(Encoding::UTF_8)
+  end
+
 end

data/logstash-filter-useragent.gemspec

@@ -22,6 +22,7 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.3'
   s.add_development_dependency 'logstash-devutils'
 end