logstash-filter-device_detector 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: b4654c9ab2e2eb6e23f7166c5ad18eeb9af885cc33a9556a2541bb82578e9ba6
+  data.tar.gz: 26199a2e87a7557932ecc837d13a6c063ca770cd4b2e6e0ea28be38701383ecd
+SHA512:
+  metadata.gz: f2301841fd07ee8c4b49d229d00ce49fdd66e505d6f2a0d9ed1125ae1410d50b885478da5e5a528b08fed655a1bdee0d8eb9dad984f7a73a202d4c292f8002c4
+  data.tar.gz: 53c9c88a4adfee4cd0cb24fa8a3a82b37977074bfb84dd186ba74aa40f0b7dc3e9d6715bc8ea555ef5d8b7ffb6b0b2ed191d9df675d59b2268087afd6f4da6dd

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+## 0.1.0
+  - Plugin created with the logstash plugin generator

data/CONTRIBUTORS

@@ -0,0 +1,10 @@
+The following is a list of people who have contributed ideas, code, bug
+reports, or in general have helped logstash along its way.
+
+Contributors:
+* Dukang - dukanghub@gmail.com
+
+Note: If you've sent us patches, bug reports, or otherwise contributed to
+Logstash, and you aren't on the list above and want to be, please let us know
+and we'll make sure you're here. Contributions from folks like you are what make
+open source awesome.

data/DEVELOPER.md

@@ -0,0 +1,2 @@
+# logstash-filter-device_detector
+Example filter plugin. This should help bootstrap your effort to write your own filter plugin!

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gemspec
+

data/LICENSE

@@ -0,0 +1,11 @@
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,91 @@
+# logstash-filter-device_detector
+
+## 说明
+
+这是一个基于ruby语言的`logstash filter`插件，用来解析`http_user_agent`，调用了外部库。
+
+## 开发环境测试
+
+### 1. 插件开发环境和测试
+
+#### Code
+- 首先，您需要安装了Bundler gem的JRuby。
+- 执行下面的命令安装依赖
+```sh
+bundle install
+```
+
+#### Test
+
+- Update your dependencies
+
+```sh
+bundle install
+```
+
+- Run tests
+
+```sh
+bundle exec rspec
+```
+
+这里在windows可能执行失败，暂时没研究什么原因。
+
+### 2. 在logstash中运行未发布的插件
+
+#### 2.1 直接在logstash使用克隆下来的插件代码
+
+进入logstash安装目录，一般是`/usr/share/logstash`
+
+- 编辑 `Gemfile` ，添加本地插件路径，如下所示:
+```ruby
+gem "logstash-filter-device_detector", :path => "/path/to/logstash-filter-device-detector"
+```
+- 安装插件
+```sh
+bin/logstash-plugin install --no-verify
+```
+- 运行logstash测试此插件
+```sh
+cd /etc/logstash/conf.d
+vim test.conf
+# 添加如下内容
+input {
+   stdin {}
+}
+filter {
+  device_detector {
+      source => "message"
+  }
+}
+output {
+  stdout { codec => rubydebug }
+}
+# 运行logstash
+/usr/share/logstash/bin/logstash -f test.conf
+```
+然后在屏幕输入useragent就可以看到效果了。
+
+#### 2.2 在logstash使用gem安装插件
+
+You can use the same **2.1** method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory or you can build the gem and install it using:
+
+- 构建gem
+```sh
+gem build logstash-filter-device-detector.gemspec
+```
+- 进入logstash安装目录，安装gem插件
+```sh
+bin/logstash-plugin install /your/local/plugin/logstash-filter-awesome.gem
+```
+- Start Logstash and proceed to test the plugin
+
+## Contributing
+
+All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
+
+Programming is not a required skill. Whatever you've seen about open source and maintainers or community members  saying "send patches or die" - you will not see that here.
+
+It is more important to the community that you are able to contribute.
+
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.

data/lib/logstash/filters/device_detector.rb

@@ -0,0 +1,118 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "device_detector"
+require 'json'
+
+class LogStash::Filters::DeviceDetector < LogStash::Filters::Base
+
+  config_name "device_detector"
+
+  config :source, :validate => :string, :default => "http_user_agent"
+
+  config :target, :validate => :string, :default => "ua"
+
+  config :tag_on_unknown, :validate => :array, :default => [ ]
+
+  config :tag_on_bot, :validate => :array, :default => [ ]
+
+  public
+  def register
+
+  end 
+
+  public
+  def filter(event)
+
+    # Receive source
+    useragent = event.get(@source)
+    return if useragent.nil? || !useragent.is_a?(String) || useragent.strip == ""
+
+    # Parse user-agent via device-detector
+    begin
+      data = DeviceDetector.new(useragent)
+    rescue StandardError => e
+      @logger.error("Uknown error while parsing device data", :exception => e, :field => @source, :event => event)
+      return
+    end
+    return unless data
+
+    # Remove original source (if its also the target)
+    event.remove(@source) if @target == @source
+
+    # Set all fields
+    begin
+      unless data.known?
+        @tag_on_unknown.each { |tag| event.tag(tag) }
+      end
+      if data.bot?
+        @tag_on_bot.each { |tag| event.tag(tag) }
+      end
+      is_mobile = false
+      is_bot = false
+      spider = ""
+      mozilla = false
+      model = true
+      platform = "Other"
+      os = "Other"
+      engine = ""
+      engine_version = ""
+      browser = ""
+      browser_version = ""
+
+      if data.device_type =~ /phone/
+        is_mobile = true
+      end
+
+      if data.bot?
+        is_bot = true
+        spider = data.bot_name
+      end
+      if data.device_type
+        platform = data.device_type
+      end
+      if data.os_full_version
+        os = "#{data.os_name} #{data.os_full_version}"
+      end
+      if data.name
+        browser = data.name
+        if data.name =~ /irefox/
+          mozilla = true
+        end
+      end
+      if data.full_version
+        browser_version = data.full_version
+      end
+      # 构造输出哈希表
+      output = {
+        "isMobile" => is_mobile,
+        "isBot" => is_bot,
+        "mozilla" => mozilla,
+        "model" => model,
+        "platform" => platform,
+        "os" => os,
+        "engine" => engine,
+        "engineVersion" => engine_version,
+        "browser" => browser,
+        "browserVersion" => browser_version
+      }
+      event.set("httpUserAgentJson", output.to_json)
+      event.set("os", os)
+      event.set("browser", "#{browser},#{browser_version}")
+      event.set("spider", spider)
+      event.set("#{@target}[browser][name]", data.name) if data.name
+      event.set("#{@target}[browser][version]", data.full_version) if data.full_version
+      event.set("#{@target}[os][name]", data.os_name) if data.os_name
+      event.set("#{@target}[os][version]", data.os_full_version) if data.os_full_version
+      event.set("#{@target}[device][name]", data.device_name) if data.device_name
+      event.set("#{@target}[device][brand]", data.device_brand) if data.device_brand
+      event.set("#{@target}[device][type]", data.device_type) if data.device_type
+      event.set("#{@target}[bot][name]", data.bot_name) if data.bot_name
+      event.set("#{@target}[bot][name]", data.bot_name) if data.bot_name
+    rescue StandardError => e
+      @logger.error("Uknown error while setting device data", :exception => e, :field => @source, :event => event)
+      return
+    end
+
+    filter_matched(event)
+  end
+end

data/logstash-filter-device-detector.gemspec

@@ -0,0 +1,24 @@
+Gem::Specification.new do |s|
+  s.name          = 'logstash-filter-device_detector'
+  s.version       = '0.1.1'
+  s.licenses      = ['Apache-2.0']
+  s.summary       = '使用device_detector解析useragent的logstash-filter插件.'
+  s.description   = 'Detects a vast amount of different devices automaticly based on regex rules.'
+  s.homepage      = 'https://github.com/Dukanghub/logstash-filter-device_detector'
+  s.authors       = ['Dukang']
+  s.email         = 'dukanghub@gmail.com'
+  s.require_paths = ['lib']
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
+  s.add_runtime_dependency "device_detector", "~> 1.0"
+  s.add_development_dependency "logstash-devutils", "~> 0"
+end

data/spec/filters/device_detector_spec.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+require_relative '../spec_helper'
+require "logstash/filters/device_detector"
+
+describe LogStash::Filters::DeviceDetector do
+  describe "Set to Hello World" do
+    let(:config) do <<-CONFIG
+      filter {
+        device_detector {
+          source => "useragent"
+          target => "device_detector"
+        }
+      }
+    CONFIG
+    end
+
+    sample("message" => "some text") do
+      expect(subject).to include("message")
+      expect(subject.get('message')).to eq('Hello World')
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"

metadata

@@ -0,0 +1,99 @@
+--- !ruby/object:Gem::Specification
+name: logstash-filter-device_detector
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Dukang
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2023-04-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: device_detector
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Detects a vast amount of different devices automaticly based on regex
+  rules.
+email: dukanghub@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- DEVELOPER.md
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/filters/device_detector.rb
+- logstash-filter-device-detector.gemspec
+- spec/filters/device_detector_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/Dukanghub/logstash-filter-device_detector
+licenses:
+- Apache-2.0
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: filter
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.25
+signing_key:
+specification_version: 4
+summary: 使用device_detector解析useragent的logstash-filter插件.
+test_files:
+- spec/filters/device_detector_spec.rb
+- spec/spec_helper.rb