logstash-filter-languagedetect 0.0.1.pre.alpha.pre.12 → 0.1.0.alpha.15

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MzdmZWM0OTlhZThlY2Q5MmQ2YmUyNzM1OTcwNWZkMzRkMzg2NTVjNQ==
+    NTgxOTYxMjU0ZTRhNzUyMDY0MzI0NjFhODFlNGE1YmI0OWJiM2NhMw==
   data.tar.gz: !binary |-
-    OTc2NTRlM2Q5NjlkMTVkZTBiZGRkYmM2NWM4YWRkZjZkMWIzMmIwMg==
+    MWQ3ZmI3ZTBmNjg5YTEwYjZhOTRhMTU3ZTllN2Y1OGMxZTFlYzk0MA==
 SHA512:
   metadata.gz: !binary |-
-    ODQ5MGYzNTBkZmVmMmRlMjhkMzFjYmJlZWVjYTk4YmM0ZjUyMmEyOTk0NTk3
-    ZjMyYzRlNWEyMWQ3OGViYjViMzUzYjk5NTliMWYzMzliYTljZTE1MjZmMzYw
-    NDRhMGQyMDZmMGI2ZGQ2MzYxYzk1OTRiMTMwN2E4NTQzMjYxYjI=
+    NjI0NjBhODQyYzgyODYzZmJhZTMyZjc4NGE1YjQ2NDNhYTEyOWE2MTRiNmZl
+    NDZiMDhiZjRjZDJmNmZiNjM2YTU3ZDBhYTFiNGZlNzE3MDQ1OGZlMzE1NGJj
+    NzMwNzEwNGNhODZmNjI2NWNiNzdhMDhlMzZkZmQ1MWZiYzliZDE=
   data.tar.gz: !binary |-
-    YzJhMzc1MDMyYjE5NDJjMDhkNjQ4ZDIzODgyMDYyNTg3OTc3MTcxYzUxNzkx
-    MmYwNWZlOTE4Y2QyZjQ0NzgyZmU2ZjBmNTQ2ZGMyN2EzYTk5MWI1Zjc4NjQw
-    MzBmNTNlYzQ5NDJjYTU1MGQ3NGFmZmFmZmViZDAzNTNmMDcwNDg=
+    MzZhZTU4MzM0ZTlhZGQxNWZkYjE5OWQ2ZDU3MTcwODgzM2EyNzkwZDFlZjA1
+    MGEzNjdiODVhZjk2ODc0OGQ4YTQyN2QyYTcyYzE5OWFkZDYyMDZhNWIzZGQ4
+    MDNiMWQ5MTRmNmJkNWVkZmVmNWVmYmI4MTNjNGRlMDcwYjlkZjQ=

data/Gemfile

@@ -1,4 +1,3 @@
 source 'https://rubygems.org'
-
+gemspec
 gem 'whatlanguage', :git => "https://github.com/peterc/whatlanguage.git", :ref => "0192301022"
-gemspec

data/lib/logstash/filters/languagedetect.rb

@@ -1,31 +1,58 @@
-# encoding: utf-8
 require "logstash/filters/base"
 require "logstash/namespace"
 require 'whatlanguage'
-# require 'whatlanguage/string'
 
-# This example filter will replace the contents of the default 
-# message field with whatever you specify in the configuration.
-#
-# It is only intended to be used as an example.
+# This filter will try to determine the language of the field given
+# by the source parameter and output the result in either the language
+# name (in English lowercase) or iso format
 class LogStash::Filters::Languagedetect < LogStash::Filters::Base
 
   # Setting the config_name here is required. This is how you
   # configure this filter from your Logstash config.
   #
   # filter {
-  #   example {
-  #     message => "My message..."
+  #   languagedetect {
+  #     source => "message"
+  #     target => "lang"
+  #     useiso => true
   #   }
   # }
   #
   config_name "languagedetect"
   
-  # Replace the message with this value.
+  # Set the source field which is used for the language check.
+  #
+  # Example:
+  # [source,ruby]
+  #     filter {
+  #       languagedetect {
+  #         # tries to determine the language in "customfield"
+  #         source => "customfield"
+  #       }
+  #     }
   config :source, :validate => :string, :default => "message"
 
+  # Set the result field in which the result will be stored.
+  #
+  # Example:
+  # [source,ruby]
+  #     filter {
+  #       languagedetect {
+  #         # outputs the result into "customoutput"
+  #         target => "customoutput"
+  #       }
+  #     }
   config :target, :validate => :string, :default => "lang"
 
+  # If useiso is true, result will be "en" instead of "english"
+  #
+  # Example:
+  # [source,ruby]
+  #     filter {
+  #       languagedetect {
+  #         useiso => true
+  #       }
+  #     }
   config :useiso, :validate => :boolean, :default => false
 
 
@@ -40,13 +67,11 @@ class LogStash::Filters::Languagedetect < LogStash::Filters::Base
     text_to_detect = event[@source]
     if @useiso
       lang_found = @wl.language_iso(text_to_detect)
-      # lang_found = text_to_detect.language_iso
     else
       lang_found = @wl.language(text_to_detect)
-      # lang_found = text_to_detect.language
     end
 
-    event[@target] = lang_found
+    event[@target] = lang_found.to_s.force_encoding(Encoding::UTF_8)
 
     # filter_matched should go in the last line of our successful code
     filter_matched(event)

data/logstash-filter-languagedetect.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-filter-languagedetect'
-  s.version = '0.0.1'
-  s.version = "#{s.version}-alpha-#{ENV['TRAVIS_BUILD_NUMBER']}" if ENV['TRAVIS']
+  s.version = '0.1.0'
+  s.version = "#{s.version}.alpha.#{ENV['TRAVIS_BUILD_NUMBER']}" if ENV['TRAVIS']
   s.licenses = ['Apache License (2.0)']
   s.summary = "This example filter replaces the contents of the message field with the specified value."
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/languagedetect_spec.rb

@@ -1,7 +1,6 @@
 # encoding: utf-8
 require File.absolute_path(File.join(File.dirname(__FILE__), '../../spec/spec_helper'))
 require File.absolute_path(File.join(File.dirname(__FILE__), '../../lib/logstash/filters/languagedetect'))
-# require "lib/logstash/filters/languagedetect"
 
 describe LogStash::Filters::Languagedetect do
   describe "with defaults and non iso languages" do
@@ -14,22 +13,22 @@ describe LogStash::Filters::Languagedetect do
 
     sample("message" => "The links between the attempted car bombings in Glasgow and London are becoming clearer") do
       expect(subject).to include('lang')
-      expect(subject['lang']).to eq(:english)
+      expect(subject['lang']).to eq("english")
     end
 
     sample("message" => "dies ist ein deutscher Text") do
       expect(subject).to include('lang')
-      expect(subject['lang']).to eq(:german)
+      expect(subject['lang']).to eq("german")
     end
 
     sample("message" => "Deux autres personnes ont été arrêtées durant la nuit") do
       expect(subject).to include('lang')
-      expect(subject['lang']).to eq(:french)
+      expect(subject['lang']).to eq("french")
     end
 
     sample("message" => "En estado de máxima alertaen su nivel de crítico") do
       expect(subject).to include('lang')
-      expect(subject['lang']).to eq(:spanish)
+      expect(subject['lang']).to eq("spanish")
     end
   end
 
@@ -45,22 +44,56 @@ describe LogStash::Filters::Languagedetect do
 
     sample("message" => "The links between the attempted car bombings in Glasgow and London are becoming clearer") do
       expect(subject).to include('lang')
-      expect(subject['lang']).to eq(:en)
+      expect(subject['lang']).to eq("en")
     end
 
     sample("message" => "dies ist ein deutscher Text") do
       expect(subject).to include('lang')
-      expect(subject['lang']).to eq(:de)
+      expect(subject['lang']).to eq("de")
     end
 
     sample("message" => "Deux autres personnes ont été arrêtées durant la nuit") do
       expect(subject).to include('lang')
-      expect(subject['lang']).to eq(:fr)
+      expect(subject['lang']).to eq("fr")
     end
 
     sample("message" => "En estado de máxima alertaen su nivel de crítico") do
       expect(subject).to include('lang')
-      expect(subject['lang']).to eq(:es)
+      expect(subject['lang']).to eq("es")
+    end
+  end
+
+  describe "non default target" do
+    let(:config) do <<-CONFIG
+      filter {
+        languagedetect {
+          useiso => true
+          target => 'test'
+        }
+      }
+    CONFIG
+    end
+
+    sample("message" => "The links between the attempted car bombings in Glasgow and London are becoming clearer") do
+      expect(subject).to include('test')
+      expect(subject['test']).to eq("en")
+    end
+  end
+
+  describe "non default source" do
+    let(:config) do <<-CONFIG
+      filter {
+        languagedetect {
+          useiso => true
+          source => 'test'
+        }
+      }
+    CONFIG
+    end
+
+    sample("test" => "The links between the attempted car bombings in Glasgow and London are becoming clearer") do
+      expect(subject).to include('lang')
+      expect(subject['lang']).to eq("en")
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-languagedetect
 version: !ruby/object:Gem::Version
-  version: 0.0.1.pre.alpha.pre.12
+  version: 0.1.0.alpha.15
 platform: ruby
 authors:
 - Torsten Feld