logstash-filter-amelia 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 76bbf584bfef4c42804a90c83de7a9b0600923d3
+  data.tar.gz: fc78f7ca2dcd2937070421329b2aad7f681cc585
+SHA512:
+  metadata.gz: 6b47323b5aa9b0730d5d28eef08703e53afbc9acaa293cd25485c48c83620d7b870a55390daa85833bd8c3f4cf850246d3e463ce76daef400f90f4377df0d7ee
+  data.tar.gz: 1eb8adeb7c10e618a2e37d470d031037edadc0de57f949500961c734c711e234f1aca0c6a6aefef2ad7eea056609c59b0112e3a57c2c1e517c6ba90d95027048

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## 2.0.0
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
+   instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
+ - Dependency on logstash-core update to 2.0
+

data/Gemfile

@@ -0,0 +1,3 @@
+source 'http://rubygems.org'
+
+gemspec

data/README.md

@@ -0,0 +1,70 @@
+# Logstash Plugin
+
+
+## Developing
+
+### 1. Plugin Developement and Testing
+
+#### Code
+- To get started, you'll need JRuby with the Bundler gem installed.
+
+- Install dependencies
+```sh
+bundle install
+```
+
+#### Test
+
+- Update your dependencies
+
+```sh
+bundle install
+```
+
+- Run tests
+
+```sh
+bundle exec rspec
+```
+
+### 2. Running your unpublished Plugin in Logstash
+
+#### 2.1 Run in a local Logstash clone
+
+- Edit Logstash `Gemfile` and add the local plugin path, for example:
+```ruby
+gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
+```
+- Install plugin
+```sh
+bin/plugin install --no-verify
+```
+- Run Logstash with your plugin
+```sh
+bin/logstash -e 'filter {awesome {}}'
+```
+At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
+
+#### 2.2 Run in an installed Logstash
+
+You can use the same **2.1** method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory or you can build the gem and install it using:
+
+- Build your plugin gem
+```sh
+gem build logstash-filter-awesome.gemspec
+```
+- Install the plugin from the Logstash home
+```sh
+bin/plugin install /your/local/plugin/logstash-filter-awesome.gem
+```
+- Start Logstash and proceed to test the plugin
+
+## Contributing
+
+All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
+
+Programming is not a required skill. Whatever you've seen about open source and maintainers or community members  saying "send patches or die" - you will not see that here.
+
+It is more important to the community that you are able to contribute.
+
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.

data/lib/logstash/filters/amelia.rb

@@ -0,0 +1,128 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# This example filter will replace the contents of the default 
+# message field with whatever you specify in the configuration.
+#
+# It is only intended to be used as an example.
+class LogStash::Filters::Amelia < LogStash::Filters::Base
+
+  # Setting the config_name here is required. This is how you
+  # configure this filter from your Logstash config.
+  #
+  # filter {
+  #   amelia {
+  #     data => "fieldname"
+  #   }
+  # }
+  #
+  config_name "amelia"
+  
+  # field name with the data to parse
+  config :data, :validate => :string, :required => true
+  
+  # scope under which to expand data
+  config :scope_name, :validate => :string, :required => true
+  
+  # direction : in / out
+  config :direction, :validate => :string, :required => true
+  
+
+  public
+  def register
+    # @scope_name and @data will be created automaticaly  populated with config options
+  end
+
+  public
+  def filter(event)
+    
+    if @direction=="in"
+      event[@scope_name] = {}
+      event[@scope_name]['direction'] = 'in'
+      
+      # parsing for folloging block
+      
+      # 0008 ehelmeri -> user
+      # 0024 9FFgaORTvtv/u3Pe3AXFFw==  -> pwd
+      # 0005 3.354 -> version
+      # 0019 ListJobsByCodeAppli -> function
+      # 02 de -> lng
+      
+      position = 0
+      [['user', 4], ['pwd', 4], ['version', 4], ['function', 4], ['lng', 2]].each do |field|
+        length = event[@data][position, field[1]].to_i
+        position += field[1]
+        event[@scope_name][field[0]]= event[@data][position, length]
+        position += length
+      end
+      
+      # parsing for rest of the block
+      # 0001 0002 1 elt de 2 clefs
+        # 0004 0007 PROG LISI012
+        # 0004 0016 FUNC ListeTracabilite
+            # parsing headers
+            
+      event[@scope_name]['parameters']= parse_field  event[@data], position
+    end
+
+    if @direction=="out"
+      event[@scope_name] = {}
+      event[@scope_name]['direction'] = 'out'
+      
+      # parsing for folloging block
+      
+      # 0001 -> 1 paquet
+      # 0001 0 -> CR sur 1 char
+      # 0000 -> libelle
+      
+      event[@scope_name]['paquet']= event[@data][0, 4].to_i
+      position = 4
+      [ ['return_code', 4], ['libelle', 4]].each do |field|
+        length = event[@data][position, field[1]].to_i
+        position += field[1]
+        event[@scope_name][field[0]]= event[@data][position, length]
+        position += length
+      end
+      
+      # parsing for rest of the block
+      # 0001 0017 1 elt à 17 clef/val
+        # 0005 0009 timeC 145252452
+        # 00090001timeExcel000050008dateM2016021600100001nbFichiers000060002status1000110001nbFicAbnErr000050008dateC2016021600050007seqID105052700120001nbAbonnement000070001timePdf000130000nomAbonnement00040007progLISI09100080001nbEpingl000050009timeM14525276300150001seqIdAbonnement000040007userSMILLOT00070001timeTxt0
+      event[@scope_name]['parameters']= parse_field  event[@data], position
+    end
+    
+    # filter_matched should go in the last line of our successful code
+    filter_matched(event)
+  end # def filter
+  
+  
+  private
+  # parses list of hashes from the block, starting a indicated position
+  def parse_field data, position
+    retour = []
+    list_length = data[position, 4].to_i
+    position += 4
+    args_length = data[position, 4].to_i
+    position += 4
+   
+    list_length.times do |i|
+      current_args = {}
+      args_length.times do
+        key_length = data[position, 4].to_i
+        position += 4
+        value_length = data[position, 4].to_i
+        position += 4
+        
+        key = data[position, key_length]
+        position += key_length
+        
+        value = data[position, value_length]
+        position += value_length
+        current_args[key] = value
+      end
+      retour << current_args
+    end
+    retour
+  end
+end # class LogStash::Filters::Example

data/logstash-filter-amelia.gemspec

@@ -0,0 +1,23 @@
+Gem::Specification.new do |s|
+  s.name = 'logstash-filter-amelia'
+  s.version         = '1.0.0'
+  s.licenses = ['Apache License (2.0)']
+  s.summary = "This filter extracts data based on the homegrown AMELIA serialisation"
+  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program. The aim is to parse non standard serialization format, see spec for format descitpion"
+  s.authors = ["LISI Automotive"]
+  s.email = 'contact@lisi-automotive.com'
+  s.homepage = "http://www.lisi-automotive.com"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core", ">= 1.5.4", "< 3.0.0"
+  s.add_development_dependency 'logstash-devutils'
+end

data/spec/filters/amelia_spec.rb

@@ -0,0 +1,80 @@
+# encoding: utf-8
+require 'spec_helper'
+require "logstash/filters/amelia"
+
+  
+
+describe LogStash::Filters::Amelia do
+  describe "parse input" do
+    # breakdown for line 0008ehelmeri00249FFgaORTvtv/u3Pe3AXFFw==00053.3540019ListJobsByCodeAppli02de0001000200040007PROGLISI01200040016FUNCListeTracabilite
+    # 0008 ehelmeri -> user
+    # 0024 9FFgaORTvtv/u3Pe3AXFFw==  -> pwd
+    # 0005 3.354 -> version
+    # 0019 ListJobsByCodeAppli -> function
+    # 02 de -> lng
+    # 0001 0002 1 elt de 2 clefs
+      # 0004 0007 PROG LISI012
+      # 0004 0016 FUNC ListeTracabilite
+    let(:config) do <<-CONFIG
+      filter {
+        amelia {
+          direction => "in"
+          data => "payload"
+          scope_name => "result"
+        }
+      }
+    CONFIG
+    end
+
+    sample("payload" => "0008ehelmeri00249FFgaORTvtv/u3Pe3AXFFw==00053.3540019ListJobsByCodeAppli02de0001000200040007PROGLISI01200040016FUNCListeTracabilite") do
+      expect(subject).to include("result")
+      expect(subject['result']['direction']).to eql 'in'
+      expect(subject['result']['user']).to eql 'ehelmeri'
+      expect(subject['result']['function']).to eql 'ListJobsByCodeAppli'
+      expect(subject['result']['lng']).to eql 'de'
+    end
+    
+    sample("payload" => "0008ehelmeri00249FFgaORTvtv/u3Pe3AXFFw==00053.3540019ListJobsByCodeAppli02de0001000200040007PROGLISI01200040016FUNCListeTracabilite") do
+      expect(subject).to include("result")
+      expect(subject['result']['parameters']).to be_a Array
+      expect(subject['result']['parameters'][0]['PROG']).to eql 'LISI012'
+      expect(subject['result']['parameters'][0]['FUNC']).to eql 'ListeTracabilite'
+    end
+  end
+  
+  describe "parse output" do
+     # breakdown for 00010001000000001001700050009timeC14525245200090001timeExcel000050008dateM2016021600100001nbFichiers000060002status1000110001nbFicAbnErr000050008dateC2016021600050007seqID105052700120001nbAbonnement000070001timePdf000130000nomAbonnement00040007progLISI09100080001nbEpingl000050009timeM14525276300150001seqIdAbonnement000040007userSMILLOT00070001timeTxt0
+    # 0001 -> 1 paquet
+    # 0001 0 -> CR sur 1 char
+    # 0000 -> libelle
+    # 0001 0017 1 elt à 17 clef/val
+    # 0005 0009 timeC 145252452
+    # 00090001timeExcel000050008dateM2016021600100001nbFichiers000060002status1000110001nbFicAbnErr000050008dateC2016021600050007seqID105052700120001nbAbonnement000070001timePdf000130000nomAbonnement00040007progLISI09100080001nbEpingl000050009timeM14525276300150001seqIdAbonnement000040007userSMILLOT00070001timeTxt0
+    let(:config) do <<-CONFIG
+      filter {
+        amelia {
+          direction => "out"
+          data => "payload"
+          scope_name => "result"
+        }
+      }
+    CONFIG
+    end
+
+    sample("payload" => "00010001000000001001700050009timeC14525245200090001timeExcel000050008dateM2016021600100001nbFichiers000060002status1000110001nbFicAbnErr000050008dateC2016021600050007seqID105052700120001nbAbonnement000070001timePdf000130000nomAbonnement00040007progLISI09100080001nbEpingl000050009timeM14525276300150001seqIdAbonnement000040007userSMILLOT00070001timeTxt0") do
+      expect(subject).to include("result")
+      expect(subject['result']['paquet']).to eql 1
+      expect(subject['result']['direction']).to eql 'out'
+      expect(subject['result']['return_code']).to eql '0'
+      expect(subject['result']['libelle']).to eql ''
+    end
+    
+    sample("payload" => "00010001000000001001700050009timeC14525245200090001timeExcel000050008dateM2016021600100001nbFichiers000060002status1000110001nbFicAbnErr000050008dateC2016021600050007seqID105052700120001nbAbonnement000070001timePdf000130000nomAbonnement00040007progLISI09100080001nbEpingl000050009timeM14525276300150001seqIdAbonnement000040007userSMILLOT00070001timeTxt0") do
+      expect(subject).to include("result")
+      expect(subject['result']['parameters']).to be_a Array
+      expect(subject['result']['parameters'][0]['timeC']).to eql '145252452'
+      expect(subject['result']['parameters'][0]['timeTxt']).to eql '0'
+      expect(subject['result']['parameters'][0].keys.length).to eql 17
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification
+name: logstash-filter-amelia
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- LISI Automotive
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-02-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.4
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.4
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not
+  a stand-alone program. The aim is to parse non standard serialization format, see
+  spec for format descitpion
+email: contact@lisi-automotive.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- Gemfile
+- README.md
+- lib/logstash/filters/amelia.rb
+- logstash-filter-amelia.gemspec
+- spec/filters/amelia_spec.rb
+- spec/spec_helper.rb
+homepage: http://www.lisi-automotive.com
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: filter
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: This filter extracts data based on the homegrown AMELIA serialisation
+test_files:
+- spec/filters/amelia_spec.rb
+- spec/spec_helper.rb