fluent-plugin-keyvalue-parser 0.1.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7190c9f67c5deb19e52a51b95a10648f7d86f6df
+  data.tar.gz: 3e7a26b53a596d609fbec7fbca341164e1dc75e9
+SHA512:
+  metadata.gz: cf3ebd7f9f3a1f4bee6f00ac6f436aa9625a11df7ae19c60e5dfb7113bf7f86652f4490b659632e84e2543c88983765001e048c5ddd16a0e45f4c4c116104934
+  data.tar.gz: 712c43402ae7aa17aed13fdb1520a71e9e427c0e94e7f23e8ca4d68618da873b5e61f49b3dcdd9c1978c7f67a62a6aebc9692aa91494bc205c29210a559b603d

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/README.md

@@ -0,0 +1,104 @@
+#fluent-plugin-keyvalue-parser
+
+[Fluent](http://www.fluentd.org/) parser plugin for key:value formatted logs.
+
+
+##Installation
+
+```shell
+$ td-agent-gem install fluent-plugin-keyvalue-parser
+```
+
+##How to use
+
+Edit `/etc/td-agent/td-agent.conf` file.
+
+* with tail plugin
+```conf
+<source>
+  type tail
+  path /var/log/netscreen.log
+  tag  netscreen_logs 
+  pair_delimiter  ","
+  key_value_seperator "="
+  pos_file /var/run/td-agent/netscreen-log.pos
+  format keyvalue
+</source>
+```
+* with parser plugin
+```conf
+<filter tag>
+ type parser
+ format keyvalue
+ pair_delimiter  ","
+ key_value_seperator "="
+ key_name keyToParse
+</filter>
+```
+using above configuration,
+```
+key1=val1,key2=value2,"some key" = somevalue,diff_key="another value"
+```
+will be parsed as
+
+```json
+{"key1":"val1", "key2":"value2","some key":"somevalue","diff_key":"another value"}
+```
+
+####NOTE
+* if the key is not in quotes and pair_delimiter occures in key,plugin will handle it.
+  
+  eg:
+  
+  In below log, *pair_delimiter = " "  (space)*  is occured in key 'src zone'. 
+    
+  `devname=FT6H duration=194 service=http proto=6 `**`src zone=Trust`**` port=40055 policy_id=194`
+  
+  will be parsed as 
+  ```json
+  {"devname":"FT6H", "duration":"194","service":"http","src zone":"Trust","policy_id":"194"}
+  ```
+* But if value is not quoted, you should use optional parameter *'adjustment_rules'* to correct the parsing.
+
+## Option Parameters
+
+- **pair_delimiter**
+    
+    delimiter which seperate each key-value pairs. can be multi-character.
+    whitespaces or tabs can be given in quotes: ie, " " or "\t" .
+    By default it is ",".     
+
+- **key_value_seperator**
+
+    A string or character that seprates key and its value.
+    By default it is "="
+- **adjustment_rules**
+
+    Regular expression rules for some keys, represented as json , to adjust parsed records accordingly.
+    
+    {key1:regex1,key2:regex2}
+    
+   eg:
+   
+   normally following logs,
+   
+  `devname=FT6H `**`service=http`**`proto=6 src zone=Trust dst zone=Untrust`
+  
+  `devname=FT6H `**`service=NETBIOS (NS)`**`proto=17 src zone=Trust dst zone=Untrust`
+  
+   will be parsed as 
+   
+   ```json
+   {"devname":"FT6H","service":"http","proto":"6","src zone":"Trust","dst zone":"Untrust"}
+   
+   {"devname":"FT6H","service":"NETBIOS","(NS) proto":"6","src zone":"Trust","dst zone":"Untrust"}
+   ```   
+   in second case, key *"service"* only received first part of its value, becouse value not quoted and delimiter(here space) occured in the value. 
+   
+   Also next key *"proto"* is wrongly parsed as *"(NS) proto"*.
+   
+   to rectify this problem, we can use,
+   
+   `adjustment_rules {"service":"NETBIOS \\(.*\\)"}`  in configuration.
+   
+   this will parse *service* key with a value containing *NETBIOS (NS)* whenever it occures.

data/Rakefile

@@ -0,0 +1,9 @@
+# encoding: utf-8
+require "bundler/gem_tasks"
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) {| spec |
+    spec.pattern = FileList['spec/*_spec.rb']
+}
+task :default => :spec

data/fluent-plugin-keyvalue-parser.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+#lib = File.expand_path('../lib', __FILE__)
+#$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-keyvalue-parser"
+  spec.version       = "0.1.4"
+  spec.authors       = ["Arun M J"]
+  spec.email         = ["arunmj001@gmail.com"]
+  spec.homepage      = "https://github.com/arunmj/fluent-plugin-keyvalue-parser"
+  spec.description   = %q{Fluentd parser plugin for key-value formatted logs.}
+  spec.summary       = %q{This Fluentd parser plugin can be used to parse key-value pairs.}
+  spec.license       = "MIT"
+  spec.has_rdoc      = false
+
+  spec.files         = `git ls-files`.split("\n")
+  spec.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  spec.executables   = `git ls-files -- bin/*`.split("\n").map{| f | File.basename(f)}
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'fluentd', "~> 0.10"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+
+  spec.required_ruby_version = '~> 2.0'
+end

data/lib/fluent/plugin/parser_keyvalue.rb

@@ -0,0 +1,139 @@
+require 'fluent/parser'
+require 'fluent/log'
+require 'fluent/time'
+require 'json'
+
+
+module Fluent
+	class TextParser
+		class KeyValueParser < Parser
+
+			QUOTE = "\""
+
+			# Register this parser as "keyvalue"
+			Plugin.register_parser("keyvalue", self)
+
+			config_param :pair_delimiter,      :string, :default => " " 
+			config_param :key_value_seperator, :string, :default => "," 
+			
+			config_param :adjustment_rules            , :default => false do |val|
+				rule_hash = false
+				if val != ""
+					begin
+						rule_hash_raw = JSON.parse(val)
+					rescue JSON::ParserError => ex
+						# Fluent::ConfigParseError, "got incomplete JSON" will be raised
+						raise Fluent::ConfigError, "#{ex.class}: #{ex.message}"
+					end
+					raise Fluent::ConfigError, "adjustment_rules is not a hash" unless rule_hash_raw.is_a?(Hash)
+					#rule_hash = make_rule_hash(rule_hash_raw)
+					rule_hash = {}
+					rule_hash_raw.each do |k,v|
+						regx_str = "(?<scavenged>#{v})#{@pair_delimiter}(?<remnants>.*$)"
+						rule_regex = //.class.new(regx_str)
+						rule_hash[k] = rule_regex
+					end
+				end
+				rule_hash
+			end
+
+
+
+			def configure(conf)
+				super			
+			end
+
+			def parse(text)
+				
+				record = {}
+				is_at_key =true
+				is_quote_open = false
+				key_name = ""
+				value = ""
+
+				text.each_char do |chr| 
+
+					if is_quote_open && chr != QUOTE
+						if is_at_key
+							key_name << chr
+						else
+							value << chr
+						end
+						next
+					end
+
+					case chr
+					when QUOTE
+						is_quote_open = !is_quote_open
+						next
+
+					when @pair_delimiter
+						if is_at_key
+							key_name << chr
+							next
+						end
+						record[key_name] = value
+						key_name = ""
+						value = ""
+						is_at_key = true
+						next
+					when @key_value_seperator
+						if is_at_key then 
+							is_at_key = false 
+						else
+							value << chr
+						end
+						next
+					else
+						if is_at_key
+							key_name << chr
+						else
+							value << chr
+						end
+						next
+					end
+				end
+
+				if key_name != "" 
+					if value != "" then 
+						record[key_name] = value 
+					else 
+						record[record.keys.last] << @pair_delimiter << key_name
+					end 
+				end
+
+				if @adjustment_rules 
+					record = adjust_record(record)
+				end
+				yield nil,record
+			end
+
+
+
+			def adjust_record(record)
+				record.keys.each_with_index do |k,i|
+					if adjustment_rules.key? k
+						$log.debug "adjusting record ... for key : #{k}"
+						neighbour_key = record.keys[i+1]
+						valstr = [record[k],neighbour_key].join(@pair_delimiter)
+						puts "origina : " + valstr
+						m = adjustment_rules[k].match(valstr)
+						unless m
+							puts "no match"
+						else
+							puts m[0],m[1],m[2]
+							m.names.each do |x|
+								puts x + " : " + m[x]
+							end
+							record[k] = m['scavenged']
+							remnants = m['remnants']
+							record[remnants] = record.delete neighbour_key
+						end
+					end
+				end
+				return record
+			end
+
+		end
+	end
+end

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-keyvalue-parser
+version: !ruby/object:Gem::Version
+  version: 0.1.4
+platform: ruby
+authors:
+- Arun M J
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-03-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd parser plugin for key-value formatted logs.
+email:
+- arunmj001@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- README.md
+- Rakefile
+- fluent-plugin-keyvalue-parser.gemspec
+- lib/fluent/plugin/parser_keyvalue.rb
+homepage: https://github.com/arunmj/fluent-plugin-keyvalue-parser
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: This Fluentd parser plugin can be used to parse key-value pairs.
+test_files: []