fluent-plugin-reassemble 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,15 @@
1
+ ---
2
+ !binary "U0hBMQ==":
3
+ metadata.gz: !binary |-
4
+ NmZmMjIwNzk2NjBlYzgwYzhkODgwYTc0ZDA5ZjE5NGI1NDkzZGFhOQ==
5
+ data.tar.gz: !binary |-
6
+ ZmIxMGNkNjljODVjNGUzY2RkNjZhM2RmYTNhZTcwN2ZhN2Q1OTE4Mw==
7
+ !binary "U0hBNTEy":
8
+ metadata.gz: !binary |-
9
+ ZTNhNmVmYzU0MWU2MDJiYWUyMjZhNWY4YWE1Zjk1MjZkNDk3NjM0MDI4MWZh
10
+ NzgxZDllYzJkMzliZTkzMjQyY2ViY2I2ZTBmYzNkZDg2Y2FiMDc3OGJlMDkx
11
+ NjZkN2VmNjYyMWJkYTNlMDg2MjUxM2I3ZDYyZDVlOTc4Y2RiZTI=
12
+ data.tar.gz: !binary |-
13
+ OTA1ZTQ0MDA2M2MwOTZmNzBkMjY1MDBiNmY0M2E2ZTFkNTUyYzJhYzM5OTZk
14
+ MTg1ZDFkYjgyNDhmMDM1NmZlNWY2YmZjODBiNjA4MDQ5M2E3Njc1ZDA2Y2Nm
15
+ ODg0Njk1YWI4OTYwN2FmMTdhNzQxMWI3ZjFiYmU0MmUzMTQ4ZmY=
data/AUTHORS ADDED
@@ -0,0 +1 @@
1
+ moaikids
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source "http://rubygems.org"
2
+
3
+ gemspec
data/README.rdoc ADDED
@@ -0,0 +1,50 @@
1
+ = re-assembly streaming data (for Fluentd output plugin)
2
+
3
+ == Overview
4
+
5
+
6
+ == Installation
7
+
8
+ == Configuration
9
+
10
+ <match pattern>
11
+ type reassemble
12
+ output_tag OUTPUT_TAG
13
+ assemble CONFIGURATION_FOR_REASSEMBLE
14
+ null_to_null true or false (default false)
15
+ null_to_empty true or false (default false)
16
+ datetime_format DATETIME_FORMAT (default '%Y/%m/%d %H:%M:%S' / used in operation = 'unixtime_to_datetime')
17
+ date_format DATE_FORMAT (default '%Y/%m/%d' / used in operation = 'unixtime_to_date')
18
+ time_format TIME_FORMAT (default '%H:%M:%S' / used in operation = 'unixtime_to_time')
19
+ tz TIMEZONE(default :local)
20
+ </match>
21
+
22
+ === 'assemble' format
23
+ {extract_key1}:{replaced_key1}:{operation1},{extract_key2}:{replaced_key2}:{operation2},....{extract_keyN}:{replaced_keyN}:{operationN}
24
+
25
+ ex1.
26
+ assemble foo1:bar1:to_i, foo2:bar2:to_s
27
+ record => {"foo1": "1", "foo2": 2}
28
+ reassemble => {"bar1": 1, "bar2": "2"}
29
+
30
+ ex2.
31
+ assemble foo1,foo2:bar2,foo3:bar3:unixtime_to_datetime,foo4:bar4:url_to_host
32
+ record => {"foo1": "1", "foo2": 2, "foo3": 1377946931, "foo4": "http://www.sada.co.jp/concert.html"}
33
+ reassemble => {"foo1": "1", "bar2": 2, "bar3": "2013-08-31 20:00:02", "bar4": "www.sada.co.jp"}
34
+
35
+ === 'operation' list
36
+ to_s to string
37
+ to_i to integer
38
+ to_f to float
39
+ unixtime_to_datetime convert from unixtime to datetime(string)
40
+ unixtime_to_date convert from unixtime to date(string)
41
+ unixtime_to_time convert from unixtime to time(string)
42
+ url_to_domain extract host from url
43
+ url_to_host extract host from url
44
+ url_to_path extract path from url
45
+
46
+ == Copyright
47
+
48
+ Copyright:: Copyright (c) 2013 moaikids
49
+ License:: Apache License, Version 2.0
50
+
data/Rakefile ADDED
@@ -0,0 +1,11 @@
1
+ #!/usr/bin/env rake
2
+ require "bundler/gem_tasks"
3
+
4
+ require 'rake/testtask'
5
+ Rake::TestTask.new(:test) do |test|
6
+ test.libs << 'lib' << 'test'
7
+ test.pattern = 'test/**/test_*.rb'
8
+ test.verbose = true
9
+ end
10
+
11
+ task :default => :test
@@ -0,0 +1,19 @@
1
+ # -*- encoding: utf-8 -*-
2
+ Gem::Specification.new do |gem|
3
+ gem.name = "fluent-plugin-reassemble"
4
+ gem.version = "0.0.1"
5
+ gem.authors = ["moaikids"]
6
+ gem.licenses = ["Apache License Version 2.0"]
7
+ gem.summary = %q{Re-assemble a stream data for Fluentd}
8
+ gem.description = %q{Re-assemble a stream data for Fluentd...}
9
+ gem.homepage = "https://github.com/moaikids/fluent-plugin-reassemble"
10
+
11
+ gem.files = `git ls-files`.split($\)
12
+ gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
13
+ gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
14
+ gem.require_paths = ["lib"]
15
+
16
+ gem.add_development_dependency "rake"
17
+ gem.add_development_dependency "shoulda"
18
+ gem.add_runtime_dependency "fluentd"
19
+ end
@@ -0,0 +1,119 @@
1
+ require 'time'
2
+ require 'uri'
3
+ require 'tzinfo'
4
+
5
+ module Fluent
6
+ class ReassembleOutput < Fluent::Output
7
+ Fluent::Plugin.register_output('reassemble', self)
8
+ config_param :output_tag, :string, :default => nil
9
+ config_param :assemble, :string, :default => nil
10
+ config_param :null_to_null, :bool, :default => false
11
+ config_param :null_to_empty, :bool, :default => false
12
+ config_param :datetime_format, :string, :default => '%Y/%m/%d %H:%M:%S'
13
+ config_param :date_format, :string, :default => '%Y/%m/%d'
14
+ config_param :time_format, :string, :default => '%H:%M:%S'
15
+ config_param :tz, :string, :default => nil
16
+
17
+ def configure(conf)
18
+ super
19
+ unless config.has_key?('output_tag')
20
+ raise Fluent::ConfigError, "you must set 'output_tag'"
21
+ end
22
+ unless config.has_key?('assemble')
23
+ raise Fluent::ConfigError, "you must set 'assemble'"
24
+ end
25
+ if config.has_key?('tz')
26
+ ENV["TZ"] = config['tz']
27
+ end
28
+ #assemble definition format
29
+ # {extract_key1}:{replaced_key1}:{operation1},{extract_key2}:{replaced_key2}:{operation2},....{extract_keyN}:{replaced_keyN}:{operationN}
30
+ @reassemble_conf = []
31
+ @assemble.split(",").each{ |conf|
32
+ extract, replace, operation = conf.split(":")
33
+ if extract.nil? || extract.empty?
34
+ next
35
+ else
36
+ extract = extract.strip
37
+ end
38
+ if replace.nil? || replace.empty?
39
+ replace = extract
40
+ else
41
+ replace = replace.strip
42
+ end
43
+ unless operation.nil? || operation.empty?
44
+ operation = operation.strip
45
+ end
46
+ @reassemble_conf.push({:extract => extract, :replace => replace, :operation => operation})
47
+ }
48
+ $log.info "reassemble conf : " + @reassemble_conf.to_s
49
+ end
50
+
51
+ def emit(tag, es, chain)
52
+ es.each {|time,record|
53
+ json = {}
54
+ @reassemble_conf.each { |conf|
55
+ extract_key = conf[:extract]
56
+ replaced_key = conf[:replace]
57
+ operation = conf[:operation]
58
+ val = convert(traverse(record, extract_key), operation)
59
+ if val
60
+ json[replaced_key] = val
61
+ elsif @null_to_null
62
+ json[replaced_key] = nil
63
+ elsif @null_to_empty
64
+ json[replaced_key] = ""
65
+ end
66
+ }
67
+ Fluent::Engine.emit(@output_tag, time, json)
68
+ }
69
+ end
70
+
71
+ def traverse(data, key)
72
+ val = data
73
+ key.split('.').each{ |k|
74
+ if val.is_a?(Hash) && val.has_key?(k)
75
+ val = val[k]
76
+ else
77
+ return nil
78
+ end
79
+ }
80
+ return val
81
+ end
82
+
83
+ def convert(val, operation)
84
+ unless val
85
+ return val
86
+ end
87
+ unless operation
88
+ return val
89
+ end
90
+
91
+ begin
92
+ case operation.downcase
93
+ when "to_s"
94
+ return val.to_s
95
+ when "to_i"
96
+ return val.to_i
97
+ when "to_f"
98
+ return val.to_f
99
+ when "unixtime_to_datetime"
100
+ return Time.at(val.to_i).strftime(@datetime_format)
101
+ when "unixtime_to_date"
102
+ return Time.at(val.to_i).strftime(@date_format)
103
+ when "unixtime_to_time"
104
+ return Time.at(val.to_i).strftime(@time_format)
105
+ when "url_to_host","url_to_domain"
106
+ return URI(val.to_s).host
107
+ when "url_to_path"
108
+ return URI(val.to_s).path
109
+ else
110
+ return val
111
+ end
112
+ rescue
113
+ $log.warn $!
114
+ return val
115
+ end
116
+
117
+ end
118
+ end
119
+ end
metadata ADDED
@@ -0,0 +1,91 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-reassemble
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - moaikids
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2013-08-31 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: rake
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ! '>='
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ! '>='
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: shoulda
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ! '>='
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ! '>='
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: fluentd
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ! '>='
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ! '>='
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ description: Re-assemble a stream data for Fluentd...
56
+ email:
57
+ executables: []
58
+ extensions: []
59
+ extra_rdoc_files: []
60
+ files:
61
+ - AUTHORS
62
+ - Gemfile
63
+ - README.rdoc
64
+ - Rakefile
65
+ - fluent-plugin-reassemble.gemspec
66
+ - lib/fluent/plugin/out_reassemble.rb
67
+ homepage: https://github.com/moaikids/fluent-plugin-reassemble
68
+ licenses:
69
+ - Apache License Version 2.0
70
+ metadata: {}
71
+ post_install_message:
72
+ rdoc_options: []
73
+ require_paths:
74
+ - lib
75
+ required_ruby_version: !ruby/object:Gem::Requirement
76
+ requirements:
77
+ - - ! '>='
78
+ - !ruby/object:Gem::Version
79
+ version: '0'
80
+ required_rubygems_version: !ruby/object:Gem::Requirement
81
+ requirements:
82
+ - - ! '>='
83
+ - !ruby/object:Gem::Version
84
+ version: '0'
85
+ requirements: []
86
+ rubyforge_project:
87
+ rubygems_version: 2.0.7
88
+ signing_key:
89
+ specification_version: 4
90
+ summary: Re-assemble a stream data for Fluentd
91
+ test_files: []