fluent-plugin-reassemble 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +15 -0
- data/AUTHORS +1 -0
- data/Gemfile +3 -0
- data/README.rdoc +50 -0
- data/Rakefile +11 -0
- data/fluent-plugin-reassemble.gemspec +19 -0
- data/lib/fluent/plugin/out_reassemble.rb +119 -0
- metadata +91 -0
checksums.yaml
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
---
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
NmZmMjIwNzk2NjBlYzgwYzhkODgwYTc0ZDA5ZjE5NGI1NDkzZGFhOQ==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
ZmIxMGNkNjljODVjNGUzY2RkNjZhM2RmYTNhZTcwN2ZhN2Q1OTE4Mw==
|
7
|
+
!binary "U0hBNTEy":
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
ZTNhNmVmYzU0MWU2MDJiYWUyMjZhNWY4YWE1Zjk1MjZkNDk3NjM0MDI4MWZh
|
10
|
+
NzgxZDllYzJkMzliZTkzMjQyY2ViY2I2ZTBmYzNkZDg2Y2FiMDc3OGJlMDkx
|
11
|
+
NjZkN2VmNjYyMWJkYTNlMDg2MjUxM2I3ZDYyZDVlOTc4Y2RiZTI=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
OTA1ZTQ0MDA2M2MwOTZmNzBkMjY1MDBiNmY0M2E2ZTFkNTUyYzJhYzM5OTZk
|
14
|
+
MTg1ZDFkYjgyNDhmMDM1NmZlNWY2YmZjODBiNjA4MDQ5M2E3Njc1ZDA2Y2Nm
|
15
|
+
ODg0Njk1YWI4OTYwN2FmMTdhNzQxMWI3ZjFiYmU0MmUzMTQ4ZmY=
|
data/AUTHORS
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
moaikids
|
data/Gemfile
ADDED
data/README.rdoc
ADDED
@@ -0,0 +1,50 @@
|
|
1
|
+
= re-assembly streaming data (for Fluentd output plugin)
|
2
|
+
|
3
|
+
== Overview
|
4
|
+
|
5
|
+
|
6
|
+
== Installation
|
7
|
+
|
8
|
+
== Configuration
|
9
|
+
|
10
|
+
<match pattern>
|
11
|
+
type reassemble
|
12
|
+
output_tag OUTPUT_TAG
|
13
|
+
assemble CONFIGURATION_FOR_REASSEMBLE
|
14
|
+
null_to_null true or false (default false)
|
15
|
+
null_to_empty true or false (default false)
|
16
|
+
datetime_format DATETIME_FORMAT (default '%Y/%m/%d %H:%M:%S' / used in operation = 'unixtime_to_datetime')
|
17
|
+
date_format DATE_FORMAT (default '%Y/%m/%d' / used in operation = 'unixtime_to_date')
|
18
|
+
time_format TIME_FORMAT (default '%H:%M:%S' / used in operation = 'unixtime_to_time')
|
19
|
+
tz TIMEZONE(default :local)
|
20
|
+
</match>
|
21
|
+
|
22
|
+
=== 'assemble' format
|
23
|
+
{extract_key1}:{replaced_key1}:{operation1},{extract_key2}:{replaced_key2}:{operation2},....{extract_keyN}:{replaced_keyN}:{operationN}
|
24
|
+
|
25
|
+
ex1.
|
26
|
+
assemble foo1:bar1:to_i, foo2:bar2:to_s
|
27
|
+
record => {"foo1": "1", "foo2": 2}
|
28
|
+
reassemble => {"bar1": 1, "bar2": "2"}
|
29
|
+
|
30
|
+
ex2.
|
31
|
+
assemble foo1,foo2:bar2,foo3:bar3:unixtime_to_datetime,foo4:bar4:url_to_host
|
32
|
+
record => {"foo1": "1", "foo2": 2, "foo3": 1377946931, "foo4": "http://www.sada.co.jp/concert.html"}
|
33
|
+
reassemble => {"foo1": "1", "bar2": 2, "bar3": "2013-08-31 20:00:02", "bar4": "www.sada.co.jp"}
|
34
|
+
|
35
|
+
=== 'operation' list
|
36
|
+
to_s to string
|
37
|
+
to_i to integer
|
38
|
+
to_f to float
|
39
|
+
unixtime_to_datetime convert from unixtime to datetime(string)
|
40
|
+
unixtime_to_date convert from unixtime to date(string)
|
41
|
+
unixtime_to_time convert from unixtime to time(string)
|
42
|
+
url_to_domain extract host from url
|
43
|
+
url_to_host extract host from url
|
44
|
+
url_to_path extract path from url
|
45
|
+
|
46
|
+
== Copyright
|
47
|
+
|
48
|
+
Copyright:: Copyright (c) 2013 moaikids
|
49
|
+
License:: Apache License, Version 2.0
|
50
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
Gem::Specification.new do |gem|
|
3
|
+
gem.name = "fluent-plugin-reassemble"
|
4
|
+
gem.version = "0.0.1"
|
5
|
+
gem.authors = ["moaikids"]
|
6
|
+
gem.licenses = ["Apache License Version 2.0"]
|
7
|
+
gem.summary = %q{Re-assemble a stream data for Fluentd}
|
8
|
+
gem.description = %q{Re-assemble a stream data for Fluentd...}
|
9
|
+
gem.homepage = "https://github.com/moaikids/fluent-plugin-reassemble"
|
10
|
+
|
11
|
+
gem.files = `git ls-files`.split($\)
|
12
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
13
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
14
|
+
gem.require_paths = ["lib"]
|
15
|
+
|
16
|
+
gem.add_development_dependency "rake"
|
17
|
+
gem.add_development_dependency "shoulda"
|
18
|
+
gem.add_runtime_dependency "fluentd"
|
19
|
+
end
|
@@ -0,0 +1,119 @@
|
|
1
|
+
require 'time'
|
2
|
+
require 'uri'
|
3
|
+
require 'tzinfo'
|
4
|
+
|
5
|
+
module Fluent
|
6
|
+
class ReassembleOutput < Fluent::Output
|
7
|
+
Fluent::Plugin.register_output('reassemble', self)
|
8
|
+
config_param :output_tag, :string, :default => nil
|
9
|
+
config_param :assemble, :string, :default => nil
|
10
|
+
config_param :null_to_null, :bool, :default => false
|
11
|
+
config_param :null_to_empty, :bool, :default => false
|
12
|
+
config_param :datetime_format, :string, :default => '%Y/%m/%d %H:%M:%S'
|
13
|
+
config_param :date_format, :string, :default => '%Y/%m/%d'
|
14
|
+
config_param :time_format, :string, :default => '%H:%M:%S'
|
15
|
+
config_param :tz, :string, :default => nil
|
16
|
+
|
17
|
+
def configure(conf)
|
18
|
+
super
|
19
|
+
unless config.has_key?('output_tag')
|
20
|
+
raise Fluent::ConfigError, "you must set 'output_tag'"
|
21
|
+
end
|
22
|
+
unless config.has_key?('assemble')
|
23
|
+
raise Fluent::ConfigError, "you must set 'assemble'"
|
24
|
+
end
|
25
|
+
if config.has_key?('tz')
|
26
|
+
ENV["TZ"] = config['tz']
|
27
|
+
end
|
28
|
+
#assemble definition format
|
29
|
+
# {extract_key1}:{replaced_key1}:{operation1},{extract_key2}:{replaced_key2}:{operation2},....{extract_keyN}:{replaced_keyN}:{operationN}
|
30
|
+
@reassemble_conf = []
|
31
|
+
@assemble.split(",").each{ |conf|
|
32
|
+
extract, replace, operation = conf.split(":")
|
33
|
+
if extract.nil? || extract.empty?
|
34
|
+
next
|
35
|
+
else
|
36
|
+
extract = extract.strip
|
37
|
+
end
|
38
|
+
if replace.nil? || replace.empty?
|
39
|
+
replace = extract
|
40
|
+
else
|
41
|
+
replace = replace.strip
|
42
|
+
end
|
43
|
+
unless operation.nil? || operation.empty?
|
44
|
+
operation = operation.strip
|
45
|
+
end
|
46
|
+
@reassemble_conf.push({:extract => extract, :replace => replace, :operation => operation})
|
47
|
+
}
|
48
|
+
$log.info "reassemble conf : " + @reassemble_conf.to_s
|
49
|
+
end
|
50
|
+
|
51
|
+
def emit(tag, es, chain)
|
52
|
+
es.each {|time,record|
|
53
|
+
json = {}
|
54
|
+
@reassemble_conf.each { |conf|
|
55
|
+
extract_key = conf[:extract]
|
56
|
+
replaced_key = conf[:replace]
|
57
|
+
operation = conf[:operation]
|
58
|
+
val = convert(traverse(record, extract_key), operation)
|
59
|
+
if val
|
60
|
+
json[replaced_key] = val
|
61
|
+
elsif @null_to_null
|
62
|
+
json[replaced_key] = nil
|
63
|
+
elsif @null_to_empty
|
64
|
+
json[replaced_key] = ""
|
65
|
+
end
|
66
|
+
}
|
67
|
+
Fluent::Engine.emit(@output_tag, time, json)
|
68
|
+
}
|
69
|
+
end
|
70
|
+
|
71
|
+
def traverse(data, key)
|
72
|
+
val = data
|
73
|
+
key.split('.').each{ |k|
|
74
|
+
if val.is_a?(Hash) && val.has_key?(k)
|
75
|
+
val = val[k]
|
76
|
+
else
|
77
|
+
return nil
|
78
|
+
end
|
79
|
+
}
|
80
|
+
return val
|
81
|
+
end
|
82
|
+
|
83
|
+
def convert(val, operation)
|
84
|
+
unless val
|
85
|
+
return val
|
86
|
+
end
|
87
|
+
unless operation
|
88
|
+
return val
|
89
|
+
end
|
90
|
+
|
91
|
+
begin
|
92
|
+
case operation.downcase
|
93
|
+
when "to_s"
|
94
|
+
return val.to_s
|
95
|
+
when "to_i"
|
96
|
+
return val.to_i
|
97
|
+
when "to_f"
|
98
|
+
return val.to_f
|
99
|
+
when "unixtime_to_datetime"
|
100
|
+
return Time.at(val.to_i).strftime(@datetime_format)
|
101
|
+
when "unixtime_to_date"
|
102
|
+
return Time.at(val.to_i).strftime(@date_format)
|
103
|
+
when "unixtime_to_time"
|
104
|
+
return Time.at(val.to_i).strftime(@time_format)
|
105
|
+
when "url_to_host","url_to_domain"
|
106
|
+
return URI(val.to_s).host
|
107
|
+
when "url_to_path"
|
108
|
+
return URI(val.to_s).path
|
109
|
+
else
|
110
|
+
return val
|
111
|
+
end
|
112
|
+
rescue
|
113
|
+
$log.warn $!
|
114
|
+
return val
|
115
|
+
end
|
116
|
+
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
metadata
ADDED
@@ -0,0 +1,91 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-reassemble
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- moaikids
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2013-08-31 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: rake
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ! '>='
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ! '>='
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: shoulda
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ! '>='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ! '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: fluentd
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ! '>='
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ! '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
description: Re-assemble a stream data for Fluentd...
|
56
|
+
email:
|
57
|
+
executables: []
|
58
|
+
extensions: []
|
59
|
+
extra_rdoc_files: []
|
60
|
+
files:
|
61
|
+
- AUTHORS
|
62
|
+
- Gemfile
|
63
|
+
- README.rdoc
|
64
|
+
- Rakefile
|
65
|
+
- fluent-plugin-reassemble.gemspec
|
66
|
+
- lib/fluent/plugin/out_reassemble.rb
|
67
|
+
homepage: https://github.com/moaikids/fluent-plugin-reassemble
|
68
|
+
licenses:
|
69
|
+
- Apache License Version 2.0
|
70
|
+
metadata: {}
|
71
|
+
post_install_message:
|
72
|
+
rdoc_options: []
|
73
|
+
require_paths:
|
74
|
+
- lib
|
75
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
76
|
+
requirements:
|
77
|
+
- - ! '>='
|
78
|
+
- !ruby/object:Gem::Version
|
79
|
+
version: '0'
|
80
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
81
|
+
requirements:
|
82
|
+
- - ! '>='
|
83
|
+
- !ruby/object:Gem::Version
|
84
|
+
version: '0'
|
85
|
+
requirements: []
|
86
|
+
rubyforge_project:
|
87
|
+
rubygems_version: 2.0.7
|
88
|
+
signing_key:
|
89
|
+
specification_version: 4
|
90
|
+
summary: Re-assemble a stream data for Fluentd
|
91
|
+
test_files: []
|