fluent-plugin-s3-input 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +1 -0
- data/Gemfile +3 -0
- data/Gemfile.lock +55 -0
- data/README.md +59 -0
- data/Rakefile +10 -0
- data/fluent-plugin-s3-input.gemspec +24 -0
- data/lib/fluent/plugin/out_s3_input.rb +61 -0
- metadata +107 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: abd2bf6ecf2453a5d47cf8702ff3fe95f50dbf79
|
4
|
+
data.tar.gz: 9c73ec74af0ef99f899d0dec6e225ee4abd105dc
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: feed1255cafe0de56b2a0df87aa82093e2b07f87a5372f51512527ad4bb2db3c37177e017908aca09f041499d3e70232436101bb1b429c116a27d95ef94da3eb
|
7
|
+
data.tar.gz: 664811196401b7a22cb7078ad39de99a27f865dd7602d351327c595c991e7f54bbf923e7199b078871a5baf927f3646b42b61bb4e08c1204ace0f552f3c04def
|
data/.gitignore
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
pkg
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,55 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
fluent-plugin-s3-input (0.0.1)
|
5
|
+
aws-sdk
|
6
|
+
fluentd
|
7
|
+
oj
|
8
|
+
|
9
|
+
GEM
|
10
|
+
remote: https://rubygems.org/
|
11
|
+
specs:
|
12
|
+
aws-sdk (2.3.22)
|
13
|
+
aws-sdk-resources (= 2.3.22)
|
14
|
+
aws-sdk-core (2.3.22)
|
15
|
+
jmespath (~> 1.0)
|
16
|
+
aws-sdk-resources (2.3.22)
|
17
|
+
aws-sdk-core (= 2.3.22)
|
18
|
+
cool.io (1.4.4)
|
19
|
+
fluentd (0.14.1)
|
20
|
+
cool.io (>= 1.4.3, < 2.0.0)
|
21
|
+
http_parser.rb (>= 0.5.1, < 0.7.0)
|
22
|
+
json (>= 1.4.3)
|
23
|
+
msgpack (>= 0.7.0)
|
24
|
+
serverengine (>= 1.6.4)
|
25
|
+
sigdump (~> 0.2.2)
|
26
|
+
strptime (>= 0.1.7)
|
27
|
+
tzinfo (>= 1.0.0)
|
28
|
+
tzinfo-data (>= 1.0.0)
|
29
|
+
yajl-ruby (~> 1.0)
|
30
|
+
http_parser.rb (0.6.0)
|
31
|
+
jmespath (1.3.0)
|
32
|
+
json (2.0.1)
|
33
|
+
msgpack (1.0.0)
|
34
|
+
oj (2.17.1)
|
35
|
+
rake (11.2.2)
|
36
|
+
serverengine (1.6.4)
|
37
|
+
sigdump (~> 0.2.2)
|
38
|
+
sigdump (0.2.4)
|
39
|
+
strptime (0.1.8)
|
40
|
+
thread_safe (0.3.5)
|
41
|
+
tzinfo (1.2.2)
|
42
|
+
thread_safe (~> 0.1)
|
43
|
+
tzinfo-data (1.2016.6)
|
44
|
+
tzinfo (>= 1.0.0)
|
45
|
+
yajl-ruby (1.2.1)
|
46
|
+
|
47
|
+
PLATFORMS
|
48
|
+
ruby
|
49
|
+
|
50
|
+
DEPENDENCIES
|
51
|
+
fluent-plugin-s3-input!
|
52
|
+
rake
|
53
|
+
|
54
|
+
BUNDLED WITH
|
55
|
+
1.12.5
|
data/README.md
ADDED
@@ -0,0 +1,59 @@
|
|
1
|
+
# fluent-plugin-s3-input
|
2
|
+
Fluentd plugin that will read a json file from S3. This plugin can be used to
|
3
|
+
handle S3 Event Notifications such as cloudtrail API logs
|
4
|
+
|
5
|
+
# Usage
|
6
|
+
|
7
|
+
S3 Event Example Intake
|
8
|
+
|
9
|
+
# Get Notified of JSON document in S3
|
10
|
+
<source>
|
11
|
+
type sqs
|
12
|
+
tag sqs.s3.event
|
13
|
+
sqs_url SQS_S3_NOTFICATION_QUEUE
|
14
|
+
receive_interval 1
|
15
|
+
max_number_of_messages 10
|
16
|
+
wait_time_seconds 20
|
17
|
+
</source>
|
18
|
+
|
19
|
+
# Remove SQS encapsulation
|
20
|
+
<filter sqs.s3.events>
|
21
|
+
type parser
|
22
|
+
format json
|
23
|
+
key_name body
|
24
|
+
</filter>
|
25
|
+
|
26
|
+
# Transform to Array
|
27
|
+
<match sqs.s3.events>
|
28
|
+
type record_splitter
|
29
|
+
split_key Records
|
30
|
+
tag sqs.s3.event
|
31
|
+
</match>
|
32
|
+
|
33
|
+
# extract s3_path
|
34
|
+
<filter sqs.s3.event>
|
35
|
+
@type record_transformer
|
36
|
+
enable_ruby
|
37
|
+
renew_record true
|
38
|
+
<record>
|
39
|
+
s3_bucket ${record['s3']['object']['key']}
|
40
|
+
s3_key ${record['s3']['bucket']['name']}
|
41
|
+
</record>
|
42
|
+
</filter>
|
43
|
+
|
44
|
+
# read and emit the json object
|
45
|
+
# this plugin!
|
46
|
+
<match sqs.s3.event>
|
47
|
+
type s3_input
|
48
|
+
s3_bucket_key s3_bucket
|
49
|
+
s3_object_key_key s3_object
|
50
|
+
uncompress gzip
|
51
|
+
tag s3.file.contents
|
52
|
+
</filter>
|
53
|
+
|
54
|
+
# Emit each record in the cloudtrail json document as a new event
|
55
|
+
<match s3.file.contents>
|
56
|
+
type record_splitter
|
57
|
+
split_key Records
|
58
|
+
tag cloudtrail
|
59
|
+
</match>
|
data/Rakefile
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
|
5
|
+
Gem::Specification.new do |spec|
|
6
|
+
spec.name = "fluent-plugin-s3-input"
|
7
|
+
spec.version = "0.0.1"
|
8
|
+
spec.authors = ["Anthony Johnson"]
|
9
|
+
spec.email = ["ansoni@gmail.com"]
|
10
|
+
spec.description = %q{Fluentd plugin to read a file from S3 and emit it}
|
11
|
+
spec.summary = %q{Fluentd plugin to read a file from S3 and emit it}
|
12
|
+
spec.homepage = "https://github.com/ansoni/fluent-plugin-s3-input"
|
13
|
+
spec.license = "MIT"
|
14
|
+
|
15
|
+
spec.files = `git ls-files`.split($/)
|
16
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
17
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
18
|
+
spec.require_paths = ["lib"]
|
19
|
+
|
20
|
+
spec.add_development_dependency "rake"
|
21
|
+
spec.add_runtime_dependency "fluentd"
|
22
|
+
spec.add_runtime_dependency "aws-sdk"
|
23
|
+
spec.add_runtime_dependency "oj"
|
24
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
|
2
|
+
module Fluent
|
3
|
+
class S3InputOutput < Output
|
4
|
+
|
5
|
+
Fluent::Plugin.register_output('s3_input', self)
|
6
|
+
|
7
|
+
# Define `router` method of v0.12 to support v0.10 or earlier
|
8
|
+
unless method_defined?(:router)
|
9
|
+
define_method("router") { Fluent::Engine }
|
10
|
+
end
|
11
|
+
|
12
|
+
config_param :aws_key_id, :string, :default => ENV['AWS_ACCESS_KEY_ID'], :secret => true
|
13
|
+
config_param :aws_sec_key, :string, :default => ENV['AWS_SECRET_ACCESS_KEY'], :secret => true
|
14
|
+
config_param :s3_bucket_key
|
15
|
+
config_param :s3_object_key_key
|
16
|
+
config_param :tag
|
17
|
+
# supports: gzip
|
18
|
+
config_param :uncompress, :string
|
19
|
+
|
20
|
+
attr_accessor :s3
|
21
|
+
|
22
|
+
def initialize
|
23
|
+
super
|
24
|
+
require 'net/http'
|
25
|
+
require 'oj'
|
26
|
+
require 'aws-sdk'
|
27
|
+
end
|
28
|
+
|
29
|
+
def configure(conf)
|
30
|
+
super
|
31
|
+
if @aws_key_id and @aws_sec_key
|
32
|
+
@s3 = Aws::S3::Client.new(
|
33
|
+
region: "us-east-1",
|
34
|
+
access_key_id: @aws_key_id,
|
35
|
+
secret_access_key: @aws_sec_key,
|
36
|
+
)
|
37
|
+
else
|
38
|
+
@s3 = Aws::S3::Client.new(region: "us-east-1")
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def emit(tag, es, chain)
|
43
|
+
tag_parts = tag.split('.')
|
44
|
+
es.each { |time, record|
|
45
|
+
s3_bucket = record[s3_bucket_key]
|
46
|
+
s3_key = record[s3_object_key_key]
|
47
|
+
resp = s3.get_object(bucket: s3_bucket, key: s3_key)
|
48
|
+
if @uncompress && @uncompress == "gzip"
|
49
|
+
input = Zlib::GzipReader.new(resp.body)
|
50
|
+
else
|
51
|
+
input = resp.body
|
52
|
+
end
|
53
|
+
new_record = Oj.load(input.read)
|
54
|
+
router.emit(@tag, time, new_record)
|
55
|
+
}
|
56
|
+
chain.next
|
57
|
+
rescue => e
|
58
|
+
$log.warn "s3_input: #{e.class} #{e.message} #{e.backtrace.join(', ')}"
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
metadata
ADDED
@@ -0,0 +1,107 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-s3-input
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Anthony Johnson
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-07-18 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: rake
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: fluentd
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: aws-sdk
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: oj
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
description: Fluentd plugin to read a file from S3 and emit it
|
70
|
+
email:
|
71
|
+
- ansoni@gmail.com
|
72
|
+
executables: []
|
73
|
+
extensions: []
|
74
|
+
extra_rdoc_files: []
|
75
|
+
files:
|
76
|
+
- ".gitignore"
|
77
|
+
- Gemfile
|
78
|
+
- Gemfile.lock
|
79
|
+
- README.md
|
80
|
+
- Rakefile
|
81
|
+
- fluent-plugin-s3-input.gemspec
|
82
|
+
- lib/fluent/plugin/out_s3_input.rb
|
83
|
+
homepage: https://github.com/ansoni/fluent-plugin-s3-input
|
84
|
+
licenses:
|
85
|
+
- MIT
|
86
|
+
metadata: {}
|
87
|
+
post_install_message:
|
88
|
+
rdoc_options: []
|
89
|
+
require_paths:
|
90
|
+
- lib
|
91
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
92
|
+
requirements:
|
93
|
+
- - ">="
|
94
|
+
- !ruby/object:Gem::Version
|
95
|
+
version: '0'
|
96
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
97
|
+
requirements:
|
98
|
+
- - ">="
|
99
|
+
- !ruby/object:Gem::Version
|
100
|
+
version: '0'
|
101
|
+
requirements: []
|
102
|
+
rubyforge_project:
|
103
|
+
rubygems_version: 2.5.1
|
104
|
+
signing_key:
|
105
|
+
specification_version: 4
|
106
|
+
summary: Fluentd plugin to read a file from S3 and emit it
|
107
|
+
test_files: []
|