fluent-plugin-s3-input 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: abd2bf6ecf2453a5d47cf8702ff3fe95f50dbf79
4
+ data.tar.gz: 9c73ec74af0ef99f899d0dec6e225ee4abd105dc
5
+ SHA512:
6
+ metadata.gz: feed1255cafe0de56b2a0df87aa82093e2b07f87a5372f51512527ad4bb2db3c37177e017908aca09f041499d3e70232436101bb1b429c116a27d95ef94da3eb
7
+ data.tar.gz: 664811196401b7a22cb7078ad39de99a27f865dd7602d351327c595c991e7f54bbf923e7199b078871a5baf927f3646b42b61bb4e08c1204ace0f552f3c04def
data/.gitignore ADDED
@@ -0,0 +1 @@
1
+ pkg
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gemspec
data/Gemfile.lock ADDED
@@ -0,0 +1,55 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ fluent-plugin-s3-input (0.0.1)
5
+ aws-sdk
6
+ fluentd
7
+ oj
8
+
9
+ GEM
10
+ remote: https://rubygems.org/
11
+ specs:
12
+ aws-sdk (2.3.22)
13
+ aws-sdk-resources (= 2.3.22)
14
+ aws-sdk-core (2.3.22)
15
+ jmespath (~> 1.0)
16
+ aws-sdk-resources (2.3.22)
17
+ aws-sdk-core (= 2.3.22)
18
+ cool.io (1.4.4)
19
+ fluentd (0.14.1)
20
+ cool.io (>= 1.4.3, < 2.0.0)
21
+ http_parser.rb (>= 0.5.1, < 0.7.0)
22
+ json (>= 1.4.3)
23
+ msgpack (>= 0.7.0)
24
+ serverengine (>= 1.6.4)
25
+ sigdump (~> 0.2.2)
26
+ strptime (>= 0.1.7)
27
+ tzinfo (>= 1.0.0)
28
+ tzinfo-data (>= 1.0.0)
29
+ yajl-ruby (~> 1.0)
30
+ http_parser.rb (0.6.0)
31
+ jmespath (1.3.0)
32
+ json (2.0.1)
33
+ msgpack (1.0.0)
34
+ oj (2.17.1)
35
+ rake (11.2.2)
36
+ serverengine (1.6.4)
37
+ sigdump (~> 0.2.2)
38
+ sigdump (0.2.4)
39
+ strptime (0.1.8)
40
+ thread_safe (0.3.5)
41
+ tzinfo (1.2.2)
42
+ thread_safe (~> 0.1)
43
+ tzinfo-data (1.2016.6)
44
+ tzinfo (>= 1.0.0)
45
+ yajl-ruby (1.2.1)
46
+
47
+ PLATFORMS
48
+ ruby
49
+
50
+ DEPENDENCIES
51
+ fluent-plugin-s3-input!
52
+ rake
53
+
54
+ BUNDLED WITH
55
+ 1.12.5
data/README.md ADDED
@@ -0,0 +1,59 @@
1
+ # fluent-plugin-s3-input
2
+ Fluentd plugin that will read a json file from S3. This plugin can be used to
3
+ handle S3 Event Notifications such as cloudtrail API logs
4
+
5
+ # Usage
6
+
7
+ S3 Event Example Intake
8
+
9
+ # Get Notified of JSON document in S3
10
+ <source>
11
+ type sqs
12
+ tag sqs.s3.event
13
+ sqs_url SQS_S3_NOTFICATION_QUEUE
14
+ receive_interval 1
15
+ max_number_of_messages 10
16
+ wait_time_seconds 20
17
+ </source>
18
+
19
+ # Remove SQS encapsulation
20
+ <filter sqs.s3.events>
21
+ type parser
22
+ format json
23
+ key_name body
24
+ </filter>
25
+
26
+ # Transform to Array
27
+ <match sqs.s3.events>
28
+ type record_splitter
29
+ split_key Records
30
+ tag sqs.s3.event
31
+ </match>
32
+
33
+ # extract s3_path
34
+ <filter sqs.s3.event>
35
+ @type record_transformer
36
+ enable_ruby
37
+ renew_record true
38
+ <record>
39
+ s3_bucket ${record['s3']['object']['key']}
40
+ s3_key ${record['s3']['bucket']['name']}
41
+ </record>
42
+ </filter>
43
+
44
+ # read and emit the json object
45
+ # this plugin!
46
+ <match sqs.s3.event>
47
+ type s3_input
48
+ s3_bucket_key s3_bucket
49
+ s3_object_key_key s3_object
50
+ uncompress gzip
51
+ tag s3.file.contents
52
+ </filter>
53
+
54
+ # Emit each record in the cloudtrail json document as a new event
55
+ <match s3.file.contents>
56
+ type record_splitter
57
+ split_key Records
58
+ tag cloudtrail
59
+ </match>
data/Rakefile ADDED
@@ -0,0 +1,10 @@
1
+ require "bundler/gem_tasks"
2
+
3
+ require 'rake/testtask'
4
+ Rake::TestTask.new(:test) do |test|
5
+ test.libs << 'lib' << 'test'
6
+ test.pattern = 'test/**/test_*.rb'
7
+ test.verbose = true
8
+ end
9
+
10
+ task :default => :test
@@ -0,0 +1,24 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+
5
+ Gem::Specification.new do |spec|
6
+ spec.name = "fluent-plugin-s3-input"
7
+ spec.version = "0.0.1"
8
+ spec.authors = ["Anthony Johnson"]
9
+ spec.email = ["ansoni@gmail.com"]
10
+ spec.description = %q{Fluentd plugin to read a file from S3 and emit it}
11
+ spec.summary = %q{Fluentd plugin to read a file from S3 and emit it}
12
+ spec.homepage = "https://github.com/ansoni/fluent-plugin-s3-input"
13
+ spec.license = "MIT"
14
+
15
+ spec.files = `git ls-files`.split($/)
16
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
17
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
18
+ spec.require_paths = ["lib"]
19
+
20
+ spec.add_development_dependency "rake"
21
+ spec.add_runtime_dependency "fluentd"
22
+ spec.add_runtime_dependency "aws-sdk"
23
+ spec.add_runtime_dependency "oj"
24
+ end
@@ -0,0 +1,61 @@
1
+
2
+ module Fluent
3
+ class S3InputOutput < Output
4
+
5
+ Fluent::Plugin.register_output('s3_input', self)
6
+
7
+ # Define `router` method of v0.12 to support v0.10 or earlier
8
+ unless method_defined?(:router)
9
+ define_method("router") { Fluent::Engine }
10
+ end
11
+
12
+ config_param :aws_key_id, :string, :default => ENV['AWS_ACCESS_KEY_ID'], :secret => true
13
+ config_param :aws_sec_key, :string, :default => ENV['AWS_SECRET_ACCESS_KEY'], :secret => true
14
+ config_param :s3_bucket_key
15
+ config_param :s3_object_key_key
16
+ config_param :tag
17
+ # supports: gzip
18
+ config_param :uncompress, :string
19
+
20
+ attr_accessor :s3
21
+
22
+ def initialize
23
+ super
24
+ require 'net/http'
25
+ require 'oj'
26
+ require 'aws-sdk'
27
+ end
28
+
29
+ def configure(conf)
30
+ super
31
+ if @aws_key_id and @aws_sec_key
32
+ @s3 = Aws::S3::Client.new(
33
+ region: "us-east-1",
34
+ access_key_id: @aws_key_id,
35
+ secret_access_key: @aws_sec_key,
36
+ )
37
+ else
38
+ @s3 = Aws::S3::Client.new(region: "us-east-1")
39
+ end
40
+ end
41
+
42
+ def emit(tag, es, chain)
43
+ tag_parts = tag.split('.')
44
+ es.each { |time, record|
45
+ s3_bucket = record[s3_bucket_key]
46
+ s3_key = record[s3_object_key_key]
47
+ resp = s3.get_object(bucket: s3_bucket, key: s3_key)
48
+ if @uncompress && @uncompress == "gzip"
49
+ input = Zlib::GzipReader.new(resp.body)
50
+ else
51
+ input = resp.body
52
+ end
53
+ new_record = Oj.load(input.read)
54
+ router.emit(@tag, time, new_record)
55
+ }
56
+ chain.next
57
+ rescue => e
58
+ $log.warn "s3_input: #{e.class} #{e.message} #{e.backtrace.join(', ')}"
59
+ end
60
+ end
61
+ end
metadata ADDED
@@ -0,0 +1,107 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-s3-input
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Anthony Johnson
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2016-07-18 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: rake
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: fluentd
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: aws-sdk
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: oj
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ description: Fluentd plugin to read a file from S3 and emit it
70
+ email:
71
+ - ansoni@gmail.com
72
+ executables: []
73
+ extensions: []
74
+ extra_rdoc_files: []
75
+ files:
76
+ - ".gitignore"
77
+ - Gemfile
78
+ - Gemfile.lock
79
+ - README.md
80
+ - Rakefile
81
+ - fluent-plugin-s3-input.gemspec
82
+ - lib/fluent/plugin/out_s3_input.rb
83
+ homepage: https://github.com/ansoni/fluent-plugin-s3-input
84
+ licenses:
85
+ - MIT
86
+ metadata: {}
87
+ post_install_message:
88
+ rdoc_options: []
89
+ require_paths:
90
+ - lib
91
+ required_ruby_version: !ruby/object:Gem::Requirement
92
+ requirements:
93
+ - - ">="
94
+ - !ruby/object:Gem::Version
95
+ version: '0'
96
+ required_rubygems_version: !ruby/object:Gem::Requirement
97
+ requirements:
98
+ - - ">="
99
+ - !ruby/object:Gem::Version
100
+ version: '0'
101
+ requirements: []
102
+ rubyforge_project:
103
+ rubygems_version: 2.5.1
104
+ signing_key:
105
+ specification_version: 4
106
+ summary: Fluentd plugin to read a file from S3 and emit it
107
+ test_files: []