fluent-plugin-s3-input 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: abd2bf6ecf2453a5d47cf8702ff3fe95f50dbf79
+  data.tar.gz: 9c73ec74af0ef99f899d0dec6e225ee4abd105dc
+SHA512:
+  metadata.gz: feed1255cafe0de56b2a0df87aa82093e2b07f87a5372f51512527ad4bb2db3c37177e017908aca09f041499d3e70232436101bb1b429c116a27d95ef94da3eb
+  data.tar.gz: 664811196401b7a22cb7078ad39de99a27f865dd7602d351327c595c991e7f54bbf923e7199b078871a5baf927f3646b42b61bb4e08c1204ace0f552f3c04def

data/.gitignore

@@ -0,0 +1 @@
+pkg

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,55 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-s3-input (0.0.1)
+      aws-sdk
+      fluentd
+      oj
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    aws-sdk (2.3.22)
+      aws-sdk-resources (= 2.3.22)
+    aws-sdk-core (2.3.22)
+      jmespath (~> 1.0)
+    aws-sdk-resources (2.3.22)
+      aws-sdk-core (= 2.3.22)
+    cool.io (1.4.4)
+    fluentd (0.14.1)
+      cool.io (>= 1.4.3, < 2.0.0)
+      http_parser.rb (>= 0.5.1, < 0.7.0)
+      json (>= 1.4.3)
+      msgpack (>= 0.7.0)
+      serverengine (>= 1.6.4)
+      sigdump (~> 0.2.2)
+      strptime (>= 0.1.7)
+      tzinfo (>= 1.0.0)
+      tzinfo-data (>= 1.0.0)
+      yajl-ruby (~> 1.0)
+    http_parser.rb (0.6.0)
+    jmespath (1.3.0)
+    json (2.0.1)
+    msgpack (1.0.0)
+    oj (2.17.1)
+    rake (11.2.2)
+    serverengine (1.6.4)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.4)
+    strptime (0.1.8)
+    thread_safe (0.3.5)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    tzinfo-data (1.2016.6)
+      tzinfo (>= 1.0.0)
+    yajl-ruby (1.2.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  fluent-plugin-s3-input!
+  rake
+
+BUNDLED WITH
+   1.12.5

data/README.md

@@ -0,0 +1,59 @@
+# fluent-plugin-s3-input
+Fluentd plugin that will read a json file from S3.  This plugin can be used to
+handle S3 Event Notifications such as cloudtrail API logs
+
+# Usage
+
+S3 Event Example Intake 
+
+	# Get Notified of JSON document in S3
+	<source>
+	  type sqs
+	  tag sqs.s3.event
+	  sqs_url SQS_S3_NOTFICATION_QUEUE
+	  receive_interval 1
+	  max_number_of_messages 10
+	  wait_time_seconds 20
+	</source>
+
+	# Remove SQS encapsulation
+	<filter sqs.s3.events>
+	  type parser
+	  format json
+	  key_name body
+	</filter>
+
+	# Transform to Array
+	<match sqs.s3.events>
+	  type record_splitter
+	  split_key Records
+	  tag sqs.s3.event
+	</match>
+
+	# extract s3_path
+	<filter sqs.s3.event>
+	  @type record_transformer
+	  enable_ruby
+	  renew_record true
+	  <record>
+            s3_bucket ${record['s3']['object']['key']}
+	    s3_key ${record['s3']['bucket']['name']}
+	  </record>
+	</filter>
+	
+	# read and emit the json object
+        # this plugin!
+	<match sqs.s3.event>
+	  type s3_input
+	  s3_bucket_key s3_bucket
+	  s3_object_key_key s3_object
+          uncompress gzip
+          tag s3.file.contents
+	</filter>
+
+	# Emit each record in the cloudtrail json document as a new event
+	<match s3.file.contents>
+	  type record_splitter
+	  split_key Records
+	  tag cloudtrail
+	</match>

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-s3-input.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-s3-input"
+  spec.version       = "0.0.1"
+  spec.authors       = ["Anthony Johnson"]
+  spec.email         = ["ansoni@gmail.com"]
+  spec.description   = %q{Fluentd plugin to read a file from S3 and emit it}
+  spec.summary       = %q{Fluentd plugin to read a file from S3 and emit it}
+  spec.homepage      = "https://github.com/ansoni/fluent-plugin-s3-input"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "rake"
+  spec.add_runtime_dependency     "fluentd"
+  spec.add_runtime_dependency     "aws-sdk"
+  spec.add_runtime_dependency     "oj"
+end

data/lib/fluent/plugin/out_s3_input.rb

@@ -0,0 +1,61 @@
+
+module Fluent
+  class S3InputOutput < Output
+
+    Fluent::Plugin.register_output('s3_input', self)
+
+    # Define `router` method of v0.12 to support v0.10 or earlier
+    unless method_defined?(:router)
+      define_method("router") { Fluent::Engine }
+    end
+
+    config_param :aws_key_id, :string, :default => ENV['AWS_ACCESS_KEY_ID'], :secret => true
+    config_param :aws_sec_key, :string, :default => ENV['AWS_SECRET_ACCESS_KEY'], :secret => true
+    config_param :s3_bucket_key
+    config_param :s3_object_key_key
+    config_param :tag
+    # supports: gzip
+    config_param :uncompress, :string
+
+    attr_accessor :s3
+
+    def initialize
+      super
+      require 'net/http'
+      require 'oj'
+      require 'aws-sdk'
+    end
+
+    def configure(conf)
+      super
+      if @aws_key_id and @aws_sec_key
+        @s3 = Aws::S3::Client.new(
+          region: "us-east-1",
+          access_key_id: @aws_key_id,
+          secret_access_key: @aws_sec_key,
+        )
+      else
+        @s3 = Aws::S3::Client.new(region: "us-east-1")
+      end
+    end
+
+    def emit(tag, es, chain)
+      tag_parts = tag.split('.')
+      es.each { |time, record|
+        s3_bucket = record[s3_bucket_key]
+        s3_key = record[s3_object_key_key]
+        resp = s3.get_object(bucket: s3_bucket, key: s3_key) 
+        if @uncompress && @uncompress == "gzip"
+          input = Zlib::GzipReader.new(resp.body)
+        else
+          input = resp.body
+        end
+        new_record = Oj.load(input.read)
+        router.emit(@tag, time, new_record)
+      }
+      chain.next
+    rescue => e
+      $log.warn "s3_input: #{e.class} #{e.message} #{e.backtrace.join(', ')}"
+    end
+  end
+end

metadata

@@ -0,0 +1,107 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-s3-input
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Anthony Johnson
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-07-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd plugin to read a file from S3 and emit it
+email:
+- ansoni@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- fluent-plugin-s3-input.gemspec
+- lib/fluent/plugin/out_s3_input.rb
+homepage: https://github.com/ansoni/fluent-plugin-s3-input
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Fluentd plugin to read a file from S3 and emit it
+test_files: []