fluent-plugin-cloudtrail 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c05ee14755f1d40a3b69861617604e852f7fbf6c
+  data.tar.gz: c5d6dc68799c5cb2d714515a06edf2b788a05faa
+SHA512:
+  metadata.gz: dd5fd648bb46d8b36b6878d32a0b6d4ad50455860684ff2de5304a3e25ec45570919c15351decc9eb6529d7e6bf753d3473d498a8e8c94d66a4f5751a55f9d67
+  data.tar.gz: 97eb583ae6103f70ed133d483dc92fbe91d1e95f69677c78df1bd0e0b88b9d5eb47c2f3f9b390d4bdf89fbec5a5f477c71848ccea80ad45af44097f9230187e9

data/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016 Blend Labs, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/lib/fluent/plugin/in_cloudtrail.rb

@@ -0,0 +1,148 @@
+require 'json'
+require 'stringio'
+require 'zlib'
+require 'aws-sdk'
+
+module Fluent
+
+  class CloudtrailInput < Input
+    USER_AGENT_NAME = 'fluent-plugin-cloudtrail-in'
+    PLUGIN_VERSION = '0.0.1'
+
+    Plugin.register_input('cloudtrail', self)
+
+    define_method('log')    { $log }   unless method_defined?(:log)
+    define_method('router') { Engine } unless method_defined?(:router)
+
+    config_param :aws_key_id,  :string, :default => nil, :secret => true
+    config_param :aws_sec_key, :string, :default => nil, :secret => true
+    # The 'region' parameter is optional because
+    # it may be set as an environment variable.
+    config_param :region, :string, :default => nil
+
+    config_param :profile,          :string, :default => nil
+    config_param :credentials_path, :string, :default => nil
+    config_param :role_arn,         :string, :default => nil
+    config_param :external_id,      :string, :default => nil
+
+    config_param :sqs_url, :string
+    config_param :receive_interval, :time, :default => 0.1
+    config_param :max_number_of_messages, :integer, :default => 10
+    config_param :wait_time_seconds, :integer, :default => 10
+
+    config_param :http_proxy, :string, :default => nil
+    config_param :debug, :bool, :default => false
+
+    config_param :tag, :string
+
+    def configure(conf)
+      super
+    end
+
+    def initialize
+      super
+    end
+
+    def start
+      super
+      load_clients
+      @finished = false
+      @thread = Thread.new(&method(:run_periodic))
+    end
+
+    def shutdown
+      super
+      @finished = true
+      @thread.join
+    end
+
+    def load_clients
+      user_agent_suffix = "#{USER_AGENT_NAME}/#{PLUGIN_VERSION}"
+      options = {
+        user_agent_suffix: user_agent_suffix
+      }
+      if @region
+        options[:region] = @region
+      end
+
+      if @aws_key_id && @aws_sec_key
+        options.update(
+          access_key_id: @aws_key_id,
+          secret_access_key: @aws_sec_key,
+        )
+      elsif @profile
+        credentials_opts = {:profile_name => @profile}
+        credentials_opts[:path] = @credentials_path if @credentials_path
+        credentials = Aws::SharedCredentials.new(credentials_opts)
+        options[:credentials] = credentials
+      elsif @role_arn
+        credentials = Aws::AssumeRoleCredentials.new(
+          client: Aws::STS::Client.new(options),
+          role_arn: @role_arn,
+          role_session_name: "fluent-plugin-cloudtrail",
+          external_id: @external_id,
+          duration_seconds: 60 * 60,
+        )
+        options[:credentials] = credentials
+      end
+
+      if @debug
+        options.update(
+          logger: Logger.new(log.out),
+          log_level: :debug
+        )
+        # XXX: Add the following options, if necessary
+        # :http_wire_trace => true
+      end
+
+      if @http_proxy
+        options[:http_proxy] = @http_proxy
+      end
+
+      @s3_client = Aws::S3::Client.new(options)
+      @sqs_client = Aws::SQS::Client.new(options)
+    end
+
+    def run_periodic
+      until @finished
+        begin
+          sleep @receive_interval
+          sqs_resp = @sqs_client.receive_message(
+            queue_url: @sqs_url,
+            max_number_of_messages: @max_number_of_messages,
+            wait_time_seconds: @wait_time_seconds
+          )
+          for message in sqs_resp.messages
+            body_obj = JSON.parse(message.body)
+            message_obj = JSON.parse(body_obj['Message'])
+            s3_bucket = message_obj['s3Bucket']
+            for s3_object_key in message_obj['s3ObjectKey']
+              s3_resp = @s3_client.get_object(
+                :bucket => s3_bucket,
+                :key => s3_object_key
+              )
+              io = StringIO.new
+              io.write s3_resp.body.read
+              io.rewind
+              gz = Zlib::GzipReader.new(io)
+              cloudtrail_data = gz.read
+              gz.close
+              cloudtrail_records = JSON.parse(cloudtrail_data)['Records']
+              for record in cloudtrail_records
+                router.emit(@tag, Time.now.to_i, record)
+              end
+            end
+
+            @sqs_client.delete_message(
+              queue_url: @sqs_url,
+              receipt_handle: message.receipt_handle
+            )
+          end
+        rescue
+          log.error "failed to emit", :error => $!.to_s, :error_class => $!.class.to_s
+          log.warn_backtrace $!.backtrace
+        end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,79 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-cloudtrail
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Craig Buchanan
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-01-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.58
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.58
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+description: 
+email: craig+rubygems@blendlabs.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- lib/fluent/plugin/in_cloudtrail.rb
+homepage: https://github.com/blend/fluent-plugin-cloudtrail
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.8
+signing_key: 
+specification_version: 4
+summary: Fluentd input plugin that inputs logs from AWS CloudTrail.
+test_files: []