fluent-plugin-dos_block_acl 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1c642a31a3a9ceceaea76768fd6b5df1b4d2cce1
+  data.tar.gz: 549d4678f4a934ee1727b6f0431a13d81e1b2108
+SHA512:
+  metadata.gz: 6b1ef004c755912178c81d4be04e78adb6d5b73999561d679fda813029ce99a1cb6e992da91c5e03518e5ed3b968e0f0ab73b8d026a8235350c7fbec483ed06d
+  data.tar.gz: c7934fff246674fe9d02d2a6749ce775d47d34846555a672a76f0a0d6eb1af98992a3d6c57e54167799b91ddd53f78da1db032b60a15dc5b2ba60be5265d9ce4

data/.document

@@ -0,0 +1,3 @@
+- 
+ChangeLog.md
+LICENSE.txt

data/.gitignore

@@ -0,0 +1,8 @@
+Gemfile.lock
+doc/
+pkg/
+**/*.gem
+.yardoc/
+.bundle/
+vendor/bundle/
+.tags

data/.travis.yml

@@ -0,0 +1,16 @@
+language: ruby
+
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1
+  - ruby-head
+
+os:
+  - linux
+  - osx
+
+gemfile:
+  - Gemfile
+
+script: 'bundle exec rake test'

data/.yardopts

@@ -0,0 +1 @@
+--markup markdown --title "fluent-plugin-dos_block_acl Documentation" --protected

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+gemspec
+
+group :development do
+  gem 'kramdown'
+end

data/README.md

@@ -0,0 +1,85 @@
+# fluent-plugin-dos_block_acl [![Build Status](https://secure.travis-ci.org/toyama0919/fluent-plugin-dos_block_acl.png?branch=master)](http://travis-ci.org/toyama0919/fluent-plugin-dos_block_acl)
+
+access block by aws network acl.
+
+aggregate unit is time_slice_format.
+
+## Installation
+```
+fluent-gem install fluent-plugin-dos_block_acl
+```
+
+## Examples(more than 10000 access per hour)
+```
+<match dos_block_acl.exsample>
+  type dos_block_acl
+  network_acl_id acl-xxxxxxx
+  ip_address_key ip_address
+  dos_threshold 10000
+  buffer_chunk_limit 256m
+  region ap-northeast-1
+  deny_rule_numbers_range 1..10
+  time_slice_format %Y%m%d_%H
+  buffer_path /tmp/dos_block_acl_hourly*.log
+  state_file /var/log/td-agent/buffer/dos_block_acl_state.log
+</match>
+```
+
+## Examples(more than 100000 access per day)
+```
+<match dos_block_acl.exsample>
+  type dos_block_acl
+  network_acl_id acl-xxxxxxx
+  ip_address_key ip_address
+  dos_threshold 10000
+  buffer_chunk_limit 256m
+  region ap-northeast-1
+  deny_rule_numbers_range 11..18
+  time_slice_format %Y%m%d
+  buffer_path /tmp/dos_block_acl_daily*.log
+  state_file /var/log/td-agent/buffer/dos_block_acl_state.log
+</match>
+```
+
+## parameter
+
+|param    | default|exsample|
+|--------|--------|--------|
+|network_acl_id||acl-xxxxxx|
+|dryrun| false|true|
+|ip_address_key||ip_address|
+|dos_threshold||1000|
+|time_slice_format |%Y%m%d|%Y%m%d_%H|
+|aws_key_id| nil||
+|aws_sec_key| nil||
+|region| nil|ap-northeast-1|
+|white_list| '127.0.0.1'|127.0.0.1,192.168.0.1,192.168.0.2|
+|deny_rule_numbers_range| '1..18'||
+|state_file| nil|/var/log/td-agent/dos_block_acl_state.log|
+
+
+
+## Notes
+
+default network acl entry limit is 20.([see](http://docs.aws.amazon.com/ja_jp/AmazonVPC/latest/UserGuide/VPC_Appendix_Limits.html))
+
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new [Pull Request](../../pull/new/master)
+
+## Information
+
+* [Homepage](https://github.com/toyama0919/fluent-plugin-dos_block_acl)
+* [Issues](https://github.com/toyama0919/fluent-plugin-dos_block_acl/issues)
+* [Documentation](http://rubydoc.info/gems/fluent-plugin-dos_block_acl/frames)
+* [Email](mailto:toyama0919@gmail.com)
+
+## Copyright
+
+Copyright (c) 2015 Hiroshi Toyama
+

data/Rakefile

@@ -0,0 +1,11 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test
+

data/exsample/exsample.conf

@@ -0,0 +1,17 @@
+<source>
+  type forward
+  port 24224
+</source>
+
+<match dos_block_acl.exsample>
+  type dos_block_acl
+  dryrun true
+  network_acl_id acl-xxxxxxx
+  ip_address_key ip_address
+  dos_threshold 10000
+  buffer_chunk_limit 256m
+  region ap-northeast-1
+  deny_rule_numbers_range 1..10
+  time_slice_format %Y%m%d_%H%M
+  state_file /var/log/td-agent/buffer/dos_block_acl_state.log
+</match>

data/fluent-plugin-dos_block_acl.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-dos_block_acl"
+  gem.version       = "0.0.1"
+  gem.summary       = %q{DosBlockAcl plugin for fluentd}
+  gem.description   = %q{DosBlockAcl plugin for fluentd}
+  gem.license       = "MIT"
+  gem.authors       = ["Hiroshi Toyama"]
+  gem.email         = "toyama0919@gmail.com"
+  gem.homepage      = "https://github.com/toyama0919/fluent-plugin-dos_block_acl"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ['lib']
+
+  gem.add_runtime_dependency "aws-sdk"
+  gem.add_development_dependency 'bundler', '~> 1.7.2'
+  gem.add_development_dependency 'fluentd', '~> 0.10.58'
+  gem.add_development_dependency 'pry', '~> 0.10.1'
+  gem.add_development_dependency 'rake', '~> 10.3.2'
+  gem.add_development_dependency 'rubocop', '~> 0.24.1'
+  gem.add_development_dependency 'rubygems-tasks', '~> 0.2'
+  gem.add_development_dependency 'yard', '~> 0.8'
+end

data/lib/fluent/plugin/out_dos_block_acl.rb

@@ -0,0 +1,171 @@
+module Fluent
+  class DosBlockAclOutput < TimeSlicedOutput
+    Fluent::Plugin.register_output('dos_block_acl', self)
+
+    config_param :network_acl_id, :string
+    config_param :dryrun, :bool, :default => false
+    config_param :ip_address_key, :string
+    config_param :dos_threshold, :integer
+    config_param :aws_key_id, :string, :default => nil
+    config_param :aws_sec_key, :string, :default => nil
+    config_param :region, :string, :default => nil
+    config_param :white_list, :string, :default => '127.0.0.1'
+    config_param :deny_rule_numbers_range, :string, :default => '1..18'
+    config_param :state_file, :string, :default => nil
+
+    def initialize
+      super
+      require 'aws-sdk'
+      require 'pathname'
+    end
+
+    def configure(conf)
+      super
+      @white_list = @white_list.split(',')
+      unless eval(@deny_rule_numbers_range).class == Range
+        raise Fluent::ConfigError, "out_dos_block_acl: @deny_rule_numbers_range is not Range!"
+      end
+      @deny_rule_numbers_range = eval(@deny_rule_numbers_range).to_a
+      @acl_entry_limit = @deny_rule_numbers_range.size
+    end
+
+    def start
+      super
+      AWS.config(access_key_id: @aws_key_id, secret_access_key: @aws_sec_key, region: @region)
+      @ec2 = AWS::EC2::Client.new
+      acls = @ec2.describe_network_acls(network_acl_ids: [@network_acl_id])
+      @allow_any_rule_number = acls[:network_acl_set].first[:entry_set].select {|r|
+                         !r[:egress] && r[:cidr_block] == "0.0.0.0/0" && r[:rule_action] == "allow"
+                       }.first[:rule_number]
+
+      state = @state_file ? load_status(@state_file) : nil
+
+      if state.nil?
+        @rule_numbers = get_deny_rule_numbers
+        @next_rule_index = get_next_rule_index
+      else
+        @rule_numbers = state[:rule_numbers]
+        @next_rule_index = state[:next_rule_index]
+      end
+      $log.info("out_dos_block_acl: use deny rule numbers => #{@rule_numbers}")
+    end
+
+    def shutdown
+      super
+      unless @state_file.nil?
+        save_status(@state_file)
+      end
+    end
+
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    def write(chunk)
+      begin
+        ip_addresses = []
+        chunk.msgpack_each do |(tag,time,record)|
+          ip_addresses << record[@ip_address_key]
+        end
+        counts = group_count(ip_addresses)
+        dos_hash = counts.select {|k, v| v >= @dos_threshold }
+        regist_deny_acl(dos_hash.keys)
+      rescue => e
+        $log.error("\n#{e.message}\n#{e.backtrace.join("\n")}")
+      end
+    end
+
+    private
+
+    def group_count(list)
+      Hash[list.group_by{ |e| e }.map{ |k, v| [k, v.length] }]
+    end
+
+    def regist_deny_acl(ip_addresses)
+      unless ip_addresses.empty?
+        ip_addresses.each do |ip_address|
+          next if @white_list.include?(ip_address)
+          deny_rules = get_deny_rules
+          unless deny_rules.any? {|r| r[:cidr_block] == "#{ip_address}/32"}
+            @next_rule_index = 0 if @rule_numbers.size == @next_rule_index
+            rule_number = @rule_numbers[@next_rule_index]
+            if deny_rules.any? {|r| r[:rule_number] == rule_number}
+              delete_deny_acl(rule_number)
+            end
+            option = {
+              network_acl_id: @network_acl_id,
+              rule_number: rule_number,
+              rule_action: 'deny',
+              protocol: '-1',
+              cidr_block: "#{ip_address}/32",
+              egress: false
+            }
+            @ec2.create_network_acl_entry(option) unless @dryrun
+            $log.info("netowork acl regist! deny ip_address => #{ip_address}, rule_number => #{rule_number}")
+            @next_rule_index = @next_rule_index + 1
+          end
+        end
+      end
+    end
+
+    def delete_deny_acl(rule_number)
+      option = {
+        network_acl_id: @network_acl_id,
+        rule_number: rule_number,
+        egress: false
+      }
+      $log.info("delete, rule_number => #{rule_number}")
+      @ec2.delete_network_acl_entry(option) unless @dryrun
+    end
+
+    def get_deny_rules
+      get_all_rules.select {|r|
+        !r[:egress] &&
+        r[:rule_number] < @allow_any_rule_number &&
+        @deny_rule_numbers_range.include?(r[:rule_number])
+      }.sort_by {|r| r[:rule_number]}
+    end
+
+    def get_deny_rule_numbers
+      get_deny_rules.map { |r| r[:rule_number]}.concat(@deny_rule_numbers_range).uniq.first(@acl_entry_limit)
+    end
+
+    def get_all_rules
+      acls = @ec2.describe_network_acls(network_acl_ids: [@network_acl_id])
+      acls[:network_acl_set].first[:entry_set]
+    end
+
+    def get_next_rule_index
+      deny_rules = get_deny_rules
+      next_rule_index = 0
+      if get_all_rules.size >= @acl_entry_limit
+        next_rule_index = 0
+      else
+        next_rule_index = deny_rules.empty? ? 0 : deny_rules.size
+      end
+      next_rule_index
+    end
+
+    def save_status(file_path)
+      begin
+        Pathname.new(file_path).open('wb') do |f|
+          Marshal.dump({
+            rule_numbers: @rule_numbers,
+            next_rule_index: @next_rule_index
+          }, f)
+        end
+      rescue => e
+        $log.warn "out_dos_block_acl: Can't write store_file #{e.class} #{e.message}"
+      end
+    end
+
+    def load_status(file_path)
+      return nil unless File.exist?(file_path)
+      state = Marshal.load(File.read(file_path))
+      state[:rule_numbers] = state[:rule_numbers].concat((1..@acl_entry_limit).to_a).uniq.first(@acl_entry_limit)
+      state[:next_rule_index] = state[:next_rule_index] > @acl_entry_limit ? 0 : state[:next_rule_index]
+      state
+    end
+  end
+end
+

data/test/helper.rb

@@ -0,0 +1,18 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+require 'fluent/plugin/out_dos_block_acl'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_out_dos_block_acl.rb

@@ -0,0 +1,59 @@
+require 'helper'
+class DosBlockAclOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  DEFAULT_CONFIG = %[
+  ]
+
+  def stub_seed_values
+    time = Time.parse("2014-01-01 00:00:00 UTC").to_i
+    records = [{"a" => 1}, {"a" => 2}]
+    return time, records
+  end
+
+  def create_driver(conf = DEFAULT_CONFIG)
+    config = %[
+      network_acl_id acl-xxxxxx
+      ip_address_key ip_address
+      dos_threshold 1000
+      time_slice_wait 10s
+      buffer_path tmp/buffer
+    ] + conf
+
+    Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::DosBlockAclOutput) do
+      def write(chunk)
+        chunk.instance_variable_set(:@key, @key)
+        super(chunk)
+      end
+    end.configure(config)
+  end
+
+  def test_configure
+    d = create_driver
+
+    {
+      :@dos_threshold => 1000,
+      :@deny_rule_numbers_range => (1..18).to_a,
+      :@ip_address_key => 'ip_address'
+    }.each { |k, v|
+      assert_equal(d.instance.instance_variable_get(k), v)
+    }
+  end
+
+  def test_configure_error
+    assert_raise(Fluent::ConfigError) do
+      create_driver(%[deny_rule_numbers_range 1])
+    end
+
+    assert_raise(Fluent::ConfigError) do
+      create_driver(%[deny_rule_numbers_range 1.10])
+    end
+  end
+
+  def test_emit
+    d = create_driver
+    time, records = stub_seed_values
+  end
+end

metadata

@@ -0,0 +1,170 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-dos_block_acl
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Hiroshi Toyama
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-01-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.7.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.7.2
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.58
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.58
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.1
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 10.3.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 10.3.2
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.24.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.24.1
+- !ruby/object:Gem::Dependency
+  name: rubygems-tasks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+description: DosBlockAcl plugin for fluentd
+email: toyama0919@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".document"
+- ".gitignore"
+- ".travis.yml"
+- ".yardopts"
+- Gemfile
+- README.md
+- Rakefile
+- exsample/exsample.conf
+- fluent-plugin-dos_block_acl.gemspec
+- lib/fluent/plugin/out_dos_block_acl.rb
+- test/helper.rb
+- test/plugin/test_out_dos_block_acl.rb
+homepage: https://github.com/toyama0919/fluent-plugin-dos_block_acl
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.2
+signing_key: 
+specification_version: 4
+summary: DosBlockAcl plugin for fluentd
+test_files:
+- test/helper.rb
+- test/plugin/test_out_dos_block_acl.rb
+has_rdoc: