fluent-plugin-cloudfront-log 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e41573e85cd6b33115e5f1525475c09c8668c099
+  data.tar.gz: ade5e4dd5a8ba5bdcc52de52396840ea4fd8ef0e
+SHA512:
+  metadata.gz: 65fcc533f4fb58e998a60a2b6bbc9e9e48088b1b7c8d5268b25fd0aec426ed3a0e512100a7abe9255816c3e4b49544a0db015ef85e6ce36c9c36fcc7a7b8d43a
+  data.tar.gz: 353fb1744667a60477e2cb03b91d7007442766823922afeac18eca682b7bad2ac6fce969205adf7b0edaddaa5c52e554f4fea7e145e21fa2e12881e8dec50f91

data/.gitignore

@@ -0,0 +1,40 @@
+# Created by https://www.gitignore.io/api/ruby
+
+### Ruby ###
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+Gemfile.lock
+.ruby-version
+.ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-cloudfront-log.gemspec
+gemspec

data/README.md

@@ -0,0 +1,36 @@
+# Fluent::Plugin::Cloudfront::Log
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/fluent/plugin/cloudfront/log`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'fluent-plugin-cloudfront-log'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-cloudfront-log
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/fluent-plugin-cloudfront-log.
+

data/Rakefile

@@ -0,0 +1,11 @@
+require "bundler/gem_tasks"
+
+require "rake/testtask"
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+
+task :default => :test

data/fluent-plugin-cloudfront-log.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-cloudfront-log"
+  spec.version       = "0.0.1"
+  spec.authors       = ["kubihie"]
+  spec.email         = ["kubihie@gmail.com"]
+
+  spec.summary       = %q{AWS CloudFront log input plugin.}
+  spec.description   = %q{AWS CloudFront log input plugin for fluentd.}
+  spec.homepage      = "https://github.com/kubihie/fluent-plugin-cloudfront-log"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "fluentd", "~> 0"
+  spec.add_dependency "aws-sdk", "~> 2.1"
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency 'test-unit', "~> 2"
+end

data/lib/fluent/plugin/in_cloudfront_log.rb

@@ -0,0 +1,129 @@
+class Fluent::Cloudfront_LogInput < Fluent::Input
+  Fluent::Plugin.register_input('cloudfront_log', self)
+
+  config_param :aws_key_id,        :string,  :default => nil, :secret => true
+  config_param :aws_sec_key,       :string,  :default => nil, :secret => true
+  config_param :log_bucket,        :string
+  config_param :log_prefix,        :string,  :default => nil
+  config_param :moved_log_bucket,  :string,  :default => @log_bucket
+  config_param :moved_log_prefix,  :string,  :default => '_moved'
+  config_param :region,            :string
+  config_param :tag,               :string,  :default => 'cloudfront.access'
+  config_param :interval,          :integer,    :default => 300
+
+  def initialize
+    super
+    require 'logger'
+    require 'aws-sdk'
+    require 'zlib'
+    require 'time'
+    require 'uri'
+  end
+
+  def configure(conf)
+    super
+
+    raise Fluent::ConfigError.new unless @log_bucket
+    raise Fluent::ConfigError.new unless @region
+
+    @moved_log_bucket = @log_bucket unless @moved_log_bucket
+    @moved_log_prefix = @log_prefix + '_moved' unless @moved_log_prefix
+    
+    log.info('@log_bucket:' + @log_bucket)
+    log.info('@moved_log_bucket:' + @moved_log_bucket)
+    log.info('@log_prefix:' + @log_prefix)
+    log.info('@moved_log_prefix:' + @moved_log_prefix)
+  end
+
+  def start
+    super
+    client
+
+    @loop = Coolio::Loop.new
+    #timer = TimerWatcher.new(@interval, true, log) do
+    #  input
+    #end
+    timer = TimerWatcher.new(@interval, true, log, &method(:input)) 
+
+    @loop.attach(timer)
+    @thread = Thread.new(&method(:run))
+  end
+
+  def shutdown
+    @loop.stop
+    @thread.join
+  end
+
+  def run
+    @loop.run
+  end
+
+  def client
+    begin
+      options = {:region => @region}
+      if @aws_key_id and @aws_sec_key
+        options[:access_key_id] = @aws_key_id
+        options[:secret_access_key] = @aws_sec_key
+      end
+      @client = Aws::S3::Client.new(options)
+    rescue => e
+      log.warn("S3 client error. #{e.message}")
+    end
+  end
+
+  def parse_header(line)
+    case line
+    when /^#Version:.+/i then
+      @version = line.sub(/^#Version:/i, '').strip
+    when /^#Fields:.+/i then
+      @fields = line.sub(/^#Fields:/i, '').strip.split("\s")
+    end  
+  end
+
+  def purge(filename)
+    log.info('copy_src:' + @log_bucket + '/' + @log_prefix + filename)
+    log.info('copy_dst:' + @moved_log_bucket + '/' + @moved_log_prefix + '/' + filename)
+    client.copy_object(:bucket => @moved_log_bucket, :copy_source => @log_bucket + '/' + @log_prefix + filename, :key => @moved_log_prefix + '/' + filename)
+    client.delete_object(:bucket => @log_bucket, :key => @log_prefix + filename)
+  end
+
+  def input
+    client.list_objects(:bucket => @log_bucket, :prefix => @log_prefix , :delimiter => '/').each do |list|
+      list.contents.each do |content|
+        filename = content.key.sub(/#{@log_prefix}/, '')
+        #log.info('filename:' + filename)
+        next if filename[-1] == '/'  #skip directory/
+        next unless filename[-2, 2] == 'gz'  #skip without gz file
+
+        access_log_gz = client.get_object(:bucket => @log_bucket, :key => content.key).body
+        access_log = Zlib::GzipReader.new(access_log_gz).read
+
+        access_log.split("\n").each do |line|
+          if line[0.1] == '#'
+            parse_header(line)
+            next
+          end
+          line = URI.unescape(line)  #hoge%2520fuga -> hoge%20fuga
+          line = URI.unescape(line)  #hoge%20fuga   -> hoge fuga
+          line = line.split("\t")
+          record = Hash[@fields.collect.zip(line)]
+          timestamp = Time.parse("#{record['date']} #{record['time']} +0900").to_i
+          router.emit(@tag, timestamp, record)
+        end
+        purge(filename)
+      end
+    end
+  end
+
+  class TimerWatcher < Coolio::TimerWatcher
+    def initialize(interval, repeat, log, &callback)
+      @callback = callback
+      @log = log
+      super(interval, repeat)
+    end
+
+    def on_timer
+      @callback.call
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,28 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/in_cloudfront_log'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_in_cloudfrontlog.rb

@@ -0,0 +1,57 @@
+require_relative '../helper'
+require 'fluent/test'
+
+class Cloudfront_LogInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  DEFAULT_CONFIG = {
+    :aws_key_id        => 'AKIAZZZZZZZZZZZZZZZZ',
+    :aws_sec_key       => '1234567890qwertyuiopasdfghjklzxcvbnm',
+    :log_bucket        => 'bucket-name',
+    :log_prefix        => 'a/b/c',
+    :moved_log_bucket  => 'bucket-name-moved',
+    :moved_log_prefix  => 'a/b/c_moved',
+    :region            => 'ap-northeast-1',
+    :tag               => 'cloudfront',
+    :interval          => '500'
+  }
+
+  def parse_config(conf = {})
+    ''.tap{|s| conf.each { |k, v| s << "#{k} #{v}\n" } }
+  end
+
+  def create_driver(conf = DEFAULT_CONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::Cloudfront_LogInput).configure(parse_config conf)
+  end
+
+  def test_configure
+    assert_nothing_raised { driver = create_driver }
+
+    exception = assert_raise(Fluent::ConfigError) {
+      conf = DEFAULT_CONFIG.clone
+      conf.delete(:log_bucket)
+      driver = create_driver(conf)
+    }
+    assert_equal("'log_bucket' parameter is required", exception.message)
+
+    exception = assert_raise(Fluent::ConfigError) {
+      conf = DEFAULT_CONFIG.clone
+      conf.delete(:region)
+      driver = create_driver(conf)
+    }
+    assert_equal("'region' parameter is required", exception.message)
+
+    conf = DEFAULT_CONFIG.clone
+    conf.delete(:moved_log_bucket)
+    driver = create_driver(conf)
+    assert_equal(driver.instance.log_bucket, driver.instance.moved_log_bucket)
+
+    conf = DEFAULT_CONFIG.clone
+    conf.delete(:moved_log_prefix)
+    driver = create_driver(conf)
+    assert_equal('_moved', driver.instance.moved_log_prefix)
+  end
+
+end

metadata

@@ -0,0 +1,123 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-cloudfront-log
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- kubihie
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-03-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+description: AWS CloudFront log input plugin for fluentd.
+email:
+- kubihie@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- Rakefile
+- fluent-plugin-cloudfront-log.gemspec
+- lib/fluent/plugin/in_cloudfront_log.rb
+- test/helper.rb
+- test/plugin/test_in_cloudfrontlog.rb
+homepage: https://github.com/kubihie/fluent-plugin-cloudfront-log
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.0
+signing_key: 
+specification_version: 4
+summary: AWS CloudFront log input plugin.
+test_files:
+- test/helper.rb
+- test/plugin/test_in_cloudfrontlog.rb