fluent-plugin-cloudwatch-logs 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 82f4aa50c7c50dd4fb39644824405fe597e44dbc
+  data.tar.gz: 7af64b0786713b5afaa6aa9e15c4a5c2b14cf41e
+SHA512:
+  metadata.gz: 0e96bd6e04ce768e9b5ace8492ff5e77d8de77f8203416711a4acb7efdfe5ddea805503c875434884d6e8b6cbebab991b19b7f492c7c9fce2fdb53a3a7ac8370
+  data.tar.gz: c204752a267e1083c8ff1e5ff2237611fc44a2293d59247761242fc8c5d1ffaddc9da64773ecd2f00cbe21506604b5ec072d14b2d17aca733b5af69fba9126d9

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-cloudwatch-logs.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Ryota Arai
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,86 @@
+# fluent-plugin-cloudwatch-logs
+
+CloudWatch Logs Plugin for Fluentd
+
+## Installation
+
+    $ gem install fluent-plugin-cloudwatch-logs
+
+## Example
+
+Start fluentd:
+
+```
+$ fluentd -c example/fluentd.conf
+```
+
+Send sample log to CloudWatch Logs:
+
+```
+$ echo '{"hello":"world"}' | fluent-cat test.cloudwatch_logs.out
+```
+
+Fetch sample log from CloudWatch Logs:
+
+```
+# stdout
+2014-07-17 00:28:02 +0900 test.cloudwatch_logs.in: {"hello":"world"}
+```
+
+## out_cloudwatch_logs
+
+```
+<match tag>
+  type cloudwatch_logs
+  log_group_name log-group-name
+  log_stream_name log-stream-name
+  sequence_token_file /var/lib/fluent/group_stream.out.seq
+  auto_create_stream true
+</match>
+```
+
+## in_cloudwatch_logs
+
+```
+<source>
+  type cloudwatch_logs
+  tag cloudwatch.in
+  log_group_name group
+  log_stream_name stream
+  state_file /var/lib/fluent/group_stream.in.state
+</source>
+```
+
+## Test
+
+Set credentials:
+
+```
+$ export AWS_REGION=us-east-1
+$ export AWS_ACCESS_KEY_ID="YOUR_ACCESS_KEY"
+$ export AWS_SECRET_ACCESS_KEY="YOUR_SECRET_KEY"
+```
+
+Run tests:
+
+```
+$ rake test
+```
+
+## TODO
+
+* out_cloudwatch_logs
+  * if the data is too big for API, split into multiple requests
+  * format
+  * check data size
+* in_cloudwatch_logs
+  * format
+  * fallback to start_time because next_token expires after 24 hours
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/fluent-plugin-cloudwatch-logs/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,9 @@
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'test'
+  test.test_files = FileList['test/plugin/*.rb']
+end
+

data/example/fluentd.conf

@@ -0,0 +1,24 @@
+<source>
+  type forward
+</source>
+
+<source>
+  type cloudwatch_logs
+  tag test.cloudwatch_logs.in
+  log_group_name fluent-plugin-cloudwatch-example
+  log_stream_name fluent-plugin-cloudwatch-example
+  state_file /tmp/fluent-plugin-cloudwatch-example.state
+</source>
+
+<match test.cloudwatch_logs.out>
+  type cloudwatch_logs
+  log_group_name fluent-plugin-cloudwatch-example
+  log_stream_name fluent-plugin-cloudwatch-example
+  sequence_token_file /tmp/fluent-plugin-cloudwatch-example.seq
+  auto_create_stream true
+</match>
+
+<match test.cloudwatch_logs.in>
+  type stdout
+</match>
+

data/fluent-plugin-cloudwatch-logs.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'fluent/plugin/cloudwatch/logs/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-cloudwatch-logs"
+  spec.version       = Fluent::Plugin::Cloudwatch::Logs::VERSION
+  spec.authors       = ["Ryota Arai"]
+  spec.email         = ["ryota.arai@gmail.com"]
+  spec.summary       = %q{CloudWatch Logs Plugin for Fluentd}
+  spec.homepage      = "https://github.com/ryotarai/fluent-plugin-cloudwatch-logs"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'fluentd'
+  spec.add_dependency 'aws-sdk-core', '2.0.0.rc12'
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake"
+end

data/lib/fluent/plugin/cloudwatch/logs.rb

@@ -0,0 +1,11 @@
+require "fluent/plugin/cloudwatch/logs/version"
+
+module Fluent
+  module Plugin
+    module Cloudwatch
+      module Logs
+        # Your code goes here...
+      end
+    end
+  end
+end

data/lib/fluent/plugin/cloudwatch/logs/version.rb

@@ -0,0 +1,9 @@
+module Fluent
+  module Plugin
+    module Cloudwatch
+      module Logs
+        VERSION = "0.0.1"
+      end
+    end
+  end
+end

data/lib/fluent/plugin/in_cloudwatch_logs.rb

@@ -0,0 +1,70 @@
+module Fluent
+  class CloudwatchLogsInput < Input
+    Plugin.register_input('cloudwatch_logs', self)
+
+    config_param :tag, :string
+    config_param :log_group_name, :string
+    config_param :log_stream_name, :string
+    config_param :state_file, :string
+    config_param :fetch_interval, :time, default: 60
+
+    def initialize
+      super
+
+      require 'aws-sdk-core'
+      @logs = Aws::CloudWatchLogs.new
+    end
+
+    def start
+      @finished = false
+      @thread = Thread.new(&method(:run))
+    end
+
+    def shutdown
+      @finished = true
+      @thread.join
+    end
+
+    private
+    def next_token
+      return nil unless File.exist?(@state_file)
+      File.read(@state_file).chomp
+    end
+
+    def store_next_token(token)
+      open(@state_file, 'w') do |f|
+        f.write token
+      end
+    end
+
+    def run
+      @next_fetch_time = Time.now
+
+      until @finished
+        if Time.now > @next_fetch_time
+          @next_fetch_time += @fetch_interval
+
+          events = get_events
+          events.each do |event|
+            time = (event.timestamp / 1000).floor
+            record = JSON.parse(event.message)
+            Engine.emit(@tag, time, record)
+          end
+        end
+        sleep 1
+      end
+    end
+
+    def get_events
+      request = {
+        log_group_name: @log_group_name,
+        log_stream_name: @log_stream_name,
+      }
+      request[:next_token] = next_token if next_token
+      response = @logs.get_log_events(request)
+      store_next_token(response.next_forward_token)
+
+      response.events
+    end
+  end
+end

data/lib/fluent/plugin/out_cloudwatch_logs.rb

@@ -0,0 +1,81 @@
+module Fluent
+  class CloudwatchLogsOutput < BufferedOutput
+    Plugin.register_output('cloudwatch_logs', self)
+
+    config_param :log_group_name, :string
+    config_param :log_stream_name, :string
+    config_param :sequence_token_file, :string
+    config_param :auto_create_stream, :bool, default: false
+
+    unless method_defined?(:log)
+      define_method(:log) { $log }
+    end
+
+    def initialize
+      super
+
+      require 'aws-sdk-core'
+      @logs = Aws::CloudWatchLogs.new
+    end
+
+    def configure(conf)
+      super
+
+      create_stream if @auto_create_stream
+    end
+
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    def write(chunk)
+      events = []
+      chunk.msgpack_each do |tag, time, record|
+        time_ms = time * 1000
+        events << {timestamp: time_ms, message: record.to_json}
+      end
+      put_events(events)
+    end
+
+    private
+    def next_sequence_token
+      return nil unless File.exist?(@sequence_token_file)
+      open(@sequence_token_file) {|f| f.read }.chomp
+    end
+
+    def store_next_sequence_token(token)
+      open(@sequence_token_file, 'w') do |f|
+        f.write token
+      end
+    end
+
+    def put_events(events)
+      args = {
+        log_events: events,
+        log_group_name: @log_group_name,
+        log_stream_name: @log_stream_name,
+      }
+      args[:sequence_token] = next_sequence_token if next_sequence_token
+
+      response = @logs.put_log_events(args)
+      store_next_sequence_token(response.next_sequence_token)
+    end
+
+    def create_stream
+      log.debug "Creating log stream '#{@log_stream_name}' in log group '#{@log_group_name}'"
+
+      begin
+        @logs.create_log_group(log_group_name: @log_group_name)
+      rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
+        log.debug "Log group '#{@log_group_name}' already exists"
+      end
+
+      begin
+        @logs.create_log_stream(log_group_name: @log_group_name, log_stream_name: @log_stream_name)
+      rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
+        log.debug "Log stream '#{@log_stream_name}' already exists"
+      end
+    end
+  end
+end
+

data/test/plugin/test_in_cloudwatch_logs.rb

@@ -0,0 +1,68 @@
+require 'test_helper'
+
+class CloudwatchLogsInputTest < Test::Unit::TestCase
+  include CloudwatchLogsTestHelper
+
+  def setup
+    Fluent::Test.setup
+    require 'fluent/plugin/in_cloudwatch_logs'
+
+  end
+
+  def teardown
+    clear_log_group
+  end
+
+  def test_configure
+    d = create_driver(<<-EOC)
+      type cloudwatch_logs
+      tag test
+      log_group_name group
+      log_stream_name stream
+      state_file /tmp/state
+    EOC
+
+    assert_equal('test', d.instance.tag)
+    assert_equal('group', d.instance.log_group_name)
+    assert_equal('stream', d.instance.log_stream_name)
+    assert_equal('/tmp/state', d.instance.state_file)
+  end
+
+  def test_emit
+    create_log_stream
+
+    time_ms = (Time.now.to_f * 1000).floor
+    put_log_events([
+      {timestamp: time_ms, message: '{"cloudwatch":"logs1"}'},
+      {timestamp: time_ms, message: '{"cloudwatch":"logs2"}'},
+    ])
+
+    sleep 5
+
+    d = create_driver
+    d.run do
+      sleep 5
+    end
+    
+    emits = d.emits
+    assert_equal(2, emits.size)
+    assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs1'}], emits[0])
+    assert_equal(['test', (time_ms / 1000).floor, {'cloudwatch' => 'logs2'}], emits[1])
+  end
+
+  private
+  def default_config
+    <<-EOC
+      tag test
+      type cloudwatch_logs
+      log_group_name #{log_group_name}
+      log_stream_name #{log_stream_name}
+      state_file /tmp/state
+    EOC
+  end
+
+  def create_driver(conf = default_config)
+    Fluent::Test::InputTestDriver.new(Fluent::CloudwatchLogsInput).configure(conf)
+  end
+end
+

data/test/plugin/test_out_cloudwatch_logs.rb

@@ -0,0 +1,73 @@
+require 'test_helper'
+require 'fileutils'
+
+class CloudwatchLogsOutputTest < Test::Unit::TestCase
+  include CloudwatchLogsTestHelper
+
+  def setup
+    Fluent::Test.setup
+    require 'fluent/plugin/out_cloudwatch_logs'
+
+    @logs = Aws::CloudWatchLogs.new
+  end
+
+  def teardown
+    clear_log_group
+    FileUtils.rm_f(sequence_token_file)
+  end
+  
+
+  def test_configure
+    d = create_driver(<<-EOC)
+      type cloudwatch_logs
+      log_group_name test_group
+      log_stream_name test_stream
+      sequence_token_file /tmp/sq
+      auto_create_stream false
+    EOC
+
+    assert_equal('test_group', d.instance.log_group_name)
+    assert_equal('test_stream', d.instance.log_stream_name)
+    assert_equal('/tmp/sq', d.instance.sequence_token_file)
+    assert_equal(false, d.instance.auto_create_stream)
+  end
+
+  def test_write
+    new_log_stream
+
+    d = create_driver
+    time = Time.now
+    d.emit({'cloudwatch' => 'logs1'}, time.to_i)
+    d.emit({'cloudwatch' => 'logs2'}, time.to_i + 1)
+    d.run
+
+    sleep 5
+
+    events = get_log_events
+    assert_equal(2, events.size)
+    assert_equal(time.to_i * 1000, events[0].timestamp)
+    assert_equal('{"cloudwatch":"logs1"}', events[0].message)
+    assert_equal((time.to_i + 1) * 1000, events[1].timestamp)
+    assert_equal('{"cloudwatch":"logs2"}', events[1].message)
+  end
+
+  private
+  def default_config
+    <<-EOC
+    type cloudwatch_logs
+    log_group_name #{log_group_name}
+    log_stream_name #{log_stream_name}
+    sequence_token_file #{sequence_token_file}
+    auto_create_stream true
+    EOC
+  end
+
+  def sequence_token_file
+    File.expand_path('../../tmp/sequence_token', __FILE__)
+  end
+
+
+  def create_driver(conf = default_config)
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::CloudwatchLogsOutput).configure(conf)
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,60 @@
+require 'test/unit'
+require 'fluent/test'
+
+require 'aws-sdk-core'
+
+module CloudwatchLogsTestHelper
+  private
+  def logs
+    @logs ||= Aws::CloudWatchLogs.new
+  end
+
+  def log_group_name
+    @log_group_name ||= "fluent-plugin-cloudwatch-test-#{Time.now.to_f}"
+  end
+
+  def log_stream_name
+    if !@log_stream_name
+      new_log_stream
+    end
+    @log_stream_name
+  end
+
+  def new_log_stream
+    @log_stream_name = Time.now.to_f.to_s
+  end
+
+  def clear_log_group
+    logs.delete_log_group(log_group_name: log_group_name)
+  rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
+    # pass
+  end
+
+  def create_log_stream
+    begin
+      logs.create_log_group(log_group_name: log_group_name)
+    rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
+      # pass
+    end
+
+    begin
+      logs.create_log_stream(log_group_name: log_group_name, log_stream_name: log_stream_name)
+    rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
+      # pass
+    end
+  end
+
+  def get_log_events
+    logs.get_log_events(log_group_name: log_group_name, log_stream_name: log_stream_name).events
+  end
+
+  def put_log_events(events)
+    args = {
+      log_events: events,
+      log_group_name: log_group_name,
+      log_stream_name: log_stream_name,
+    }
+    logs.put_log_events(args)
+  end
+end
+

metadata

@@ -0,0 +1,117 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-cloudwatch-logs
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Ryota Arai
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-07-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.0.0.rc12
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.0.0.rc12
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- ryota.arai@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- example/fluentd.conf
+- fluent-plugin-cloudwatch-logs.gemspec
+- lib/fluent/plugin/cloudwatch/logs.rb
+- lib/fluent/plugin/cloudwatch/logs/version.rb
+- lib/fluent/plugin/in_cloudwatch_logs.rb
+- lib/fluent/plugin/out_cloudwatch_logs.rb
+- test/plugin/test_in_cloudwatch_logs.rb
+- test/plugin/test_out_cloudwatch_logs.rb
+- test/test_helper.rb
+homepage: https://github.com/ryotarai/fluent-plugin-cloudwatch-logs
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: CloudWatch Logs Plugin for Fluentd
+test_files:
+- test/plugin/test_in_cloudwatch_logs.rb
+- test/plugin/test_out_cloudwatch_logs.rb
+- test/test_helper.rb