fluent-plugin-sqs-check-size 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ece7c9d08385f16b07165fa5eff29ac03ad3cb96
+  data.tar.gz: ab459560bad394c5c842e296761c8abe6abddace
+SHA512:
+  metadata.gz: f33c92c6f95331ecfe50fea093bbfcc124cccb08fc756f6d539a1c9c78c41151a869921925805cf8a81c70b671f59f55c5a7dabb784b71ed97b4523912148d4c
+  data.tar.gz: ec3e43ef1f58a3a3130733e27c7f1d3cfa2d3bea53a68956ad27c2ea9bd7634a1cf782ec7a5abfb8ac655ddb6b8e2c12ee4e34cba3e030e16d5ed05c6706b9fb

data/AUTHORS

@@ -0,0 +1 @@
+Yuri Odagiri <ixixizko _at_ gmail.com>

data/README.rdoc

@@ -0,0 +1,92 @@
+= Amazon SQS output plugin for Fluent
+
+== Install
+ $ gem install fluent-plugin-sqs
+
+== Component
+
+=== SQSOutput
+
+Store fluent-event as queue message to amazon SQS.
+
+=== SQSInput
+
+Read events from from amazon SQS.
+
+== Configuratin
+
+=== SQSOutput
+
+    <match sqs.**>
+
+      type sqs
+
+      # following attibutes are required
+
+      queue_name {queue_instance_key}
+
+      # following attibutes are required if you don't use IAM Role nor export credentials to ENV
+
+      aws_key_id {your_aws_key_id}
+      aws_sec_key {your_aws_secret_key}
+
+      # following attibutes are optional
+    　
+      create_queue {boolean}
+      sqs_endpoint {endpointURL}
+
+      ### endpoint list ###
+      # Asia Pacific (Tokyo) [Default] : sqs.ap-northeast-1.amazonaws.com
+      # Asia Pacific (Singapore)       : sqs.ap-southeast-1.amazonaws.com
+      # US-East (Virginia)             : sqs.us-east-1.amazonaws.com
+      # US-West (Oregon)               : sqs.us-west-2.amazonaws.com
+      # US-West (N.California)         : sqs.us-west-1.amazonaws.com
+      # EU-West (Ireland)              : sqs.eu-west-1.amazonaws.com
+      # South America (São Paulo)      : sns.sa-east-1.amazonaws.com
+
+      delay_seconds {delivery_deley_seconds}
+
+      include_tag {boolean}
+      tag_property_name {tag's property name in json}
+
+
+    </match>
+
+=== SQSInput
+
+    <source>
+
+      type sqs
+
+      # following attibutes are required
+
+      sqs_url {queue_url}
+
+      # following attibutes are required if you don't use IAM Role nor export credentials to ENV
+
+      aws_key_id {your_aws_key_id}
+      aws_sec_key {your_aws_secret_key}
+
+      # following attibutes are optional
+    　
+      sqs_endpoint {endpointURL}
+
+      ### endpoint list ###
+      # Asia Pacific (Tokyo) [Default] : sqs.ap-northeast-1.amazonaws.com
+      # Asia Pacific (Singapore)       : sqs.ap-southeast-1.amazonaws.com
+      # US-East (Virginia)             : sqs.us-east-1.amazonaws.com
+      # US-West (Oregon)               : sqs.us-west-2.amazonaws.com
+      # US-West (N.California)         : sqs.us-west-1.amazonaws.com
+      # EU-West (Ireland)              : sqs.eu-west-1.amazonaws.com
+      # South America (São Paulo)      : sns.sa-east-1.amazonaws.com
+
+      receive_interval {receive_interval_seconds}
+
+    </source>
+
+== Tool
+
+== Copyright
+
+Copyright:: Copyright (c) 2011- Yuri Odagiri
+License::   Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,12 @@
+require 'rake'
+require 'rake/testtask'
+require 'rake/clean'
+require "bundler/gem_tasks"
+
+Rake::TestTask.new(:test) do |t|
+  t.test_files = Dir['test/*_test.rb']
+  t.ruby_opts = ['-rubygems'] if defined? Gem
+  t.ruby_opts << '-I.'
+end
+
+task :default => [:build]

data/VERSION

@@ -0,0 +1 @@
+1.6.0

data/lib/fluent/plugin/in_sqs.rb

@@ -0,0 +1,68 @@
+module Fluent
+
+  require 'aws-sdk'
+
+  class SQSInput < Input
+    Plugin.register_input('sqs', self)
+
+    def initialize
+      super
+    end
+
+    config_param :aws_key_id, :string, :default => nil, :secret => true
+    config_param :aws_sec_key, :string, :default => nil, :secret => true
+    config_param :tag, :string
+    config_param :sqs_endpoint, :string, :default => 'sqs.ap-northeast-1.amazonaws.com'
+    config_param :sqs_url, :string
+    config_param :receive_interval, :time, :default => 1
+    config_param :max_number_of_messages, :integer, :default => 1
+
+    def configure(conf)
+      super
+
+    end
+
+    def start
+      super
+
+      AWS.config(
+        :access_key_id => @aws_key_id,
+        :secret_access_key => @aws_sec_key
+        )
+
+      @queue = AWS::SQS.new(:sqs_endpoint => @sqs_endpoint).queues[@sqs_url]
+
+      @finished = false
+      @thread = Thread.new(&method(:run_periodic))
+    end
+
+    def shutdown
+      super
+
+      @finished = true
+      @thread.join
+    end
+
+    def run_periodic
+      until @finished
+        begin
+          sleep @receive_interval
+          @queue.receive_message(:limit => @max_number_of_messages) do |message|
+            record = {}
+            record['body'] = message.body.to_s
+            record['handle'] = message.handle.to_s
+            record['id'] = message.id.to_s
+            record['md5'] = message.md5.to_s
+            record['url'] = message.queue.url.to_s
+            record['sender_id'] = message.sender_id.to_s
+
+            Engine.emit(@tag, Time.now.to_i, record)
+          end
+        rescue
+          $log.error "failed to emit or receive", :error => $!.to_s, :error_class => $!.class.to_s
+          $log.warn_backtrace $!.backtrace
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_sqs.rb

@@ -0,0 +1,72 @@
+module Fluent
+
+    require 'aws-sdk'
+    require 'objspace'
+
+    class SQSOutput < BufferedOutput
+
+        Fluent::Plugin.register_output('sqs', self)
+
+        include SetTagKeyMixin
+        config_set_default :include_tag_key, false
+
+        include SetTimeKeyMixin
+        config_set_default :include_time_key, true
+
+        config_param :aws_key_id, :string, :default => nil, :secret => true
+        config_param :aws_sec_key, :string, :default => nil, :secret => true
+        config_param :queue_name, :string
+        config_param :create_queue, :bool, :default => true
+        config_param :sqs_endpoint, :string, :default => 'sqs.ap-northeast-1.amazonaws.com'
+        config_param :delay_seconds, :integer, :default => 0
+        config_param :include_tag, :bool, :default => true
+        config_param :tag_property_name, :string, :default => '__tag'
+	config_param :max_size, :integer, :default => 256000
+
+        def configure(conf)
+            super
+        end
+
+        def start
+            super
+
+            AWS.config(
+                :access_key_id => @aws_key_id,
+                :secret_access_key => @aws_sec_key)
+
+            @sqs = AWS::SQS.new(
+                :sqs_endpoint => @sqs_endpoint)
+            if @create_queue then
+                @queue = @sqs.queues.create(@queue_name)
+            else
+                @queue = @sqs.queues.named(@queue_name)
+            end
+        end
+
+        def shutdown
+            super
+        end
+
+        def format(tag, time, record)
+            if @include_tag then
+                record[@tag_property_name] = tag
+            end
+
+            record.to_msgpack
+        end
+
+        def write(chunk)
+            records = []
+            chunk.msgpack_each {|record|
+                if ObjectSpace.memsize_of(record) < @max_size
+                    records << { :message_body => Yajl.dump(record), :delay_seconds => @delay_seconds }
+                else
+                    log.info "Could not send log to SQS: the size of log exceeded max_size"
+                end
+            }
+            until records.length <= 0 do
+                @queue.batch_send(records.slice!(0..9))
+            end
+        end
+    end
+end

data/spec/lib/fluent/plugin/in_sqs_spec.rb

@@ -0,0 +1,106 @@
+require 'spec_helper'
+
+describe do
+  let(:driver) {
+    AWS.stub!
+    Fluent::Test.setup
+    Fluent::Test::InputTestDriver.new(Fluent::SQSInput).configure(config)
+  }
+  let(:instance) {driver.instance}
+
+  describe 'config' do
+    let(:config) {
+      %[
+         aws_key_id AWS_KEY_ID
+         aws_sec_key AWS_SEC_KEY
+         tag     TAG
+         sqs_url SQS_URL
+         max_number_of_messages 10
+      ]
+    }
+    
+    context do
+      subject {instance.aws_key_id}
+      it{should == 'AWS_KEY_ID'}
+    end
+
+    context do
+      subject {instance.aws_sec_key}
+      it{should == 'AWS_SEC_KEY'}
+    end
+
+    context do
+      subject {instance.tag}
+      it{should == 'TAG'}
+    end
+
+    context do
+      subject {instance.sqs_url}
+      it{should == 'SQS_URL'}
+    end
+
+    context do
+      subject {instance.receive_interval}
+      it{should == 1}
+    end
+
+    context do
+      subject {instance.max_number_of_messages}
+      it{should == 10}
+    end
+  end
+  
+  describe 'emit' do
+    let(:message) do
+      { 'body' => 'body',
+        'handle' => 'handle',
+        'id' => 'id',
+        'md5' => 'md5',
+        'url' => 'url',
+        'sender_id' => 'sender_id'
+      }
+    end
+    let(:emmits) {
+      allow(Time).to receive(:now).and_return(0)
+
+      class AWS::SQS::Queue
+        def receive_message(opts)
+          yield OpenStruct.new(
+            { 'body' => 'body',
+              'handle' => 'handle',
+              'id' => 'id',
+              'md5' => 'md5',
+              'queue' => OpenStruct.new(:url => 'url'),
+              'sender_id' => 'sender_id',
+              'sent_at' => 0
+            })
+        end
+      end
+      expect_any_instance_of(AWS::SQS::Queue).to receive(:receive_message).with({:limit => 10}).at_least(:once).and_call_original
+
+      d = driver
+      d.run do
+        sleep 2
+      end
+
+      d.emits
+    }
+
+    context do
+      let(:config) {
+        %[
+           aws_key_id AWS_KEY_ID
+           aws_sec_key AWS_SEC_KEY
+           tag     TAG
+           sqs_url SQS_URL
+           max_number_of_messages 10
+        ]
+      }
+
+      subject {emmits.first}
+      it{should ==  ['TAG', 0, message]}
+    end
+
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,25 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+
+  require 'fluent/load'
+  require 'fluent/test'
+
+  require 'fluent/plugin/in_sqs'
+  require 'ostruct'
+  require 'pry'
+  require 'rr'
+end

metadata

@@ -0,0 +1,193 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-sqs-check-size
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Nao Akechi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-07-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.9.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.9.5
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+description: 
+email: nao.akechi@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- README.rdoc
+files:
+- AUTHORS
+- Rakefile
+- VERSION
+- lib/fluent/plugin/in_sqs.rb
+- lib/fluent/plugin/out_sqs.rb
+- spec/lib/fluent/plugin/in_sqs_spec.rb
+- spec/spec_helper.rb
+- README.rdoc
+homepage: https://github.com/nakc/fluent-plugin-sqs
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 3
+summary: Amazon SQS input/output plugin for Fluent event collector
+test_files:
+- spec/lib/fluent/plugin/in_sqs_spec.rb
+- spec/spec_helper.rb