fluent-plugin-sqs 0.2.3 → 1.1.0

data/AUTHORS

@@ -1 +1 @@
-Yudai Odagiri <ixixizko _at_ gmail.com>
+Yuri Odagiri <ixixizko _at_ gmail.com>

data/README.rdoc

@@ -9,6 +9,10 @@
 
 Store fluent-event as queue message to amazon SQS.
 
+=== SQSInput
+
+Read events from from amazon SQS.
+
 == Configuratin
 
 === SQSOutput
@@ -40,13 +44,38 @@ Store fluent-event as queue message to amazon SQS.
 
     </match>
 
-== TODO
+=== SQSInput
+
+    <source>
+
+      type sqs
+
+      # following attibutes are required
+
+      aws_key_id {your_aws_key_id}
+      aws_sec_key {your_aws_secret_key}
+      sqs_url {queue_url}
+
+      # following attibutes are optional
+    　
+      sqs_endpoint {endpointURL}
+
+      ### endpoint list ###
+      # Asia Pacific (Tokyo) [Default] : sqs.ap-northeast-1.amazonaws.com
+      # Asia Pacific (Singapore)       : sqs.ap-southeast-1.amazonaws.com
+      # US-East (Virginia)             : sqs.us-east-1.amazonaws.com
+      # US-West (Oregon)               : sqs.us-west-2.amazonaws.com
+      # US-West (N.California)         : sqs.us-west-1.amazonaws.com
+      # EU-West (Ireland)              : sqs.eu-west-1.amazonaws.com
+      # South America (São Paulo)      : sns.sa-east-1.amazonaws.com
+
+      receive_interval {receive_interval_seconds}
 
-=== input plugin
+    </source>
 
 == Tool
 
 == Copyright
 
-Copyright:: Copyright (c) 2011- Yudai Odagiri
+Copyright:: Copyright (c) 2011- Yuri Odagiri
 License::   Apache License, Version 2.0

data/Rakefile

@@ -7,15 +7,15 @@ begin
   Jeweler::Tasks.new do |gemspec|
     gemspec.name = "fluent-plugin-sqs"
     gemspec.summary = "Amazon SQS output plugin for Fluent event collector"
-    gemspec.author = "Yudai Odagiri"
+    gemspec.author = "Yuri Odagiri"
     gemspec.email = "ixixizko@gmail.com"
     gemspec.homepage = "https://github.com/ixixi/fluent-plugin-sqs"
     gemspec.has_rdoc = false
     gemspec.require_paths = ["lib"]
     gemspec.add_dependency "fluentd", "~> 0.10.0"
     gemspec.add_dependency "aws-sdk", "~> 1.3.2"
-    gemspec.test_files = Dir["test/**/*.rb"]
-    gemspec.files = Dir["lib/**/*", "test/**/*.rb"] + %w[VERSION AUTHORS Rakefile]
+    gemspec.test_files = Dir["test/**/*.rb", "spec/**/*.rb"]
+    gemspec.files = Dir["lib/**/*", "test/**/*.rb", "spec/**/*.rb"] + %w[VERSION AUTHORS Rakefile]
     gemspec.executables = []
   end
   Jeweler::GemcutterTasks.new

data/VERSION

@@ -1 +1 @@
-0.2.3
+1.1.0

data/lib/fluent/plugin/in_sqs.rb

@@ -0,0 +1,67 @@
+module Fluent
+
+  require 'aws-sdk'
+
+  class SQSInput < Input
+    Plugin.register_input('sqs', self)
+
+    def initialize
+      super
+    end
+
+    config_param :aws_key_id, :string
+    config_param :aws_sec_key, :string
+    config_param :tag, :string
+    config_param :sqs_endpoint, :string, :default => 'sqs.ap-northeast-1.amazonaws.com'
+    config_param :sqs_url, :string
+    config_param :receive_interval, :time, :default => 1
+
+    def configure(conf)
+      super
+
+    end
+
+    def start
+      super
+
+      AWS.config(
+        :access_key_id => @aws_key_id,
+        :secret_access_key => @aws_sec_key
+        )
+
+      @queue = AWS::SQS.new(:sqs_endpoint => @sqs_endpoint).queues[@sqs_url]
+
+      @finished = false
+      @thread = Thread.new(&method(:run_periodic))
+    end
+
+    def shutdown
+      super
+
+      @finished = true
+      @thread.join
+    end
+
+    def run_periodic
+      until @finished
+        begin
+          sleep @receive_interval
+          @queue.receive_message do |message|
+            record = {}
+            record[:body] = message.body.to_s
+            record[:handle] = message.handle.to_s
+            record[:id] = message.id.to_s
+            record[:md5] = message.md5.to_s
+            record[:url] = message.queue.url.to_s
+            record[:sender_id] = message.sender_id.to_s
+
+            Engine.emit(@tag, Time.now, record)
+          end
+        rescue
+          $log.error "failed to emit or receive", :error => $!.to_s, :error_class => $!.class.to_s
+          $log.warn_backtrace $!.backtrace
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_sqs.rb

@@ -28,10 +28,10 @@ module Fluent
             
             AWS.config(
                 :access_key_id => @aws_key_id,
-                :secret_access_key => @aws_sec_key,
-                :sqs_endpoint => @sqs_endpoint )
+                :secret_access_key => @aws_sec_key)
 
-            @sqs = AWS::SQS.new
+            @sqs = AWS::SQS.new(
+                :sqs_endpoint => @sqs_endpoint)
             @queue = @sqs.queues.create(@queue_name)
             
         end

data/spec/lib/fluent/plugin/in_sqs_spec.rb

@@ -0,0 +1,96 @@
+require 'spec_helper'
+
+describe do
+  let(:driver) {
+    AWS.stub!
+    Fluent::Test::InputTestDriver.new(Fluent::SQSInput).configure(config)
+  }
+  let(:instance) {driver.instance}
+
+  describe 'config' do
+    let(:config) {
+      %[
+         aws_key_id AWS_KEY_ID
+         aws_sec_key AWS_SEC_KEY
+         tag     TAG
+         sqs_url SQS_URL
+      ]
+    }
+    
+    context do
+      subject {instance.aws_key_id}
+      it{should == 'AWS_KEY_ID'}
+    end
+
+    context do
+      subject {instance.aws_sec_key}
+      it{should == 'AWS_SEC_KEY'}
+    end
+
+    context do
+      subject {instance.tag}
+      it{should == 'TAG'}
+    end
+
+    context do
+      subject {instance.sqs_url}
+      it{should == 'SQS_URL'}
+    end
+
+    context do
+      subject {instance.receive_interval}
+      it{should == 1}
+    end
+  end
+  
+  describe 'emit' do
+    let(:message) do
+      { :body => 'body',
+        :handle => 'handle',
+        :id => 'id',
+        :md5 => 'md5',
+        :url => 'url',
+        :sender_id => 'sender_id'
+      }
+    end
+    let(:emmits) {
+      stub(Time).now {0}
+
+      class AWS::SQS::Queue
+        def receive_message
+          yield OpenStruct.new(
+            { :body => 'body',
+              :handle => 'handle',
+              :id => 'id',
+              :md5 => 'md5',
+              :queue => OpenStruct.new(:url => 'url'),
+              :sender_id => 'sender_id',
+              :sent_at => 0
+            })
+        end
+      end
+      d = driver
+      d.run do
+        sleep 2
+      end
+      
+      d.emits
+    }
+
+    context do
+      let(:config) {
+        %[
+           aws_key_id AWS_KEY_ID
+           aws_sec_key AWS_SEC_KEY
+           tag     TAG
+           sqs_url SQS_URL
+        ]
+      }
+
+      subject {emmits.first}
+      it{should ==  ['TAG', 0, message]}
+    end
+
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,25 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+
+  require 'fluent/load'
+  require 'fluent/test'
+
+  require 'fluent/plugin/in_sqs'
+  require 'ostruct'
+  require 'pry'
+  require 'rr'
+end

metadata

@@ -1,79 +1,199 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: fluent-plugin-sqs
-version: !ruby/object:Gem::Version 
+version: !ruby/object:Gem::Version
+  version: 1.1.0
   prerelease: 
-  version: 0.2.3
 platform: ruby
-authors: 
-- Yudai Odagiri
+authors:
+- Yuri Odagiri
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2013-06-15 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: fluentd
+date: 2014-01-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
   prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-sqs
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rr
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
+      - !ruby/object:Gem::Version
         version: 0.10.0
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: aws-sdk
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
         version: 1.3.2
   type: :runtime
-  version_requirements: *id002
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.3.2
 description: 
 email: ixixizko@gmail.com
 executables: []
-
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
 - README.rdoc
-files: 
+files:
 - AUTHORS
 - Rakefile
 - VERSION
+- lib/fluent/plugin/in_sqs.rb
 - lib/fluent/plugin/out_sqs.rb
+- spec/lib/fluent/plugin/in_sqs_spec.rb
+- spec/spec_helper.rb
 - README.rdoc
 homepage: https://github.com/ixixi/fluent-plugin-sqs
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
 rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: Amazon SQS output plugin for Fluent event collector
-test_files: []
-
+test_files:
+- spec/lib/fluent/plugin/in_sqs_spec.rb
+- spec/spec_helper.rb