fluent-plugin-sqs-poll 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 546e6db771fc22d674a53d9c72b1fa173fe11e4b
+  data.tar.gz: 45328baf2a51121d0a43a5b0f7103baeafe40de6
+SHA512:
+  metadata.gz: 925a21d42c16a361ac28117996837d013722857ccb83358e256ae789093caf3106947c375cdbc6a681693a6fedace0e23f4cda9d0197473a3b4eec4ded4d1c03
+  data.tar.gz: 93807b7a6a40b3b3d7491fb246487a91b345480f33927fc0c6b8decf1efaf14365eed15ef5d643aba8519f802a814cff08b37019b0dd161ced5135280c82aa84

data/.gitignore

@@ -0,0 +1,35 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Richard Li
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

data/README.md

@@ -0,0 +1,47 @@
+# fluent-plugin-sqs-poll
+fluent input plugin poll data from AWS SQS
+
+## Install
+
+```bash
+gem install fluent-plugin-sqs-poll
+```
+
+## Configuration
+
+If either **aws_access_key** or **aws_secret_key** is missing, it will automatically
+fall back to use IAM role for access control.
+
+```
+<source>
+  type sqs_poll
+  aws_access_key {optional: your_aws_access_key}
+  aws_secret_key {optional: your_aws_secret_key}
+  tag {required: tag}
+  sqs_url {required: SQS_url}
+  max_number_of_messages {optional: # of messges poll at a time, default: 1}
+</source>
+```
+
+## Usage
+
+When messages are pulled off SQS queue, it's stored in key **body** of the
+emmitted output. If your SQS message is in a known format, you probably want to
+use in conjunction with **fluent-plugin-parser**. For example, if your message
+from SQS is in JSON format, then you would want to do:
+
+```
+<source>
+  type sqs_poll
+  tag sqs.message
+  sqs_url https://sqs.us-east-1.amazonaws.com/123456789/some_queue
+  max_number_of_messages 10
+</source>
+
+<match sqs.message>
+  type parser
+  remove_prefix sqs
+  format json
+  key_name body
+</match>
+```

data/fluent-plugin-sqs-poll.gemspec

@@ -0,0 +1,21 @@
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-sqs-poll"
+  spec.version       = "0.1.0"
+  spec.authors       = ["Richard Li"]
+  spec.email         = ["evilcat@wisewolfsolutions.com"]
+  spec.description   = %q{fluent input plugin use aws-sdk v2 sqs poller to receive messages}
+  spec.summary       = %q{fluent sqs poll input}
+  spec.homepage      = "https://github.com/ecwws/fluent-plugin-sqs-poll"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "aws-sdk", '~> 2'
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec", ">= 3.0.0"
+end

data/lib/fluent/plugin/in_sqs_poll.rb

@@ -0,0 +1,63 @@
+module Fluent
+  require 'aws-sdk'
+
+  class SQSPollInput < Input
+    Fluent::Plugin.register_input('sqs_poll', self)
+
+    config_param :aws_access_key, :string, :default => nil, :secret => true
+    config_param :aws_secret_key, :string, :default => nil, :secret => true
+    config_param :tag, :string
+    config_param :sqs_url, :string
+    config_param :max_number_of_messages, :integer, default: 1
+
+    def configure(conf)
+      super
+    end
+
+    def start
+      super
+
+      @terminate = false
+      @thread = Thread.new(&method(:poll))
+    end
+
+    def shutdown
+      super
+
+      @terminate = true
+      @thread.join
+    end
+
+    def poll
+      region = @sqs_url.split('.')[1]
+      Aws.config.update(region: region)
+
+      if @aws_access_key && @aws_secret_key
+        Aws.config.update(
+          credentials: Aws::Credentials.new(@aws_access_key, @aws_secret_key)
+        )
+      end
+
+      poller = Aws::SQS::QueuePoller.new(@sqs_url)
+
+      poller.poll(max_number_of_messages: @max_number_of_messages) do |messages|
+        throw :stop_polling if @terminate
+        messages.each do |msg|
+          begin
+            Engine.emit(@tag, Time.now.to_i,
+              {
+                'body' => msg.body,
+                'handle' => msg.receipt_handle,
+                'id' => msg.message_id,
+                'md5' => msg.md5_of_body,
+              }
+            )
+          rescue Exception => e
+            $log.error("SQS exception", error: e.to_s, error_class: e.class.to_s)
+            $log.warn_backtrace(e.backtrace)
+          end
+        end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,106 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-sqs-poll
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Richard Li
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-09-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+description: fluent input plugin use aws-sdk v2 sqs poller to receive messages
+email:
+- evilcat@wisewolfsolutions.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- LICENSE
+- README.md
+- fluent-plugin-sqs-poll.gemspec
+- lib/fluent/plugin/in_sqs_poll.rb
+homepage: https://github.com/ecwws/fluent-plugin-sqs-poll
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: fluent sqs poll input
+test_files: []
+has_rdoc: