fluent-plugin-aws-sqs 1.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 2e0e807c91cfb9c16b261fbf2302ddd2537856f9eacaca6b5a17bea6c451f904
+  data.tar.gz: 702ca145ce473611c73ea372a810e8c35bdbdc319c95d80bc1ae1600bf71c4e9
+SHA512:
+  metadata.gz: b4f6aa63ee94baac86ac116378e9f795d3bc431234f92e5e989aa197e75dcea8d7ce58dc069016b9c028237fb5f57b93bc552df9e2eb8fab79c5f6cab13fbe18
+  data.tar.gz: 8b51ce8aa459d03962c2fef4ffffd7f31a7788d44a992bb033e99bf0f374c621d479b047504dfe85392c944676cbf51dcda80e6459dbc11905ac0388cdc5d6a2

data/.gitignore

@@ -0,0 +1,7 @@
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+.history/
+.idea/*

data/.gitlab-ci.yml

@@ -0,0 +1,73 @@
+image: ruby:2.6.3
+
+# Cache gems in between builds
+cache:
+  key: ${CI_COMMIT_REF_SLUG}
+  paths:
+    - vendor/ruby
+
+before_script:
+  - gem install bundler:2.0.2
+  - bundle install -j $(nproc) --path vendor  # Install dependencies into ./vendor/ruby
+
+stages:
+  - test
+  - release
+
+variables:
+  DOCKER_TLS_CERTDIR: ""
+
+unit-test:
+  tags:
+    - dcos-multi-runner
+  stage: test
+  script:
+    - bundle exec rake test
+  except:
+    - tags
+
+.release: &release
+  tags:
+    - dcos-multi-runner
+  stage: release
+  before_script:
+    - echo "Setup ssh inside the runner.."
+    - git config --global user.email $GITLAB_USER_EMAIL
+    - git config --global user.name $GITLAB_USER_LOGIN
+    - 'which ssh-agent || ( apk --update add openssh-client )'
+    - eval $(ssh-agent -s)
+    - ssh-add <(echo "$SSH_PRIVATE_KEY")
+    - mkdir -p ~/.ssh
+    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
+    - echo "Clone the source code.."
+    - GIT_URL=$(echo $CI_PROJECT_URL | cut -d'/' -f3)
+    - git clone git@$GIT_URL:$CI_PROJECT_PATH.git
+    - cd $CI_PROJECT_NAME
+    - git reset --hard $CI_COMMIT_SHA
+  script:
+    - mkdir -p ~/.gem
+    - 'echo -e "---\n:rubygems_api_key: $GEM_HOST_API_KEY" > ~/.gem/credentials && echo "created gem credentials"'
+    - chmod 0600 ~/.gem/credentials
+    - gem install gem-release
+    - gem bump fluent-plugin-aws-sqs --version $TYPE --file ./lib/fluent/plugin/version.rb --commit --push --tag
+    - gem build fluent-plugin-aws-sqs.gemspec
+    - export VERSION=$(cat ./lib/fluent/plugin/version.rb | grep VERSION | sed -ne 's/[^0-9]*\(\([0-9]\.\)\{0,4\}[0-9][^.]\).*/\1/p' | sed 's/"$//')
+    - gem push fluent-plugin-aws-sqs-$VERSION.gem
+  when: manual
+  only:
+    - master
+
+release:patch:
+  extends: .release
+  variables:
+    TYPE: patch
+
+release:minor:
+  extends: .release
+  variables:
+    TYPE: minor
+
+release:major:
+  extends: .release
+  variables:
+    TYPE: major

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent_parser_s.gemspec
+gemspec

data/Rakefile

@@ -0,0 +1,7 @@
+# in Rakefile
+require 'rake/testtask'
+  Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end

data/VERSION

@@ -0,0 +1 @@
+1.0.0

data/fluent-plugin-aws-sqs.gemspec

@@ -0,0 +1,28 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require ('./lib/fluent/plugin/version.rb')
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-aws-sqs"
+  spec.version       = SQS::VERSION
+
+  spec.authors       = ["Shai Moria", "Niv Lipetz", "Jacob Goldenberg"]
+  spec.email         = ["shai.moria@zooz.com", "niv.lipetz@zooz.com", "jacob.goldenberg@zooz.com"]
+  spec.description   = "Fluentd SQS plugin to read data from AWS SQS"
+  spec.summary       = "Fluentd SQS plugin to read data from AWS SQS"
+  spec.homepage      = "https://github.com/zooz/fluent-plugin-aws-sqs"
+
+  spec.files         = `git ls-files`.split($\)
+  spec.require_paths = ["lib"]
+  spec.license = "Apache-2.0"
+
+  spec.required_ruby_version = '>= 2.1'
+
+  spec.add_runtime_dependency "fluentd", ">= 0.14.0"
+  spec.add_runtime_dependency "aws-sdk-sqs", ">= 1.22.0"
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake", "~> 12.0"
+  spec.add_development_dependency "test-unit", ">= 3.1.0"
+  spec.add_development_dependency "test-unit-rr"
+end

data/lib/fluent/plugin/in_sqs.rb

@@ -0,0 +1,80 @@
+require 'fluent/plugin/input'
+require 'aws-sdk-sqs'
+
+module Fluent::Plugin
+  class SQSInput < Input
+    Fluent::Plugin.register_input('sqs', self)
+
+    helpers :timer
+
+    config_param :aws_key_id, :string, default: nil, secret: true
+    config_param :aws_sec_key, :string, default: nil, secret: true
+    config_param :tag, :string
+    config_param :region, :string, default: 'ap-northeast-1'
+    config_param :sqs_url, :string, default: nil
+    config_param :receive_interval, :time, default: 0.1
+    config_param :max_number_of_messages, :integer, default: 10
+    config_param :wait_time_seconds, :integer, default: 10
+    config_param :visibility_timeout, :integer, default: nil
+    config_param :delete_message, :bool, default: false
+    config_param :stub_responses, :bool, default: false
+
+    def configure(conf)
+      super
+
+      Aws.config = {
+        access_key_id: @aws_key_id,
+        secret_access_key: @aws_sec_key,
+        region: @region
+      }
+    end
+
+    def start
+      super
+
+      timer_execute(:in_sqs_run_periodic_timer, @receive_interval, &method(:run))
+    end
+
+    def client
+      @client ||= Aws::SQS::Client.new(stub_responses: @stub_responses)
+    end
+
+    def queue
+      @queue ||= Aws::SQS::Resource.new(client: client).queue(@sqs_url)
+    end
+
+    def shutdown
+      super
+    end
+
+    def run
+      queue.receive_messages(
+        max_number_of_messages: @max_number_of_messages,
+        wait_time_seconds: @wait_time_seconds,
+        visibility_timeout: @visibility_timeout
+      ).each do |message|
+        record = parse_message(message)
+
+        message.delete if @delete_message
+
+        router.emit(@tag, Fluent::Engine.now, record)
+      end
+    rescue
+      log.error 'failed to emit or receive', error: $ERROR_INFO.to_s, error_class: $ERROR_INFO.class.to_s
+      log.warn_backtrace $ERROR_INFO.backtrace
+    end
+
+    private
+
+    def parse_message(message)
+      {
+        'body' => message.body.to_s,
+        'receipt_handle' => message.receipt_handle.to_s,
+        'message_id' => message.message_id.to_s,
+        'md5_of_body' => message.md5_of_body.to_s,
+        'queue_url' => message.queue_url.to_s,
+        'sender_id' => message.attributes['SenderId'].to_s
+      }
+    end
+  end
+end

data/lib/fluent/plugin/version.rb

@@ -0,0 +1,3 @@
+module SQS
+    VERSION = "1.0.2"
+end 

data/spec/lib/fluent/plugin/in_sqs_spec.rb

@@ -0,0 +1,120 @@
+require 'spec_helper'
+require 'fluent/plugin/in_sqs'
+
+describe Fluent::Plugin::SQSInput do
+  let(:driver) do
+    Fluent::Test.setup
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::SQSInput).configure(config)
+  end
+  subject { driver.instance }
+
+  describe '#configure' do
+    let(:config) do
+      %(
+         aws_key_id AWS_KEY_ID
+         aws_sec_key AWS_SEC_KEY
+         tag TAG
+         region REGION
+         sqs_url http://SQS_URL
+         receive_interval 1
+         max_number_of_messages 10
+         wait_time_seconds 10
+         visibility_timeout 1
+         delete_message true
+         stub_responses true
+      )
+    end
+
+    context 'fluentd input configuration settings' do
+      it { expect(subject.aws_key_id).to eq('AWS_KEY_ID') }
+      it { expect(subject.aws_sec_key).to eq('AWS_SEC_KEY') }
+      it { expect(subject.tag).to eq('TAG') }
+      it { expect(subject.region).to eq('REGION') }
+      it { expect(subject.sqs_url).to eq('http://SQS_URL') }
+      it { expect(subject.receive_interval).to eq(1) }
+      it { expect(subject.max_number_of_messages).to eq(10) }
+      it { expect(subject.wait_time_seconds).to eq(10) }
+      it { expect(subject.visibility_timeout).to eq(1) }
+      it { expect(subject.delete_message).to eq(true) }
+      it { expect(subject.stub_responses).to eq(true) }
+    end
+
+    context 'AWS configuration settings' do
+      subject { Aws.config }
+
+      before { driver.instance }
+
+      it { expect(subject[:access_key_id]).to eq('AWS_KEY_ID') }
+      it { expect(subject[:secret_access_key]).to eq('AWS_SEC_KEY') }
+      it { expect(subject[:region]).to eq('REGION') }
+    end
+  end
+
+  describe '#run' do
+    let(:message_attributes) do
+      {
+        body: 'body',
+        receipt_handle: 'receipt_handle',
+        message_id: 'message_id',
+        md5_of_body: 'md5_of_body',
+        queue_url: 'queue_url',
+        attributes: { 'SenderId' => 'sender_id' }
+      }
+    end
+    let(:queue) { double(:queue, receive_messages: true) }
+    let(:message) { double(:message, **message_attributes.merge(delete: nil)) }
+    let(:messages) { [message] }
+
+    context 'with no delete messages param' do
+      let(:config) do
+        %(
+         tag TAG
+         max_number_of_messages 10
+         wait_time_seconds 10
+         visibility_timeout 1
+         delete_message false
+      )
+      end
+
+      before do
+        allow(subject).to receive(:queue) { queue }
+      end
+
+      it 'parse through messages and emit it' do
+        expect(queue).to receive(:receive_messages)
+          .with(max_number_of_messages: 10, wait_time_seconds: 10, visibility_timeout: 1) { messages }
+        expect(subject).to receive(:parse_message).with(message) { message_attributes }
+        expect(message).not_to receive(:delete)
+        expect(subject.router).to receive(:emit).with('TAG', kind_of(Fluent::EventTime), message_attributes)
+
+        subject.run
+      end
+    end
+
+    context 'with no delete messages param' do
+      let(:config) do
+        %(
+         tag TAG
+         max_number_of_messages 10
+         wait_time_seconds 10
+         visibility_timeout 1
+         delete_message true
+      )
+      end
+
+      before do
+        allow(subject).to receive(:queue) { queue }
+      end
+
+      it 'parse through messages and emit it' do
+        expect(queue).to receive(:receive_messages)
+          .with(max_number_of_messages: 10, wait_time_seconds: 10, visibility_timeout: 1) { messages }
+        expect(subject).to receive(:parse_message).with(message) { message_attributes }
+        expect(message).to receive(:delete)
+        expect(subject.router).to receive(:emit).with('TAG', kind_of(Fluent::EventTime), message_attributes)
+
+        subject.run
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,28 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+
+  require 'fluent/load'
+  require 'fluent/test'
+  require 'fluent/test/helpers'
+  require 'fluent/test/driver/output'
+  require 'fluent/test/driver/input'
+
+  require 'ostruct'
+  require 'pry'
+  require 'rr'
+  # prevent Test::Unit's AutoRunner from executing during RSpec's rake task
+  Test::Unit.run = true if defined?(Test::Unit) && Test::Unit.respond_to?(:run=)
+end

metadata

@@ -0,0 +1,142 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-aws-sqs
+version: !ruby/object:Gem::Version
+  version: 1.0.2
+platform: ruby
+authors:
+- Shai Moria
+- Niv Lipetz
+- Jacob Goldenberg
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-09-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-sqs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.22.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.22.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd SQS plugin to read data from AWS SQS
+email:
+- shai.moria@zooz.com
+- niv.lipetz@zooz.com
+- jacob.goldenberg@zooz.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".gitlab-ci.yml"
+- Gemfile
+- README.md
+- Rakefile
+- VERSION
+- fluent-plugin-aws-sqs.gemspec
+- lib/fluent/plugin/in_sqs.rb
+- lib/fluent/plugin/version.rb
+- spec/lib/fluent/plugin/in_sqs_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/zooz/fluent-plugin-aws-sqs
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Fluentd SQS plugin to read data from AWS SQS
+test_files: []