logstash-output-sns 1.0.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5104315bfaa6592b7a01af7f60e91f2fbf01033b
-  data.tar.gz: b26ecbe7cf8da6ec5f7c67cd2c281c259bd6d4fc
+  metadata.gz: 6b4ac99bfe2f2202031197d1244322ea5922043b
+  data.tar.gz: 3708b40a61096f6f3f26a0e89126172812ad7d93
 SHA512:
-  metadata.gz: 141bd61ec417b7d05663248698f90d08a6a96d39c5557d26ed0ac33e0a89ad140fcede305c80654f1b0b9c2b3bcf0db375ac9105b664d3593a15b84a41a5ccd9
-  data.tar.gz: 2a249724c9182a327ed908dd435e252ca6c4ae6cea1088aba51826b848e5a0807c6946937aca2912dcbad2c3b72e7afe0f3fa77d0aeed1cd8ff4a35bbeb959a1
+  metadata.gz: 3c5f526f7fd22744a1eed1e441b9a1c3ad505d5fbad019155bac79224c80c8c153216462bb96259fcc76aca4a5d0c9beb2a6819bfdd9c5f6e39c4624c6fdffa2
+  data.tar.gz: 4c27f467d148a9575a79f0390e0958e2948890a1e4b9f98c54bd0ec8d8f33b63599014600516c2cb926287ce413b8904d2c34f8211c13dc6aacae292f3b1b862

data/CHANGELOG.md

@@ -0,0 +1,6 @@
+# 1.0.0
+  * Full refactor. 
+  * This plugin now uses codecs for all formatting. The 'format' option has now been removed. Please use a codec.
+# 0.1.5
+  * If no `subject` are specified fallback to the %{host} key (https://github.com/logstash-plugins/logstash-output-sns/pull/2)
+  * Migrate the SNS Api to use the AWS-SDK v2

data/Gemfile

@@ -1,2 +1,2 @@
 source 'https://rubygems.org'
-gemspec
+gemspec

data/lib/logstash/outputs/sns.rb

@@ -6,7 +6,7 @@ require "logstash/plugin_mixins/aws_config"
 # SNS output.
 #
 # Send events to Amazon's Simple Notification Service, a hosted pub/sub
-# framework.  It supports subscribers of type email, HTTP/S, SMS, and SQS.
+# framework.  It supports various subscription types, including email, HTTP/S, SMS, and SQS.
 #
 # For further documentation about the service see:
 #
@@ -17,25 +17,23 @@ require "logstash/plugin_mixins/aws_config"
 #  * `sns` - If no ARN is found in the configuration file, this will be used as
 #  the ARN to publish.
 #  * `sns_subject` - The subject line that should be used.
-#  Optional. The "%{host}" will be used if not present and truncated at
-#  `MAX_SUBJECT_SIZE_IN_CHARACTERS`.
-#  * `sns_message` - The message that should be
-#  sent. Optional. The event serialzed as JSON will be used if not present and
+#  Optional. The "%{host}" will be used if `sns_subject` is not present. The subject
+#  will be truncated to 100 characters. If `sns_subject` is set to a non-string value a JSON version of that value will be saved.
+#  * `sns_message` - Optional string of message to be sent. If this is set to a non-string value it will be encoded with the specified `codec`. If this is not set the entire event will be encoded with the codec.
 #  with the @message truncated so that the length of the JSON fits in
-#  `MAX_MESSAGE_SIZE_IN_BYTES`.
+#  `32768` bytes.
 #
 class LogStash::Outputs::Sns < LogStash::Outputs::Base
-  include LogStash::PluginMixins::AwsConfig
+  include LogStash::PluginMixins::AwsConfig::V2
 
   MAX_SUBJECT_SIZE_IN_CHARACTERS  = 100
   MAX_MESSAGE_SIZE_IN_BYTES       = 32768
+  NO_SUBJECT = "NO SUBJECT"
 
   config_name "sns"
 
-  # Message format.  Defaults to plain text.
-  config :format, :validate => [ "json", "plain" ], :default => "plain"
-
-  # SNS topic ARN.
+  # Optional ARN to send messages to. If you do not set this you must
+  # include the `sns` field in your events to set the ARN on a per-message basis!
   config :arn, :validate => :string
 
   # When an ARN for an SNS topic is specified here, the message
@@ -46,23 +44,16 @@ class LogStash::Outputs::Sns < LogStash::Outputs::Base
   #
   config :publish_boot_message_arn, :validate => :string
 
-  public
-  def aws_service_endpoint(region)
-    return {
-        :sns_endpoint => "sns.#{region}.amazonaws.com"
-    }
-  end
-
   public
   def register
-    require "aws-sdk"
+    require "aws-sdk-resources"
 
-    @sns = AWS::SNS.new(aws_options_hash)
+    @sns = Aws::SNS::Client.new(aws_options_hash)
 
-    # Try to publish a "Logstash booted" message to the ARN provided to
-    # cause an error ASAP if the credentials are bad.
-    if @publish_boot_message_arn
-      @sns.topics[@publish_boot_message_arn].publish("Logstash successfully booted", :subject => "Logstash booted")
+    publish_boot_message_arn()
+
+    @codec.on_event do |event, encoded|
+      send_sns_message(event_arn(event), event_subject(event), encoded)
     end
   end
 
@@ -70,54 +61,58 @@ class LogStash::Outputs::Sns < LogStash::Outputs::Base
   def receive(event)
     return unless output?(event)
 
-    arn     = Array(event["sns"]).first || @arn
-
-    raise "An SNS ARN required." unless arn
-
-    message = Array(event["sns_message"]).first
-    subject = Array(event["sns_subject"]).first || event["host"]
-
-    # Ensure message doesn't exceed the maximum size.
-    if message
-      # TODO: Utilize `byteslice` in JRuby 1.7: http://jira.codehaus.org/browse/JRUBY-5547
-      message = message.slice(0, MAX_MESSAGE_SIZE_IN_BYTES)
-    else
-      if @format == "plain"
-        message = self.class.format_message(event)
+    if (sns_msg = event["sns_message"])
+      if sns_msg.is_a?(String)
+        send_sns_message(event_arn(event), event_subject(event), sns_msg)
       else
-        message = self.class.json_message(event)
+        @codec.encode(sns_msg)
       end
+    else
+      @codec.encode(event)
     end
+  end
 
-    # Log event.
-    @logger.debug("Sending event to SNS topic [#{arn}] with subject [#{subject}] and message:")
-    message.split("\n").each { |line| @logger.debug(line) }
-
-    # Publish the message.
-    @sns.topics[arn].publish(message, :subject => subject.slice(0, MAX_SUBJECT_SIZE_IN_CHARACTERS))
+  private
+  def publish_boot_message_arn
+    # Try to publish a "Logstash booted" message to the ARN provided to
+    # cause an error ASAP if the credentials are bad.
+    if @publish_boot_message_arn
+      send_sns_message(@publish_boot_message_arn, 'Logstash booted', 'Logstash successfully booted')
+    end
   end
 
-  def self.json_message(event)
-    json      = event.to_json
-    json_size = json.bytesize
+  private
+  def send_sns_message(arn, subject, message)
+    raise ArgumentError, 'An SNS ARN is required.' unless arn
 
-    # Truncate only the message if the JSON structure is too large.
-    if json_size > MAX_MESSAGE_SIZE_IN_BYTES
-      # TODO: Utilize `byteslice` in JRuby 1.7: http://jira.codehaus.org/browse/JRUBY-5547
-      event["message"] = event["message"].slice(0, (event["message"].bytesize - (json_size - MAX_MESSAGE_SIZE_IN_BYTES)))
-    end
+    trunc_subj = subject.slice(0, MAX_SUBJECT_SIZE_IN_CHARACTERS)
+    trunc_msg = message.slice(0, MAX_MESSAGE_SIZE_IN_BYTES)
+
+    @logger.debug? && @logger.debug("Sending event to SNS topic [#{arn}] with subject [#{trunc_subj}] and message: #{trunc_msg}")
 
-    event.to_json
+    @sns.publish({
+                   :topic_arn => arn,
+                   :subject => trunc_subj,
+                   :message => trunc_msg
+                 })
   end
 
-  def self.format_message(event)
-    message =  "Date: #{event.timestamp}\n"
-    message << "Source: #{event["source"]}\n"
-    message << "Tags: #{(event["tags"] || []).join(', ')}\n"
-    message << "Fields: #{event.to_hash.inspect}\n"
-    message << "Message: #{event["message"]}"
+  private
+  def event_subject(event)
+    sns_subject = event["sns_subject"]
+    if sns_subject.is_a?(String)
+      sns_subject
+    elsif sns_subject
+      LogStash::Json.dump(sns_subject)
+    elsif event["host"]
+      event["host"]
+    else
+      NO_SUBJECT
+    end
+  end
 
-    # TODO: Utilize `byteslice` in JRuby 1.7: http://jira.codehaus.org/browse/JRUBY-5547
-    message.slice(0, MAX_MESSAGE_SIZE_IN_BYTES)
+  private
+  def event_arn(event)
+    event["sns"] || @arn
   end
 end

data/logstash-output-sns.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-sns'
-  s.version         = '1.0.0'
+  s.version         = '2.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Send events to Amazon's Simple Notification Service a hosted pub/sub framework"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -21,9 +21,7 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
-  s.add_runtime_dependency 'logstash-mixin-aws'
-  s.add_runtime_dependency 'aws-sdk'
+  s.add_runtime_dependency 'logstash-mixin-aws', '>= 1.0.0'
 
   s.add_development_dependency 'logstash-devutils'
 end
-

data/spec/outputs/sns_spec.rb

@@ -2,17 +2,141 @@
 require "logstash/devutils/rspec/spec_helper"
 require 'logstash/outputs/sns'
 require 'logstash/event'
+require "logstash/plugin_mixins/aws_config"
+
+require "aws-sdk" # TODO: Why is this not automatically brought in by the aws_config plugin?
 
 describe LogStash::Outputs::Sns do
-  describe '#format_message' do
-    it 'should allow to output to sns with empty tags' do
-      event = LogStash::Event.new({ "message" => "42 is the answer" })
-      expect(LogStash::Outputs::Sns.format_message(event)).to match(/Tags:\s\n/m)
+  let(:arn) { "arn:aws:sns:us-east-1:999999999:logstash-test-sns-topic" }
+  let(:sns_subject) { "The Plain in Spain" }
+  let(:sns_message) { "That's where the rain falls, plainly." }
+  let(:mock_client) { double("Aws::SNS::Client") }
+  let(:instance) {
+    allow(Aws::SNS::Client).to receive(:new).and_return(mock_client)
+    inst = LogStash::Outputs::Sns.new
+    allow(inst).to receive(:publish_boot_message_arn).and_return(nil)
+    inst.register
+    inst
+  }
+
+  describe "receiving an event" do
+    let(:expected_subject) { double("expected_subject")}
+    subject {
+      inst = instance
+      allow(inst).to receive(:send_sns_message).with(any_args)
+      allow(inst).to receive(:event_subject).
+                       with(any_args).
+                       and_return(expected_subject)
+      inst.receive(event)
+      inst
+    }
+
+    shared_examples("publishing correctly") do
+      it "should send a message to the correct ARN if the event has 'arn' set" do
+        expect(subject).to have_received(:send_sns_message).with(arn, anything, anything)
+      end
+
+      it "should send the message" do
+        expect(subject).to have_received(:send_sns_message).with(anything, anything, expected_message)
+      end
+
+      it "should send the subject" do
+        expect(subject).to have_received(:send_sns_message).with(anything, expected_subject, anything)
+      end
+    end
+
+    describe "with an explicit message" do
+      let(:expected_subject) { sns_subject }
+      let(:expected_message) { sns_message }
+      let(:event) { LogStash::Event.new("sns" => arn, "sns_subject" => sns_subject,
+                                        "sns_message" => sns_message) }
+      include_examples("publishing correctly")
+    end
+
+    describe "without an explicit message" do
+      # Testing codecs sucks. It'd be nice if codecs had to implement some sort of encode_sync method
+      let(:expected_message) {
+        c = subject.codec.clone
+        result = nil;
+        c.on_event {|event, encoded| result = encoded }
+        c.encode(event)
+        result
+      }
+      let(:event) { LogStash::Event.new("sns" => arn, "sns_subject" => sns_subject) }
+
+      include_examples("publishing correctly")
+    end
+  end
+
+  describe "determining the subject" do
+    it "should return 'sns_subject' when set" do
+      event = LogStash::Event.new("sns_subject" => "foo")
+      expect(subject.send(:event_subject, event)).to eql("foo")
+    end
+
+    it "should return the sns subject as JSON if not a string" do
+      event = LogStash::Event.new("sns_subject" => ["foo", "bar"])
+      expect(subject.send(:event_subject, event)).to eql(LogStash::Json.dump(["foo", "bar"]))
     end
 
-    it 'should allow to output to sns with a list of tags' do
-      event = LogStash::Event.new({"tags" => ["elasticsearch", "logstash", "kibana"] })
-      expect(LogStash::Outputs::Sns.format_message(event)).to match(/\nTags:\selasticsearch,\slogstash,\skibana\n/m)
+    it "should return the host if 'sns_subject' not set" do
+      event = LogStash::Event.new("host" => "foo")
+      expect(subject.send(:event_subject, event)).to eql("foo")
+    end
+
+    it "should return 'NO SUBJECT' when subject cannot be determined" do
+      event = LogStash::Event.new("foo" => "bar")
+      expect(subject.send(:event_subject, event)).to eql(LogStash::Outputs::Sns::NO_SUBJECT)
+    end
+  end
+
+  describe "sending an SNS notification" do
+    let(:good_publish_args) {
+      {
+        :topic_arn => arn,
+        :subject => sns_subject,
+        :message => sns_message
+      }
+    }
+    let(:long_message) { "A" * (LogStash::Outputs::Sns::MAX_MESSAGE_SIZE_IN_BYTES + 1) }
+    let(:long_subject) { "S" * (LogStash::Outputs::Sns::MAX_SUBJECT_SIZE_IN_CHARACTERS + 1) }
+    subject { instance }
+
+    it "should raise an ArgumentError if no arn is provided" do
+      expect {
+        subject.send(:send_sns_message, nil, sns_subject, sns_message)
+      }.to raise_error(ArgumentError)
+    end
+
+    it "should send a well formed message through to SNS" do
+      expect(mock_client).to receive(:publish).with(good_publish_args)
+      subject.send(:send_sns_message, arn, sns_subject, sns_message)
+    end
+
+    it "should attempt to publish a boot message" do
+      expect(subject).to have_received(:publish_boot_message_arn).once
+      x = case "foo"
+            when "bar"
+              "hello"
+          end
+    end
+
+    it "should truncate long messages before sending" do
+      max_size = LogStash::Outputs::Sns::MAX_MESSAGE_SIZE_IN_BYTES
+      expect(mock_client).to receive(:publish) {|args|
+                               expect(args[:message].bytesize).to eql(max_size)
+                             }
+
+      subject.send(:send_sns_message, arn, sns_subject, long_message)
+    end
+
+    it "should truncate long subjects before sending" do
+      max_size = LogStash::Outputs::Sns::MAX_SUBJECT_SIZE_IN_CHARACTERS
+      expect(mock_client).to receive(:publish) {|args|
+                               expect(args[:subject].bytesize).to eql(max_size)
+                             }
+
+      subject.send(:send_sns_message, arn, long_subject, sns_message)
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-sns
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-06-24 00:00:00.000000000 Z
+date: 2015-06-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -36,26 +36,12 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.0
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
-  prerelease: false
-  type: :runtime
-- !ruby/object:Gem::Dependency
-  name: aws-sdk
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.0
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
@@ -111,7 +97,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.2.2
+rubygems_version: 2.1.9
 signing_key:
 specification_version: 4
 summary: Send events to Amazon's Simple Notification Service a hosted pub/sub framework