fluent-plugin-slack 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7b835ec9f32cc25b75bd4e0a18d7aa52097ff61e
-  data.tar.gz: 21a5be237380dc845f1ead2dad7e547500d88c19
+  metadata.gz: c08f44114d6d2335cb8cee39f0053e79000270bc
+  data.tar.gz: 25186d53c993522e99e92acfb479859d0e6e265d
 SHA512:
-  metadata.gz: 7e36906812d93f8887a0994209c7efecf432b71e85e75cbc3b0fa9e7660c2dea8f04fc0eba411a35a11671d28c9d55ed646e153e408e71f5b9ffe79934b42578
-  data.tar.gz: 7090cf28cbf42af855170144a1c9b446837704f3f84d5c6340a9e9b1d3d86b9dce7f50a76c90d3773da22dbfa9cc68491404b86f94f35c5aebb2afc3738e79c2
+  metadata.gz: 9e454cae0012869146257e6c1160b739a4620f27966eeb192991b677d7243de00d24516146de16ca38be2971bd403f7d31197d1fc9f24f54d162ab64c968932c
+  data.tar.gz: 1a277e5b7ad062af8e25618b7742c680a3c9878ef36f28f4a7ad47a2a782861c909930251dcf0a699890885a95eb331e31fc27158ff5f06ddf003f514bdc91fa

data/.gitignore

@@ -3,3 +3,6 @@
 /coverage/
 /vendor/
 Gemfile.lock
+tmp/
+.ruby-version
+.env

data/README.md

@@ -0,0 +1,87 @@
+# Fluent event to slack plugin.
+
+# Installation
+
+```
+$ fluent-gem install fluent-plugin-slack
+```
+
+# Usage (Incoming Webhook)
+
+```apache
+<match slack>
+  type slack
+  webhook_url https://hooks.slack.com/services/XXX/XXX/XXX
+  channel general
+  username sowasowa
+  color good
+  icon_emoji :ghost:
+  flush_interval 60s
+</match>
+```
+
+```ruby
+fluent_logger.post('slack', {
+  :message  => 'Hello<br>World!'
+})
+```
+
+# Usage (Slack API)
+
+```apache
+<match slack>
+  type slack
+  token xoxb-XXXXXXXXXX-XXXXXXXXXXXXXXXXXXXXXXXX
+  channel general
+  username sowasowa
+  color good
+  icon_emoji :ghost:
+  flush_interval 60s
+</match>
+```
+
+```ruby
+fluent_logger.post('slack', {
+  :message  => 'Hello<br>World!'
+})
+```
+
+### Parameter
+
+|parameter|description|default|
+|---|---|---|
+|webhook_uri|Incoming Webhook URI (Required for Incoming Webhook mode)||
+|token|Token for Slack API (Required for Slack API mode)||
+|username|name of bot|fluentd|
+|color|color to use|good|
+|icon_emoji|emoji to use as the icon|`:question:`|
+|channel|channel to send messages (without first '#')||
+|channel_keys|keys used to format channel. %s will be replaced with value specified by channel_keys if this option is used|nil|
+|title|title format. %s will be replaced with value specified by title_keys. title is created from the first appeared record on each tag|nil|
+|title_keys|keys used to format the title|nil|
+|message|message format. %s will be replaced with value specified by message_keys|%s|
+|message_keys|keys used to format messages|message|
+
+`fluent-plugin-slack` uses `SetTimeKeyMixin` and `SetTagKeyMixin`, so you can also use:
+
+|parameter|description|default|
+|---|---|---|
+|timezone|timezone such as `Asia/Tokyo`||
+|localtime|use localtime as timezone|true|
+|utc|use utc as timezone||
+|time_key|key name for time used in xxx_keys|time|
+|time_format|time format. This will be formatted with Time#strftime.|%H:%M:%S|
+|tag_key|key name for tag used in xxx_keys|tag|
+
+`fluent-plugin-slack` is a kind of BufferedOutput plugin, so you can also use [Buffer Parameters](http://docs.fluentd.org/articles/out_exec#buffer-parameters).
+
+# Contributors
+
+- [@sonots](https://github.com/sonots)
+- [@kenjiskywalker](https://github.com/kenjiskywalker)
+
+# Copyright
+
+* Copyright:: Copyright (c) 2014- Keisuke SOGAWA
+* License::   Apache License, Version 2.0
+

data/VERSION

@@ -1 +1 @@
-0.4.0
+0.5.0

data/fluent-plugin-slack.gemspec

@@ -16,11 +16,12 @@ Gem::Specification.new do |gem|
   gem.require_paths = ['lib']
 
   gem.add_dependency "fluentd", ">= 0.10.8"
-  gem.add_dependency "activesupport", "~>3.2.0"
-  gem.add_dependency "tzinfo", ">=0.3.38"
 
   gem.add_development_dependency "rake", ">= 10.1.1"
   gem.add_development_dependency "rr", ">= 1.0.0"
   gem.add_development_dependency "pry"
-  gem.add_development_dependency("minitest", ["~> 4.0"])
+  gem.add_development_dependency "pry-nav"
+  gem.add_development_dependency "test-unit", "~> 3.0.2"
+  gem.add_development_dependency "test-unit-rr", "~> 1.0.3"
+  gem.add_development_dependency "dotenv"
 end

data/lib/fluent/plugin/out_buffered_slack.rb

@@ -1,118 +1,199 @@
+require_relative 'slack_client'
+
 module Fluent
-  class BufferedSlackOutputError < StandardError; end
-  class BufferedSlackOutput < Fluent::TimeSlicedOutput
-    Fluent::Plugin.register_output('buffered_slack', self)
-    config_param :api_key,    :string, default: nil
-    config_param :token,      :string, default: nil
-    config_param :team,       :string, default: nil
-    config_param :channel,    :string
-    config_param :username,   :string
-    config_param :color,      :string
-    config_param :icon_emoji, :string
-    config_param :timezone,   :string, default: nil
-    config_param :rtm,        :bool  , default: false
-    config_param :webhook_url,:string, default: nil
-
-    attr_reader :slack
+  class SlackOutput < Fluent::BufferedOutput
+    Fluent::Plugin.register_output('buffered_slack', self) # old version compatiblity
+    Fluent::Plugin.register_output('slack', self)
 
-    def format(tag, time, record)
-      [tag, time, record].to_msgpack
-    end
+    include SetTimeKeyMixin
+    include SetTagKeyMixin
 
-    def write(chunk)
-      messages = {}
-      chunk.msgpack_each do |tag, time, record|
-        messages[tag] = '' if messages[tag].nil?
-        messages[tag] << "[#{Time.at(time).in_time_zone(@timezone).strftime("%H:%M:%S")}] #{record['message']}\n"
-      end
-      begin
+    config_set_default :include_time_key, true
+    config_set_default :include_tag_key, true
+   
+    config_param :webhook_url,   :string, default: nil # incoming webhook
+    config_param :token,         :string, default: nil # api token
+    config_param :username,      :string, default: 'fluentd'
+    config_param :color,         :string, default: 'good'
+    config_param :icon_emoji,    :string, default: ':question:'
 
-        # https://api.slack.com/rtm
-        if @rtm
-          params = {
-            :token       => @token,
-            :channel     => @channel,
-            :text        => messages.values,
-            :username    => @username,
-            :icon_emoji  => @icon_emoji,
-            :attachments => [{
-              :color  => @color,
-              :text   => messages.values
-            }].to_json
-          }
-          get_request(params)
-        else
-          payload = {
-            channel:      @channel,
-              username:   @username,
-              icon_emoji: @icon_emoji,
-              attachments: [{
-                fallback: messages.keys.join(','),
-                color:    @color,
-                fields:   messages.map{|k,v| {title: k, value: v} }
-              }]}
-          post_request(
-            payload: payload.to_json
-          )
-        end
-      rescue => e
-        $log.error("Slack Error: #{e.backtrace[0]} / #{e.message}")
-      end
+    config_param :channel,       :string
+    config_param :channel_keys,  default: nil do |val|
+      val.split(',')
+    end
+    config_param :title,         :string, default: nil
+    config_param :title_keys,    default: nil do |val|
+      val.split(',')
+    end
+    config_param :message,       :string, default: nil
+    config_param :message_keys,  default: nil do |val|
+      val.split(',')
     end
 
+    # for test
+    attr_reader :slack, :time_format, :localtime, :timef
+
     def initialize
       super
-      require 'active_support/time'
       require 'uri'
-      require 'net/http'
     end
 
     def configure(conf)
+      conf['time_format'] ||= '%H:%M:%S' # old version compatiblity
+      conf['localtime'] ||= true unless conf['utc']
+ 
       super
 
-      @channel    = URI.unescape(conf['channel'])
-      @username   = conf['username']   || 'fluentd'
-      @color      = conf['color']      || 'good'
-      @icon_emoji = conf['icon_emoji'] || ':question:'
+      @channel = URI.unescape(@channel) # old version compatibility
+      @channel = '#' + @channel unless @channel.start_with?('#')
 
-      if @rtm
-        @token      = conf['token']
+      if @webhook_url
+        # following default values are for old version compatibility
+        @title         ||= '%s'
+        @title_keys    ||= %w[tag]
+        @message       ||= '[%s] %s'
+        @message_keys  ||= %w[time message]
+        @slack = Fluent::SlackClient::IncomingWebhook.new(@webhook_url)
       else
-        @timezone   = conf['timezone'] || 'UTC'
-        @team       = conf['team']
-        @api_key    = conf['api_key']
-        @webhook_url= conf['webhook_url']
+        unless @token
+          raise Fluent::ConfigError.new("`token` is required to call slack api")
+        end
+        @message      ||= '%s'
+        @message_keys ||= %w[message]
+        @slack = Fluent::SlackClient::WebApi.new
+      end
+      @slack.log = log
+      @slack.debug_dev = log.out if log.level <= Fluent::Log::LEVEL_TRACE
+
+      begin
+        @message % (['1'] * @message_keys.length)
+      rescue ArgumentError
+        raise Fluent::ConfigError, "string specifier '%s' for `message`  and `message_keys` specification mismatch"
+      end
+      if @title and @title_keys
+        begin
+          @title % (['1'] * @title_keys.length)
+        rescue ArgumentError
+          raise Fluent::ConfigError, "string specifier '%s' for `title` and `title_keys` specification mismatch"
+        end
+      end
+      if @channel_keys
+        begin
+          @channel % (['1'] * @channel_keys.length)
+        rescue ArgumentError
+          raise Fluent::ConfigError, "string specifier '%s' for `channel` and `channel_keys` specification mismatch"
+        end
+      end
+    end
+
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    def write(chunk)
+      begin
+        payloads = build_payloads(chunk)
+        payloads.each {|payload| @slack.post_message(payload) }
+      rescue Net::OpenTimeout, Net::ReadTimeout => e
+        log.warn "out_slack:", :error => e.to_s, :error_class => e.class.to_s
+        raise e # let Fluentd retry
+      rescue => e
+        log.error "out_slack:", :error => e.to_s, :error_class => e.class.to_s
+        log.warn_backtrace e.backtrace
+        # discard. @todo: add more retriable errors
       end
     end
 
     private
-    def response_check(res)
-      if res.code != "200"
-        raise BufferedSlackOutputError, "Slack.com - #{res.code} - #{res.body}"
+
+    def build_payloads(chunk)
+      if @title
+        build_title_payloads(chunk)
+      else
+        build_plain_payloads(chunk)
       end
     end
 
-    def get_request(params)
-      query = URI.encode_www_form(params)
-      uri = URI.parse("https://slack.com/api/chat.postMessage?#{query}")
-      http = Net::HTTP.new(uri.host, uri.port)
-      http.use_ssl = true
-      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-      res = http.get(uri.request_uri)
-      response_check(res)
+    def common_payload
+      return @common_payload if @common_payload
+      @common_payload = {
+        username:   @username,
+        icon_emoji: @icon_emoji,
+      }
+      @common_payload[:token] = @token if @token
+      @common_payload
     end
 
-    def endpoint
-      URI.parse @webhook_url || "https://#{@team}.slack.com/services/hooks/incoming-webhook?token=#{@api_key}"
+    Field = Struct.new("Field", :title, :value)
+
+    def build_title_payloads(chunk)
+      ch_fields = {}
+      chunk.msgpack_each do |tag, time, record|
+        channel = build_channel(record)
+        per     = tag # title per tag
+        ch_fields[channel]      ||= {}
+        ch_fields[channel][per] ||= Field.new(build_title(record), '')
+        ch_fields[channel][per].value << "#{build_message(record)}\n"
+      end
+      ch_fields.map do |channel, fields|
+        {
+          channel: channel,
+          attachments: [{
+            :color    => @color,
+            :fallback => fields.values.map(&:title).join(' '), # fallback is the message shown on popup
+            :fields   => fields.values.map(&:to_h)
+          }],
+        }.merge(common_payload)
+      end
     end
 
-    def post_request(data)
-      req = Net::HTTP::Post.new endpoint.request_uri
-      req.set_form_data(data)
-      http = Net::HTTP.new endpoint.host, endpoint.port
-      http.use_ssl = (endpoint.scheme == "https")
-      res = http.request(req)
-      response_check(res)
+    def build_plain_payloads(chunk)
+      messages = {}
+      chunk.msgpack_each do |tag, time, record|
+        channel = build_channel(record)
+        messages[channel] ||= ''
+        messages[channel] << "#{build_message(record)}\n"
+      end
+      messages.map do |channel, text|
+        {
+          channel: channel,
+          attachments: [{
+            :color    => @color,
+            :fallback => text,
+            :text     => text,
+          }],
+        }.merge(common_payload)
+      end
+    end
+
+    def build_message(record)
+      values = fetch_keys(record, @message_keys)
+      @message % values
+    end
+
+    def build_title(record)
+      return @title unless @title_keys
+
+      values = fetch_keys(record, @title_keys)
+      @title % values
+    end
+
+    def build_channel(record)
+      return @channel unless @channel_keys
+
+      values = fetch_keys(record, @channel_keys)
+      @channel % values
+    end
+
+    def fetch_keys(record, keys)
+      Array(keys).map do |key|
+        begin
+          record.fetch(key).to_s
+        rescue KeyError
+          log.warn "out_slack: the specified key '#{key}' not found in record. [#{record}]"
+          ''
+        end
+      end
     end
   end
 end

data/lib/fluent/plugin/out_slack.rb

@@ -0,0 +1,199 @@
+require_relative 'slack_client'
+
+module Fluent
+  class SlackOutput < Fluent::BufferedOutput
+    Fluent::Plugin.register_output('buffered_slack', self) # old version compatiblity
+    Fluent::Plugin.register_output('slack', self)
+
+    include SetTimeKeyMixin
+    include SetTagKeyMixin
+
+    config_set_default :include_time_key, true
+    config_set_default :include_tag_key, true
+   
+    config_param :webhook_url,   :string, default: nil # incoming webhook
+    config_param :token,         :string, default: nil # api token
+    config_param :username,      :string, default: 'fluentd'
+    config_param :color,         :string, default: 'good'
+    config_param :icon_emoji,    :string, default: ':question:'
+
+    config_param :channel,       :string
+    config_param :channel_keys,  default: nil do |val|
+      val.split(',')
+    end
+    config_param :title,         :string, default: nil
+    config_param :title_keys,    default: nil do |val|
+      val.split(',')
+    end
+    config_param :message,       :string, default: nil
+    config_param :message_keys,  default: nil do |val|
+      val.split(',')
+    end
+
+    # for test
+    attr_reader :slack, :time_format, :localtime, :timef
+
+    def initialize
+      super
+      require 'uri'
+    end
+
+    def configure(conf)
+      conf['time_format'] ||= '%H:%M:%S' # old version compatiblity
+      conf['localtime'] ||= true unless conf['utc']
+ 
+      super
+
+      @channel = URI.unescape(@channel) # old version compatibility
+      @channel = '#' + @channel unless @channel.start_with?('#')
+
+      if @webhook_url
+        # following default values are for old version compatibility
+        @title         ||= '%s'
+        @title_keys    ||= %w[tag]
+        @message       ||= '[%s] %s'
+        @message_keys  ||= %w[time message]
+        @slack = Fluent::SlackClient::IncomingWebhook.new(@webhook_url)
+      else
+        unless @token
+          raise Fluent::ConfigError.new("`token` is required to call slack api")
+        end
+        @message      ||= '%s'
+        @message_keys ||= %w[message]
+        @slack = Fluent::SlackClient::WebApi.new
+      end
+      @slack.log = log
+      @slack.debug_dev = log.out if log.level <= Fluent::Log::LEVEL_TRACE
+
+      begin
+        @message % (['1'] * @message_keys.length)
+      rescue ArgumentError
+        raise Fluent::ConfigError, "string specifier '%s' for `message`  and `message_keys` specification mismatch"
+      end
+      if @title and @title_keys
+        begin
+          @title % (['1'] * @title_keys.length)
+        rescue ArgumentError
+          raise Fluent::ConfigError, "string specifier '%s' for `title` and `title_keys` specification mismatch"
+        end
+      end
+      if @channel_keys
+        begin
+          @channel % (['1'] * @channel_keys.length)
+        rescue ArgumentError
+          raise Fluent::ConfigError, "string specifier '%s' for `channel` and `channel_keys` specification mismatch"
+        end
+      end
+    end
+
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    def write(chunk)
+      begin
+        payloads = build_payloads(chunk)
+        payloads.each {|payload| @slack.post_message(payload) }
+      rescue Net::OpenTimeout, Net::ReadTimeout => e
+        log.warn "out_slack:", :error => e.to_s, :error_class => e.class.to_s
+        raise e # let Fluentd retry
+      rescue => e
+        log.error "out_slack:", :error => e.to_s, :error_class => e.class.to_s
+        log.warn_backtrace e.backtrace
+        # discard. @todo: add more retriable errors
+      end
+    end
+
+    private
+
+    def build_payloads(chunk)
+      if @title
+        build_title_payloads(chunk)
+      else
+        build_plain_payloads(chunk)
+      end
+    end
+
+    def common_payload
+      return @common_payload if @common_payload
+      @common_payload = {
+        username:   @username,
+        icon_emoji: @icon_emoji,
+      }
+      @common_payload[:token] = @token if @token
+      @common_payload
+    end
+
+    Field = Struct.new("Field", :title, :value)
+
+    def build_title_payloads(chunk)
+      ch_fields = {}
+      chunk.msgpack_each do |tag, time, record|
+        channel = build_channel(record)
+        per     = tag # title per tag
+        ch_fields[channel]      ||= {}
+        ch_fields[channel][per] ||= Field.new(build_title(record), '')
+        ch_fields[channel][per].value << "#{build_message(record)}\n"
+      end
+      ch_fields.map do |channel, fields|
+        {
+          channel: channel,
+          attachments: [{
+            :color    => @color,
+            :fallback => fields.values.map(&:title).join(' '), # fallback is the message shown on popup
+            :fields   => fields.values.map(&:to_h)
+          }],
+        }.merge(common_payload)
+      end
+    end
+
+    def build_plain_payloads(chunk)
+      messages = {}
+      chunk.msgpack_each do |tag, time, record|
+        channel = build_channel(record)
+        messages[channel] ||= ''
+        messages[channel] << "#{build_message(record)}\n"
+      end
+      messages.map do |channel, text|
+        {
+          channel: channel,
+          attachments: [{
+            :color    => @color,
+            :fallback => text,
+            :text     => text,
+          }],
+        }.merge(common_payload)
+      end
+    end
+
+    def build_message(record)
+      values = fetch_keys(record, @message_keys)
+      @message % values
+    end
+
+    def build_title(record)
+      return @title unless @title_keys
+
+      values = fetch_keys(record, @title_keys)
+      @title % values
+    end
+
+    def build_channel(record)
+      return @channel unless @channel_keys
+
+      values = fetch_keys(record, @channel_keys)
+      @channel % values
+    end
+
+    def fetch_keys(record, keys)
+      Array(keys).map do |key|
+        begin
+          record.fetch(key).to_s
+        rescue KeyError
+          log.warn "out_slack: the specified key '#{key}' not found in record. [#{record}]"
+          ''
+        end
+      end
+    end
+  end
+end