fluent-plugin-cloudwatch-transform 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fd04f4ba0fff4e3fe7723aae95ac042e1e6d1947
-  data.tar.gz: d7147b9d1f1de9c84b86ffe2fb596027e476ab59
+  metadata.gz: a025e5995d9b256900600da9feecbf60bbd67682
+  data.tar.gz: f193e656ccffe26f843e26d642151157c758b99a
 SHA512:
-  metadata.gz: 12fe4bc88da9697f3a395cdf1ed5bc98f84fe3e09b356e7d9249bb6134c2babec85c83e958122b8bcb8f5dcc2f97c50ca6f70dbc73bbd235d750eb1aa397e8f8
-  data.tar.gz: bb90d407337f5d4d861930b3c030d0ae873d4ae1469440fa21ea1defe89426853dd43faa1f33696e5748b2c12d2cffe6827147e86651ed8efdca7111f622bc51
+  metadata.gz: c283f6a9b5296a131ea618b42c6b88afe2ee168ea157d033141f57c9db1a2f3da95a56c1b3d188d43892b6d214aac7037efa58f1e215cc6497988af78b3d46dc
+  data.tar.gz: 5758b0ac68903a90700a9b19921df25346b843f46130e2fb39c7e2b6b9ea33fe3a5ad67b74728986e17f5ac933ace0bf063c2b9953014d6da7138f2c7669c3b7

data/README.md

@@ -2,6 +2,11 @@
 
 fluent-plugin-cloudwatch-transform is an output plug-in for [Fluentd](http://fluentd.org)
 
+It can transform the alerts from fluent-plugin-cloudwatch to key-value pairs as "event_name" and "value", 
+also add more information from the tag you added in fluent-plugin-cloudwatch.
+
+It also used "highwatermark" gem to store status timestamp to state file or redis cache
+
 ## Installation
 
 These instructions assume you already have fluentd installed. 
@@ -31,18 +36,21 @@ Or install it yourself as:
 
     $ gem build fluent-plugin-cloudwatch-transform.gemspec
 
-    $ sudo gem install ./fluent-plugin-cloudwatch-transform-0.0.3.gem
+    $ sudo gem install ./fluent-plugin-cloudwatch-transform-0.0.5.gem
     
 
 ## Usage
 
 ### fluent configure
 require fluent-plugin-cloudwatch for input.
+require highwatermark to store timestamp to file or redis
 Add the following into your fluentd config.
 
     <source>
       type cloudwatch
-      tag  #tag with application name, like: alert.cloudwatch.raw.Form & Printing Services (FPS)
+      tag  #tag with prefix alert.cloudwatch.raw
+      # followed with region and availability zone, application name, wiki url in sequence  
+      # for example: alert.cloudwatch.raw.region1-AZ1.Form & Printing Services (FPS)."http://runbook.wiki.com"
       aws_key_id   #your id 
       aws_sec_key  #your key
       cw_endpoint  #your endpoint
@@ -56,12 +64,26 @@ Add the following into your fluentd config.
     <match alert.cloudwatch.raw.**>
      type cloudwatch_transform
      tag  alert.cloudwatch.out
+     state_type file #could be <redis/file/memory>
+     state_file /path/to/state/file/or/redis/conf  # the file path for store state or the redis configure
+
     </match>
 
     <match alert.cloudwatch.out> 
      type stdout
     </match>
 
+
+Example of redis.conf (if set state_type = 'redis'):
+
+```
+# redis configure
+host: 127.0.0.1
+port: 6379
+
+```
+
+
 ### input example
     {
        "HealthyHostCount": 6
@@ -78,7 +100,9 @@ Add the following into your fluentd config.
         },
         "receive_time_input": "1426189324",
         "application_name": "Form & Printing Services (FPS)",
-        "intermediary_source": "cloudwatch",
+        "intermediary_source": "region1-AZ1",
+        "runbook": "http://runbook.wiki.com"
+        "event_type" : "alert.cloudwatch"
         "tag": "alert.cloudwatch.out"
     }
 

data/fluent-plugin-cloudwatch-transform.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-cloudwatch-transform"
-  gem.version       = "0.0.4"
+  gem.version       = "0.0.5"
   gem.date          = '2015-03-26'
   gem.authors       = ["Ling Zhang"]
   gem.email         = ["zhangling.ice@gmail.com"]
@@ -22,4 +22,5 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency 'bundler', '~> 1.3'
   gem.add_development_dependency 'rake', '~> 0.9', '>= 0.9.2'
   gem.add_development_dependency 'rspec', '~> 2.11', '>= 2.11.0'
+  gem.add_runtime_dependency "highwatermark", '~> 0.1'
 end

data/lib/fluent/plugin/out_cloudwatch_transform.rb

@@ -6,15 +6,24 @@ module Fluent
     
     #config_param :tag, :string, default:'alert.cloudwatch.out' 
     config_param :tag, :string
+    config_param  :state_type,    :string,  :default => nil
+    config_param  :state_file,    :string,  :default => nil
 
     # This method is called before starting.
     def configure(conf)
       super
     end
 
+    def initialize
+      require 'highwatermark'
+      super
+    end # def initialize
+
     # This method is called when starting.
     def start
       super
+      @highwatermark = Highwatermark::HighWaterMark.new(@state_file, @state_type)
+
     end
 
     # This method is called when shutting down.
@@ -55,6 +64,15 @@ module Fluent
         newhash["runbook"] =  runbook_url
         newhash["event_type"] = "alert.cloudwatch"
 
+        if @highwatermark.last_records(@tag)
+          last_hwm = @highwatermark.last_records(@tag)
+          $log.info "got hwm form state file: #{last_hwm.to_i}"
+        else
+          $log.info "no hwm yet"
+        end
+
+        @highwatermark.update_records(timestamp.to_s,@tag)
+
         #log the transformed message and emit it
         $log.info "Tranformed message  #{newhash}"
     		Fluent::Engine.emit @tag, time.to_i, newhash

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudwatch-transform
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Ling Zhang
@@ -84,6 +84,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: 2.11.0
+- !ruby/object:Gem::Dependency
+  name: highwatermark
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.1'
 description: FLuentd plugin for transform cloudwatch alerts
 email:
 - zhangling.ice@gmail.com