fluent-plugin-azureoms 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 460994a7573dbeb51cd3a9180b252bcb91df6d7f
-  data.tar.gz: 1053d0974d92d5b07bd9c4a9a5c369bd2d5f1315
+  metadata.gz: 680cf704074eb4a3f28c32fcfcac08ff893542c7
+  data.tar.gz: eccc175af820a495c5246d9ad786153a8b5ffad5
 SHA512:
-  metadata.gz: 16539bf9a26f62ce16e9b4ce2b809dfe2da9ec65637a83c245d6099d985d3aa4055244fe70942b37754074d970105f57b41db7adca5bdecff13521a893f95367
-  data.tar.gz: '0635551900d0d5fbb003be834ebb78f3f0d8a1b151917c92cf0f40a18afaa4a29aa2658232ed2096aa5c3092ccc88e927737cd14dfcb255fd04c77906385c243'
+  metadata.gz: 795a8755ed33d412f8f1d59b88ea0ba4c2b955fe193c7980907665037576c23f184619410ee96ed50f8ef84af5994c26eeffc5e6e96061b7fa4c53c49333aff0
+  data.tar.gz: d680350bceff3c4d4060dee39660e3ff65458a805cc9b501999622d519a1ae9b7fd3e8038afcf02982a45fa0ab15079e22fa1a8a8d15d2eedc999d0946b905d3

data/.gitignore

@@ -0,0 +1 @@
+pkg/

data/Gemfile.lock

@@ -1,53 +1,51 @@
-PATH
-  remote: .
-  specs:
-    fluent-plugin-azureoms (0.1.0)
-      fluentd (>= 0.14.10, < 2)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    cool.io (1.5.3)
-    cool.io (1.5.3-x64-mingw32)
-    dig_rb (1.0.1)
-    fluentd (1.1.2)
-      cool.io (>= 1.4.5, < 2.0.0)
-      dig_rb (~> 1.0.0)
-      http_parser.rb (>= 0.5.1, < 0.7.0)
-      msgpack (>= 0.7.0, < 2.0.0)
-      serverengine (>= 2.0.4, < 3.0.0)
-      sigdump (~> 0.2.2)
-      strptime (>= 0.2.2, < 1.0.0)
-      tzinfo (~> 1.0)
-      tzinfo-data (~> 1.0)
-      yajl-ruby (~> 1.0)
-    http_parser.rb (0.6.0)
-    msgpack (1.2.4)
-    msgpack (1.2.4-x64-mingw32)
-    power_assert (1.1.1)
-    rake (12.3.1)
-    serverengine (2.0.6)
-      sigdump (~> 0.2.2)
-    sigdump (0.2.4)
-    strptime (0.2.3)
-    test-unit (3.2.7)
-      power_assert
-    thread_safe (0.3.6)
-    tzinfo (1.2.5)
-      thread_safe (~> 0.1)
-    tzinfo-data (1.2018.4)
-      tzinfo (>= 1.0.0)
-    yajl-ruby (1.3.1)
-
-PLATFORMS
-  ruby
-  x64-mingw32
-
-DEPENDENCIES
-  bundler (~> 1.14)
-  fluent-plugin-azureoms!
-  rake (~> 12.0)
-  test-unit (~> 3.0)
-
-BUNDLED WITH
-   1.16.1
+PATH
+  remote: .
+  specs:
+    fluent-plugin-azureoms (0.2.0)
+      fluentd (>= 0.14.10, < 2)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    cool.io (1.5.3)
+    dig_rb (1.0.1)
+    fluentd (1.1.2)
+      cool.io (>= 1.4.5, < 2.0.0)
+      dig_rb (~> 1.0.0)
+      http_parser.rb (>= 0.5.1, < 0.7.0)
+      msgpack (>= 0.7.0, < 2.0.0)
+      serverengine (>= 2.0.4, < 3.0.0)
+      sigdump (~> 0.2.2)
+      strptime (>= 0.2.2, < 1.0.0)
+      tzinfo (~> 1.0)
+      tzinfo-data (~> 1.0)
+      yajl-ruby (~> 1.0)
+    http_parser.rb (0.6.0)
+    msgpack (1.2.4)
+    power_assert (1.1.1)
+    rake (12.3.1)
+    serverengine (2.0.6)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.4)
+    strptime (0.2.3)
+    test-unit (3.2.7)
+      power_assert
+    thread_safe (0.3.6)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+    tzinfo-data (1.2018.4)
+      tzinfo (>= 1.0.0)
+    yajl-ruby (1.3.1)
+
+PLATFORMS
+  ruby
+  x64-mingw32
+
+DEPENDENCIES
+  bundler (~> 1.14)
+  fluent-plugin-azureoms!
+  rake (~> 12.0)
+  test-unit (~> 3.0)
+
+BUNDLED WITH
+   1.16.1

data/fluent-plugin-azureoms.gemspec

@@ -3,9 +3,10 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-azureoms"
-  spec.version = "0.1.0"
+  spec.version = "0.2.0"
   spec.authors = ["Mark Simms"]
   spec.email   = ["masimms@microsoft.com"]
+  #spec.git     = "git://www.github.com/mabsimms/fluent-plugin-azureoms.git"
 
   spec.summary       = "Fluentd output plugin for Azure Log Analytics"
   spec.description   = "Fluentd output plugin for Azure Log Analytics"

data/lib/fluent/plugin/out_azureoms.rb

@@ -1,144 +1,157 @@
-#
-# Copyright 2018- TODO: Write your name
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-require "fluent/plugin/output"
-require "time"
-require "base64"
-require "openssl"
-require "uri"
-require "net/https"
-require "json"
-
-module Fluent
-  module Plugin
-    class AzureomsOutput < Fluent::Plugin::Output
-      
-      helpers :event_emitter, :compat_parameters, :record_accessor
-
-      Fluent::Plugin.register_output("azureoms", self)
-
-      config_param :workspace, :string
-      config_param :key, :string, secret: true
-      config_param :timestamp_field, :string, default: "timestamp"
-      config_param :log_name, :string, default: "AzureLog"
-
-      def configure(conf)
-        # This also calls config_param (don't access configuration parameters before
-        # calling super)
-        super        
-      end
-
-      # This output plugin uses the raw HTTP data collector
-      # API per https://docs.microsoft.com/en-us/azure/log-analytics/log-analytics-data-collector-api
-
-      def prefer_buffered_processing 
-        true
-      end
-
-      def process(tag, es)
-        print "Writing single record set (synchronous)\n"
-        es.each do |time, record|           
-          # Convert record into a JSON body payload for OMS
-          record[:timestamp] = Time.at(time).iso8601          
-
-          # Publish event
-          send_data(workspace, key, record.to_json, log_name)        
-        end 
-      end 
-
-      # Synchronous buffered output
-      def write(chunk) 
-        log.debug "Writing buffered record set (synchronous)"
-        log.debug "writing data to file", chunk_id: dump_unique_id_hex(chunk.unique_id)
-        
-        elements = Array.new
-        
-        chunk.each do |time, record|
-          log.debug "Writing record #{record.inspect}"    
-
-          # Fold the timestamp into the record
-          record[:timestamp] = Time.at(time).iso8601
-
-          # Append the record to the content in the appropriate format
-          elements.push(record)
-
-          # TODO - check size of content buffer and flush when it approaches 
-          # watermark
-          if false 
-            log.debug "Elements buffer approaching max send size; flushing TODO"    
-            send_data(workspace, key, elements.to_json, log_name)     
-            elements.clear
-          end 
-        end
-
-        if elements.length > 0
-          send_data(workspace, key, elements.to_json, log_name)     
-        end        
-      end
-
-      def send_data(customer_id, shared_key, content, log_type)        
-          current_time = Time.now.utc
-          signature = build_signature(
-            shared_key, current_time, content.length, 
-            "POST", "application/json", "/api/logs")
-          publish_data(log_name, signature, current_time, content)
-      end
-
-      def publish_data(log_name, signature, time, json)
-        url = "https://#{workspace}.ods.opinsights.azure.com/api/logs?api-version=2016-04-01"
-        uri = URI url
-
-        rfc1123date = time.utc.strftime("%a, %d %b %Y %H:%M:%S GMT")
-
-        response = Net::HTTP.start(uri.hostname, uri.port, 
-          :use_ssl => uri.scheme == 'https') do |http|
-
-          req = Net::HTTP::Post.new(uri.to_s)
-          req.body = json.to_s
-          
-          # Signature and headers
-          req['Content-Type'] = 'application/json'
-          req['Log-Type'] = log_name
-          req['Authorization'] = signature
-          req['x-ms-date'] = rfc1123date
-      
-          log.debug "Publishing record of length #{req.body.length} to OMS workspace #{workspace}"    
-          http.request(req)          
-        end
-
-        case response 
-        when Net::HTTPSuccess
-          log.debug "Successfully published record of length #{json.length} to OMS workspace #{workspace}"                 
-        else
-          # TODO - throw error
-          log.warn "Could not publish record of length #{json.length} to OMS workspace #{workspace} because #{response}"
-        end 
-        response
-      end
-  
-      def build_signature(shared_key, date, content_length, method, content_type, resource)
-        rfc1123date = date.utc.strftime("%a, %d %b %Y %H:%M:%S GMT")
-        string_to_hash = "#{method}\n#{content_length}\n#{content_type}\nx-ms-date:#{rfc1123date}\n#{resource}"        
-        decoded_key = Base64.decode64(shared_key)
-        secure_hash = OpenSSL::HMAC.digest('SHA256', decoded_key, string_to_hash)
-              
-        encoded_hash = Base64.encode64(secure_hash).strip()
-        authorization = "SharedKey #{workspace}:#{encoded_hash}"
-      
-        return authorization
-      end    
-    end
-  end
-end
+#
+# Copyright 2018- TODO: Write your name
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "fluent/plugin/output"
+require "time"
+require "base64"
+require "openssl"
+require "uri"
+require "net/https"
+require "json"
+
+module Fluent
+  module Plugin
+    class AzureomsOutput < Fluent::Plugin::Output
+      
+      helpers :event_emitter, :compat_parameters, :record_accessor
+
+      Fluent::Plugin.register_output("azureoms", self)
+
+      DEFAULT_BUFFER_TYPE = "memory"
+
+      config_param :workspace, :string
+      config_param :key, :string, secret: true
+      config_param :timestamp_field, :string, default: "timestamp"
+
+      config_section :buffer do
+        config_set_default :@type, DEFAULT_BUFFER_TYPE
+        config_set_default :chunk_keys, ['tag']
+        config_set_default :timekey_use_utc, true
+      end
+
+      def configure(conf)
+        # This also calls config_param (don't access configuration parameters before
+        # calling super)
+        super       
+
+        # Require chunking by tag keys
+        raise Fluent::ConfigError, "'tag' in chunk_keys is required." if not @chunk_key_tag
+      end
+
+      # This output plugin uses the raw HTTP data collector
+      # API per https://docs.microsoft.com/en-us/azure/log-analytics/log-analytics-data-collector-api
+
+      def prefer_buffered_processing 
+        true
+      end
+
+      def process(tag, es)
+        print "Writing single record set (synchronous)\n"
+        es.each do |time, record|           
+          # Convert record into a JSON body payload for OMS
+          record[:timestamp] = Time.at(time).iso8601          
+
+          # Publish event
+          send_data(workspace, key, record.to_json, tag)        
+        end 
+      end 
+
+      # Synchronous buffered output
+      def write(chunk) 
+        log.debug "Writing buffered record set (synchronous)"
+        log.debug "writing data to file", chunk_id: dump_unique_id_hex(chunk.unique_id)
+        log.debug "writing data to log type", chunk.metadata.tag
+
+        tag = chunk.metadata.tag
+        elements = Array.new
+        
+        chunk.each do |time, record|
+          log.debug "Writing record #{record.inspect}"    
+
+          # Fold the timestamp into the record
+          record[:timestamp] = Time.at(time).iso8601
+
+          # Append the record to the content in the appropriate format
+          elements.push(record)
+
+          # TODO - check size of content buffer and flush when it approaches 
+          # watermark
+          if false 
+            log.debug "Elements buffer approaching max send size; flushing #{elements.length} to logname #{tag}"    
+            send_data(workspace, key, elements.to_json, tag)     
+            elements.clear
+          end 
+        end
+
+        if elements.length > 0
+          log.debug "Flushing #{elements.length} to logname #{tag}"    
+          send_data(workspace, key, elements.to_json, tag)     
+        end        
+      end
+
+      def send_data(customer_id, shared_key, content, log_name)        
+          current_time = Time.now.utc
+          signature = build_signature(
+            shared_key, current_time, content.length, 
+            "POST", "application/json", "/api/logs")
+          publish_data(log_name, signature, current_time, content)
+      end
+
+      def publish_data(log_name, signature, time, json)
+        url = "https://#{workspace}.ods.opinsights.azure.com/api/logs?api-version=2016-04-01"
+        uri = URI url
+
+        rfc1123date = time.utc.strftime("%a, %d %b %Y %H:%M:%S GMT")
+
+        response = Net::HTTP.start(uri.hostname, uri.port, 
+          :use_ssl => uri.scheme == 'https') do |http|
+
+          req = Net::HTTP::Post.new(uri.to_s)
+          req.body = json.to_s
+          
+          # Signature and headers
+          req['Content-Type'] = 'application/json'
+          req['Log-Type'] = log_name
+          req['Authorization'] = signature
+          req['x-ms-date'] = rfc1123date
+      
+          log.debug "Publishing record of length #{req.body.length} to OMS workspace #{workspace}"    
+          http.request(req)          
+        end
+
+        case response 
+        when Net::HTTPSuccess
+          log.debug "Successfully published record of length #{json.length} to OMS workspace #{workspace}"                 
+        else
+          # TODO - throw error
+          log.warn "Could not publish record of length #{json.length} to OMS workspace #{workspace} because #{response}"
+        end 
+        response
+      end
+  
+      def build_signature(shared_key, date, content_length, method, content_type, resource)
+        rfc1123date = date.utc.strftime("%a, %d %b %Y %H:%M:%S GMT")
+        string_to_hash = "#{method}\n#{content_length}\n#{content_type}\nx-ms-date:#{rfc1123date}\n#{resource}"        
+        decoded_key = Base64.decode64(shared_key)
+        secure_hash = OpenSSL::HMAC.digest('SHA256', decoded_key, string_to_hash)
+              
+        encoded_hash = Base64.encode64(secure_hash).strip()
+        authorization = "SharedKey #{workspace}:#{encoded_hash}"
+      
+        return authorization
+      end    
+    end
+  end
+end

data/test/plugin/test_out_azureoms.rb

@@ -8,15 +8,10 @@ require "pp"
 
 class AzureomsOutputTest < Test::Unit::TestCase
   include Fluent::Test::Helpers
-
   
   json_string = '[{"DemoField1":"DemoValue1","DemoField2":"DemoValue2"},{"DemoField3":"DemoValue3","DemoField4":"DemoValue4"}]'
   log_name = "DemoExample";
 
-  hashed_string = '3oXy0IjaKA/V2/Kx+/BlXx7y6tvLYMM8/jwpWBO3c+c='
-  signature = 'SharedKey 424b6a54-6de2-44b5-9768-04497fb2c7e6:1aHwzUHSJ6Ok4ExaKG3i7Opgzy/YqxcjqXH58rssu+I='
-  time = Time::strptime("21-03-2018 00:12:10+00:00", "%d-%m-%Y %H:%M:%S%z")
-
   content_length = json_string.length
   method = "POST"
   content_type = "application/json"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-azureoms
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Mark Simms
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-27 00:00:00.000000000 Z
+date: 2018-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -79,6 +79,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitignore"
 - Gemfile
 - Gemfile.lock
 - LICENSE
@@ -111,7 +112,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.14
+rubygems_version: 2.5.2.1
 signing_key: 
 specification_version: 4
 summary: Fluentd output plugin for Azure Log Analytics