fluent-plugin-azure-loganalytics 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: b312ebea1de59bae4bc49210d65b1a8314edc225
-  data.tar.gz: a73eaccd5388ade37c58eefba7ac8f6f7dc8d989
+SHA256:
+  metadata.gz: 22b06d1614e45b6e4bd69d561fd9e1baa607aab324877c55279d68e6f95cdf89
+  data.tar.gz: 810e6437147632a77ec333e484282c320a75482c889d1cfe59d5e4410eb495e0
 SHA512:
-  metadata.gz: 7efc7b88b43025f3d49a67178e1538df0b7e5970d8ebb69fcbf6578c432f0c5dd2f30c2fb99736b83de34d2e8b1095c1c3d27ae8bd581d8418de388f5f4a5807
-  data.tar.gz: cc066b03aa2d1f3e836846fbc377aeeaceccca2e51e98de73270ab25657b77d6c5b6def784b17f5aed1cf4136585c9016c2255b0a137ab843ee13569cf17f516
+  metadata.gz: b17088b54b49a51b04d15545b7e15c16cef996ec4ac7c1ff2c1739e383a15f3064a11db2d7bc763833ef325790935c89dc11ff85589b56da9f3f700e3a69ade4
+  data.tar.gz: f80ef8d11245daf1bafa759dbfd26b3b55dbe6649eaa3bf6389c8a5f78a00c2bb76d5499c94051f5b8b5b7b540dd3249d7ec63885df763382faf8b6f85c25422

data/ChangeLog.md

@@ -1,3 +1,7 @@
+## 0.4.0
+* Add endpoint parameter for sovereign cloud - [PR#8](https://github.com/yokawasa/fluent-plugin-azure-loganalytics/pull/8)
+* Changed dependency for azure-loganalytics-datacollector-api to `>= 0.1.5` - [PR#8](https://github.com/yokawasa/fluent-plugin-azure-loganalytics/pull/8)
+
 ## 0.3.1
 
 * Add requirements section - [PR#2](https://github.com/yokawasa/fluent-plugin-azure-loganalytics/pull/2)

data/README.md

@@ -11,10 +11,20 @@
 |  < 0.3.0 | >= v0.12.0 | >= 1.9 |
 
 ## Installation
+### Installing gems into system Ruby
 ```
 $ gem install fluent-plugin-azure-loganalytics
 ```
 
+### Installing gems into td-agent’s Ruby
+If you installed td-agent and want to add this custom plugins, use td-agent-gem to install as td-agent has own Ruby so you should install gems into td-agent’s Ruby, not system Ruby:
+
+```
+$ /usr/sbin/td-agent-gem install fluent-plugin-azure-loganalytics
+```
+Please see also [I installed td-agent and want to add custom plugins. How do I do it?](https://docs.fluentd.org/v0.12/articles/faq#i-installed-td-agent-and-want-to-add-custom-plugins.-how-do-i-do-it?)
+
+
 ## Configuration
 
 ### Azure Log Analytics
@@ -33,6 +43,7 @@ Once you have the workspace, get Workspace ID and Shared Key (either Primary Key
     customer_id CUSTOMER_ID   # Customer ID aka WorkspaceID String
     shared_key KEY_STRING     # The primary or the secondary Connected Sources client authentication key
     log_type EVENT_TYPE_NAME  # The name of the event type. ex) ApacheAccessLog
+    endpoint myendpoint
     add_time_field true
     time_field_name mytime
     time_format %s
@@ -45,6 +56,7 @@ Once you have the workspace, get Workspace ID and Shared Key (either Primary Key
  * **customer\_id (required)** - Your Operations Management Suite workspace ID
  * **shared\_key (required)** - The primary or the secondary Connected Sources client authentication key
  * **log\_type (required)** - The name of the event type that is being submitted to Log Analytics. log_type only supports alpha characters
+ * **endpoint (optional)** - Default:'ods.opinsights.azure.com'. The service endpoint. You may want to use this param in case of sovereign cloud that has a different endpoint from the public cloud
  * **time\_generated\_field (optional)** - Default:''(empty string) The name of the time generated field. Be carefule that the value of field should strictly follow the ISO 8601 format (YYYY-MM-DDThh:mm:ssZ). See also [this](https://docs.microsoft.com/en-us/azure/log-analytics/log-analytics-data-collector-api#create-a-request) for more details
  * **add\_time\_field (optional)** - Default:true. This option allows to insert a time field to record
  * **time\_field\_name (optional)** - Default:time. This is required only when add_time_field is true
@@ -59,7 +71,7 @@ Once you have the workspace, get Workspace ID and Shared Key (either Primary Key
 fluent-plugin-azure-loganalytics adds **time** and **tag** attributes by default if **add_time_field** and **add_tag_field** are true respectively. Below are two types of the plugin configurations - Default and All options configuration.
 
 ### (1) Default Configuration (No options)
-<u>fluent.conf</u>
+<u>fluent_1.conf</u>
 ```
 <source>
     @type tail                         # input plugin
@@ -78,7 +90,33 @@ fluent-plugin-azure-loganalytics adds **time** and **tag** attributes by default
 ```
 
 ### (2) Configuration with All Options
-<u>fluent.conf</u>
+<u>fluent_2.conf</u>
+```
+<source>
+    @type tail                         # input plugin
+    path /var/log/apache2/access.log   # monitoring file
+    pos_file /tmp/fluentd_pos_file     # position file
+    format apache                      # format
+    tag azure-loganalytics.access      # tag
+</source>
+
+<match azure-loganalytics.**>
+    @type azure-loganalytics
+    customer_id 818f7bbc-8034-4cc3-b97d-f068dd4cd658
+    shared_key ppC5500KzCcDsOKwM1yWUvZydCuC3m+ds/2xci0byeQr1G3E0Jkygn1N0Rxx/yVBUrDE2ok3vf4ksCzvBmQXHw==(dummy)
+    log_type ApacheAccessLog
+    add_time_field true
+    time_field_name mytime
+    time_format %s
+    localtime true
+    add_tag_field true
+    tag_field_name mytag
+</match>
+```
+### (3) Configuration with Typecast filter
+
+You want to add typecast filter when you want to cast fields type. The filed type of code and size are cast by typecast filter.
+<u>fluent_typecast.conf</u>
 ```
 <source>
     @type tail                         # input plugin
@@ -88,6 +126,11 @@ fluent-plugin-azure-loganalytics adds **time** and **tag** attributes by default
     tag azure-loganalytics.access      # tag
 </source>
 
+<filter **>
+    @type typecast
+    types host:string,user:string,method:string,path:string,referer:string,agent:string,code:integer,size:integer
+</filter>
+
 <match azure-loganalytics.**>
     @type azure-loganalytics
     customer_id 818f7bbc-8034-4cc3-b97d-f068dd4cd658
@@ -101,6 +144,54 @@ fluent-plugin-azure-loganalytics adds **time** and **tag** attributes by default
     tag_field_name mytag
 </match>
 ```
+[note] you need to install [fluent-plugin-filter-typecast](https://github.com/sonots/fluent-plugin-filter_typecast) for the sample configuration above. 
+```
+gem install fluent-plugin-filter_typecast
+```
+### (4) Configuration with CSV format as input and specific field type as output
+You want to send to Log Analytics, logs generated with known delimiter (like comma, semi-colon) then you can use the csv format of fluentd and the keys/types properties.
+This can be used with any log, here implemented with Nginx custom log.
+<u>fluent_csv.conf</u>
+
+Suppose your log is formated the way below in the  /etc/nginx/conf.d/log.conf:
+```
+log_format appcustomlog '"$time_iso8601";"$hostname";$bytes_sent;$request_time;$upstream_response_length;$upstream_response_time;$content_length;"$remote_addr";$status;"$host";"$request";"$http_user_agent"';
+```
+And this log is activated throught the /etc/nginx/conf.d/virtualhost.conf :
+```
+server {
+	...
+	access_log /var/log/nginx/access.log appcustomlog;
+	...
+}
+```
+You can use the following configuration for the source to tail the log file and format it with proper field type.
+```
+<source>
+  @type tail
+  path /var/log/nginx/access.log
+  pos_file /var/log/td-agent/access.log.pos
+  tag nginx.accesslog
+  format csv
+  delimiter ;
+  keys time,hostname,bytes_sent,request_time,content_length,remote_addr,status,host,request,http_user_agent
+  types time:time,hostname:string,bytes_sent:float,request_time:float,content_length:string,remote_addr:string,status:integer,host:string,request:string,http_user_agent:string
+  time_key time
+  time_format %FT%T%z
+</source>
+
+<match nginx.accesslog>
+    @type azure-loganalytics
+    customer_id 818f7bbc-8034-4cc3-b97d-f068dd4cd658
+    shared_key ppC5500KzCcDsOKwM1yWUvZydCuC3m+ds/2xci0byeQr1G3E0Jkygn1N0Rxx/yVBUrDE2ok3vf4ksCzvBmQXHw==(dummy)
+    log_type NginxAcessLog
+    time_generated_field time
+    time_format %FT%T%z
+    add_tag_field true
+    tag_field_name mytag
+</match>
+```
+
 
 ## Sample inputs and expected records
 
@@ -117,9 +208,19 @@ The output record for sample input can be seen at Log Analytics portal like this
 
 ![fluent-plugin-azure-loganalytics output image](https://github.com/yokawasa/fluent-plugin-azure-loganalytics/raw/master/img/Azure-LogAnalytics-Output-Image.png)
 
+<u>Sample Input (nginx custom access log)</u>
+```
+"2017-12-13T11:31:59+00:00";"nginx0001";21381;0.238;20882;0.178;-;"193.192.35.178";200;"mynginx.domain.com";"GET /mysite/picture.jpeg HTTP/1.1";"Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/63.0.3239.84 Safari/537.36"
+```
+
+<u>Output Record</u>
+
+Part of the output record for sample input can be seen at Log Analytics portal like this with field of type _s (string) or _d (double):
+
+![fluent-plugin-azure-loganalytics output image](https://github.com/yokawasa/fluent-plugin-azure-loganalytics/raw/master/img/Azure-LogAnalytics-Output-Image-2.png)
 
 ## Tests
-### Running test code
+### Running test code (using System rake)
 ```
 $ git clone https://github.com/yokawasa/fluent-plugin-azure-loganalytics.git
 $ cd fluent-plugin-azure-loganalytics
@@ -131,6 +232,18 @@ $ vi test/plugin/test_azure_loganalytics.rb
 $ rake test
 ```
 
+### Running test code (using td-agent's rake)
+```
+$ git clone https://github.com/yokawasa/fluent-plugin-azure-loganalytics.git
+$ cd fluent-plugin-azure-loganalytics
+
+# edit CONFIG params of test/plugin/test_azure_loganalytics.rb
+$ vi test/plugin/test_azure_loganalytics.rb
+
+# run test 
+$ /opt/td-agent/embedded/bin/rake test
+```
+
 ### Creating package, running and testing locally
 ```
 $ rake build
@@ -148,9 +261,9 @@ $ ab -n 5 -c 2 http://localhost/test/foo.html
 
 ## Links
 
-* http://yokawasa.github.io/fluent-plugin-azure-loganalytics
 * https://rubygems.org/gems/fluent-plugin-azure-loganalytics
 * https://rubygems.org/gems/azure-loganalytics-datacollector-api
+* [How to install td-agent and luent-plugin-azure-loganalytics plugin on RHEL](docs/install-tdagent-and-the-plugin-on-rhel.md)
 
 ## Contributing
 

data/VERSION

@@ -1 +1 @@
-0.3.1
+0.4.0

data/docs/install-tdagent-and-the-plugin-on-rhel.md

@@ -0,0 +1,68 @@
+# How to install td-agent and luent-plugin-azure-loganalytics plugin on RHEL
+
+This is a quick installation procedure of td-agent and the custom plugin (fluent-plugin-azure-loganalytics) on Red Hat Enterprise Linux (7.4) 
+
+$ cat /etc/os-release
+```
+NAME="Red Hat Enterprise Linux Server"
+VERSION="7.4 (Maipo)"
+ID="rhel"
+ID_LIKE="fedora"
+VARIANT="Server"
+VARIANT_ID="server"
+VERSION_ID="7.4"
+PRETTY_NAME="Red Hat Enterprise Linux Server 7.4 (Maipo)"
+ANSI_COLOR="0;31"
+CPE_NAME="cpe:/o:redhat:enterprise_linux:7.4:GA:server"
+HOME_URL="https://www.redhat.com/"
+BUG_REPORT_URL="https://bugzilla.redhat.com/"
+
+REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 7"
+REDHAT_BUGZILLA_PRODUCT_VERSION=7.4
+REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux"
+REDHAT_SUPPORT_PRODUCT_VERSION="7.4"
+```
+
+## 0. prerequisites (for Redhat/Centos)
+Install GCC and Development Tools on a CentOS / RHEL 7 server
+```
+$ suod yum group install "Development Tools"
+```
+
+## 1. Install td-agent (fluentd)
+
+Following the [fluentd official page](https://docs.fluentd.org/v0.12/articles/install-by-rpm), install like this:
+
+```
+$ curl -L https://toolbelt.treasuredata.com/sh/install-redhat-td-agent2.sh | sh
+
+$ td-agent --version
+td-agent 0.12.40
+```
+
+## 2. Launching Daemon
+```
+$ sudo /etc/init.d/td-agent start
+$ sudo /etc/init.d/td-agent status
+```
+## 3. Post Sample Logs via HTTP
+By default, /etc/td-agent/td-agent.conf is configured to take logs from HTTP and route them to stdout (/var/log/td-agent/td-agent.log). You can post sample log records using the curl command.
+
+```
+$ curl -X POST -d 'json={"json":"message"}' http://localhost:8888/debug.test
+
+# Checking log (/var/log/td-agent/td-agent.log) and see if the log is written
+$ cat /var/log/td-agent/td-agent.log
+```
+
+## 4. Install the custom plugin
+```
+$ sudo /usr/sbin/td-agent-gem install fluent-plugin-azure-loganalytics
+```
+
+## 5. Testing the plugin
+```
+$ git clone https://github.com/yokawasa/fluent-plugin-azure-loganalytics.git
+$ cd fluent-plugin-azure-loganalytics
+$ /opt/td-agent/embedded/bin/rake test
+```

data/examples/fluent_csv.conf

@@ -0,0 +1,23 @@
+<source>
+  @type tail                                 # input plugin
+  path /var/log/nginx/access.log             # monitoring file
+  pos_file /var/log/td-agent/access.log.pos  # position file
+  format csv                                 # format
+  tag nginx.accesslog                        # tag
+  delimiter ;                                # record delimiter used in source log
+  keys time,hostname,bytes_sent,request_time,content_length,remote_addr,status,host,request,http_user_agent
+  types time:time,hostname:string,bytes_sent:float,request_time:float,content_length:string,remote_addr:string,status:integer,host:string,request:string,http_user_agent:string
+  time_key time
+  time_format %FT%T%z
+</source>
+
+<match nginx.accesslog>
+    @type azure-loganalytics
+    customer_id CUSTOMER_ID   # Customer ID aka WorkspaceID String
+    shared_key KEY_STRING     # The primary or the secondary Connected Sources client authentication key
+    log_type EVENT_TYPE_NAME  # The name of the event type. ex) NginxAcessLog
+    time_generated_field time
+    time_format %FT%T%z
+    add_tag_field true
+    tag_field_name mytag
+</match>

data/examples/fluent_typecast.conf

@@ -0,0 +1,25 @@
+<source>
+    @type tail                         # input plugin
+    path /var/log/apache2/access.log   # monitoring file
+    pos_file /tmp/fluentd_pos_file     # position file
+    format apache                      # format
+    tag azure-loganalytics.access      # tag
+</source>
+
+<filter **>
+    @type typecast
+    types host:string,user:string,method:string,path:string,referer:string,agent:string,code:integer,size:integer
+</filter>
+
+<match azure-loganalytics.**>
+    @type azure-loganalytics
+    customer_id CUSTOMER_ID   # Customer ID aka WorkspaceID String
+    shared_key KEY_STRING     # The primary or the secondary Connected Sources client authentication key
+    log_type EVENT_TYPE_NAME  # The name of the event type. ex) ApacheAccessLog
+    add_time_field true
+    time_field_name mytime
+    time_format %s
+    localtime true
+    add_tag_field true
+    tag_field_name mytag
+</match>

data/fluent-plugin-azure-loganalytics.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |gem|
 
   gem.add_dependency "fluentd", [">= 0.14.15", "< 2"]
   gem.add_dependency "rest-client"
-  gem.add_dependency "azure-loganalytics-datacollector-api", [">= 0.1.2"]
+  gem.add_dependency "azure-loganalytics-datacollector-api", [">= 0.1.5"]
   gem.add_development_dependency "bundler", "~> 1.11"
   gem.add_development_dependency "rake", "~> 10.0"
   gem.add_development_dependency "test-unit"

data/lib/fluent/plugin/out_azure-loganalytics.rb

@@ -16,6 +16,8 @@ module Fluent::Plugin
                  :desc => "Your Operations Management Suite workspace ID"
     config_param :shared_key, :string, :secret => true,
                  :desc => "The primary or the secondary Connected Sources client authentication key"
+    config_param :endpoint, :string, :default =>'ods.opinsights.azure.com',
+                 :desc => "The service endpoint"
     config_param :log_type, :string,
                  :desc => "The name of the event type that is being submitted to Log Analytics. log_type only alpha characters"
     config_param :time_generated_field, :string, :default => '',
@@ -59,7 +61,7 @@ module Fluent::Plugin
     def start
       super
       # start
-      @client=Azure::Loganalytics::Datacollectorapi::Client::new(@customer_id,@shared_key)
+      @client=Azure::Loganalytics::Datacollectorapi::Client::new(@customer_id,@shared_key,@endpoint)
     end
 
     def shutdown

data/test/plugin/test_azure_loganalytics.rb

@@ -22,18 +22,16 @@ class AzureLogAnalyticsOutputTest < Test::Unit::TestCase
 
   def test_configure
     d = create_driver
-    assert_equal '<Customer ID aka WorkspaceID String>', d.instance.customer_id
-    assert_equal '<Primary Key String>', d.instance.shared_key
     assert_equal 'ApacheAccessLog', d.instance.log_type
-    assert_true d.instance.add_time_field
-    assert_true d.instance.localtime
-    assert_true d.instance.add_tag_field
+    assert_equal true, d.instance.add_time_field
+    assert_equal true, d.instance.localtime
+    assert_equal true, d.instance.add_tag_field
     assert_equal 'tag', d.instance.tag_field_name
   end
 
   def test_format
     d = create_driver
-    time = event_time("2011-01-02 13:14:15 UTC")
+    time = event_time("2017-11-24 01:14:15 UTC")
     d.run(default_tag: 'test') do
       d.feed(time, {"a"=>1})
       d.feed(time, {"a"=>2})
@@ -57,9 +55,9 @@ class AzureLogAnalyticsOutputTest < Test::Unit::TestCase
       d.feed(
         time,
         {
-          :Log_ID => "5cdad72f-c848-4df0-8aaa-ffe033e75d57",
-          :date => "2016-12-10 09:44:32 JST",
-          :processing_time => "372",
+          :Log_ID => "5cdad72a-c848-4df0-8aaa-ffe033e75d57",
+          :date => "2017-11-24 01:44:32 JST",
+          :processing_time => 372,
           :remote => "101.202.74.59",
           :user => "-",
           :method => "GET / HTTP/1.1",
@@ -67,15 +65,15 @@ class AzureLogAnalyticsOutputTest < Test::Unit::TestCase
           :size => "-",
           :referer => "-",
           :agent => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:27.0) Gecko/20100101 Firefox/27.0",
-          :eventtime => "2016-12-10T09:44:32Z"
+          :eventtime => "2017-11-24T01:44:32Z"
         })
 
       d.feed(
         time,
         {
-          :Log_ID => "7260iswx-8034-4cc3-uirtx-f068dd4cd659",
-          :date => "2016-12-10 09:45:14 JST",
-          :processing_time => "105",
+          :Log_ID => "7260iswa-8034-4cc3-uirtx-f068dd4cd659",
+          :date => "2017-11-24 01:45:14 JST",
+          :processing_time => 105,
           :remote => "201.78.74.59",
           :user => "-",
           :method => "GET /manager/html HTTP/1.1",
@@ -83,7 +81,7 @@ class AzureLogAnalyticsOutputTest < Test::Unit::TestCase
           :size => "-",
           :referer => "-",
           :agent => "Mozilla/5.0 (Windows NT 5.1; rv:5.0) Gecko/20100101 Firefox/5.0",
-          :eventtime => "2016-12-10T09:45:14Z"
+          :eventtime => "2017-11-24T01:45:14Z"
         })
     end
     data = d.events

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-azure-loganalytics
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Yoichi Kawasaki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-05 00:00:00.000000000 Z
+date: 2019-06-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -50,14 +50,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.2
+        version: 0.1.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.2
+        version: 0.1.5
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -113,10 +113,14 @@ files:
 - README.md
 - Rakefile
 - VERSION
+- docs/install-tdagent-and-the-plugin-on-rhel.md
 - examples/fluent_1.conf
 - examples/fluent_2.conf
+- examples/fluent_csv.conf
+- examples/fluent_typecast.conf
 - fluent-plugin-azure-loganalytics.gemspec
 - img/Azure-LogAnalytics-Fluentd.png
+- img/Azure-LogAnalytics-Output-Image-2.png
 - img/Azure-LogAnalytics-Output-Image.png
 - lib/fluent/plugin/out_azure-loganalytics.rb
 - test/helper.rb
@@ -140,8 +144,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Azure Log Analytics output plugin for Fluentd