azure-loganalytics-datacollector-api 0.1.4 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f69486484bc9d5e52f6ecaf48c049ff178f85594a1ff83a9c1922db1a30666bc
-  data.tar.gz: c4b1c23ccc77e0273b1447ae5464279270f05ca3f96d2f0e8d2d9e516d198758
+  metadata.gz: '0499f95d897bc46dae9b39636ac2dda0c9aa5750b70bc2703c0d3dc41bfa34f1'
+  data.tar.gz: 057401456546b8ac6b23e75bbc0e6c61a8a4c9ddb534f54508018011d3268d85
 SHA512:
-  metadata.gz: 1e5dd0c8db1d44a861d2506200d7f93b668ac3d8d8a444bea73bed0e464166aa4c194f123a7482e007ed4a67da01ce5aea60933aa95cd667b04a32bdcba3591e
-  data.tar.gz: 72d70760c358307f6b1fb8520bdb30d42e30217174c35875d236dad9799245b9aff7f4b5708a6d329a5c49adc8c73063aeec989a27afa44104c0de8986c2147e
+  metadata.gz: ed73c860640a73f1d0d0176e482361ebb30a082634410c44b7d540fc444b56e1a6548d1f81b43f10f5aef9967e1e8d17b63379c377250ed00b050157c29d4f79
+  data.tar.gz: 5bcb8d45153164a92ddeeaae37544d7722806c8726e2653d2f3fefa07e65f93f854e26e4e657cb9fef1d1d1c26d68d69daa963266b28cb62bfcaef1cbde42b29

data/ChangeLog.md

@@ -1,5 +1,23 @@
+## 0.4.0
+* restclient retries request on the following status code - [issue #10](https://github.com/yokawasa/azure-log-analytics-data-collector/issues/10)
+  * 429 - Too Many Requests
+  * 500 - Internal Server Error
+  * 503 -	Service Unavailable
+
+## 0.3.0
+* Enhance log type validation: check not only alpha but also numeric, underscore, and character length (may not exceed 100) - [issue #11](https://github.com/yokawasa/azure-log-analytics-data-collector/issues/11)
+
+## 0.2.0
+* Support setting the x-ms-AzureResourceId Header - [issue #8](https://github.com/yokawasa/azure-log-analytics-data-collector/issues/8)
+
+## 0.1.6
+* fix CVE-2020-8130 - [issue #7](https://github.com/yokawasa/azure-log-analytics-data-collector/issues/7)
+
+## 0.1.5
+* Add endpoint parameter with the default value for Azure public - [PR#5](https://github.com/yokawasa/azure-log-analytics-data-collector/pull/5)
+
 ## 0.1.4
-* Add `set_proxy` method to client - [issue#2](https://github.com/yokawasa/azure-log-analytics-data-collector/issues/2)
+* Add `set_proxy` method to client - [issue#3](https://github.com/yokawasa/azure-log-analytics-data-collector/issues/3)
 
 ## 0.1.2
 * fixedup bug - [issue #1](https://github.com/yokawasa/azure-log-analytics-data-collector/issues/1)

data/README.md

@@ -1,5 +1,23 @@
 # azure-log-analytics-data-collector Ruby Client
-Azure Log Analytics Data Collector API Ruby Client
+
+[Azure Log Analytics Data Collector API](https://docs.microsoft.com/en-us/azure/azure-monitor/platform/data-collector-api) Client Libraries for Ruby. The repository was originally created for multiple programming languages, but it was refactored as a dedicated one for Ruby client. Python and PHP client libraries were moved to [azure-log-analytics-data-colloector-python](https://github.com/yokawasa/azure-log-analytics-data-collector-python) and [azure-log-analytics-data-colloector-php](https://github.com/yokawasa/azure-log-analytics-data-collector-php) respectively.
+
+
+## Retry policy
+
+The client internal leverage [rest-client] to send HTTP request to the API. The client library retries request using the rest-client on the following status code (which is [recommended action](https://docs.microsoft.com/en-us/azure/azure-monitor/platform/data-collector-api)).
+  * `429` - Too Many Requests
+  * `500` - Internal Server Error
+  * `503` -	Service Unavailable
+
+By default, the client library retres for a total of `3` times, sleeping `5 sec` between retries. The number of retries and sleeping time between retries can be changed with `set_retries` in the client class.
+
+```ruby
+  def set_retries(max_retries, retry_sleep_period)
+    @max_retries = max_retries
+    @retry_sleep_period = retry_sleep_period
+  end
+```
 
 ## Installation
 ```bash
@@ -78,7 +96,51 @@ else
 end
 ```
 
-### Sample3 - use proxy to access the API
+### Sample3 - With time_generated_field and azure_resource_id option
+Supported setting azure_resource_id option from version [0.2.0](https://github.com/yokawasa/azure-log-analytics-data-collector/releases/tag/v0.2.0)
+```ruby
+require "azure/loganalytics/datacollectorapi/client"
+
+customer_id = '<Customer ID aka WorkspaceID String>'
+shared_key = '<The primary or the secondary Connected Sources client authentication key>'
+log_type = "MyCustomLog"
+
+posting_records = []
+record1= {
+  :string => "MyText1",
+  :boolean => true,
+  :number => 100,
+  :timegen => "2017-11-23T11:13:35.576Z" # YYYY-MM-DDThh:mm:ssZ
+}
+record2= {
+  :string => "MyText2",
+  :boolean => false,
+  :number => 200,
+  :timegen => "2017-11-23T12:13:35.576Z" # YYYY-MM-DDThh:mm:ssZ
+}
+posting_records.push(record1)
+posting_records.push(record2)
+
+time_generated_field = "timegen"
+
+# Azure Resource ID
+# [Azure Resource ID Format]
+# /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}
+azure_resource_id ="/subscriptions/11111111-1111-1111-1111-111111111111/resourceGroups/otherResourceGroup/providers/Microsoft.Storage/storageAccounts/examplestorage"
+
+client=Azure::Loganalytics::Datacollectorapi::Client::new( customer_id, shared_key)
+res = client.post_data(log_type, posting_records, time_generated_field, azure_resource_id)
+puts res
+puts "res code=#{res.code}"
+
+if Azure::Loganalytics::Datacollectorapi::Client.is_success(res)
+  puts "operation was succeeded!"
+else
+  puts "operation was failured!"
+end
+```
+
+### Sample4 - use proxy to access the API
 ```ruby
 require "azure/loganalytics/datacollectorapi/client"
 
@@ -114,4 +176,3 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/yokawa
 ## License
 
 The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
-

data/SampleCodes.md

@@ -0,0 +1,12 @@
+# Sample codes for azure-log-analytics-data-collector
+ [Azure Log Analytics HTTP Data Collecotr API](https://docs.microsoft.com/en-us/azure/log-analytics/log-analytics-data-collector-api) Client libraries
+
+## Client libraries and Sample code
+
+| Language | Source Code | Project URL | 
+| ------------- | ------------- |------------- |
+| Ruby     | This repository | [Rubygem Package](https://rubygems.org/gems/azure-loganalytics-datacollector-api) |
+| Python     |  [Source code](https://github.com/yokawasa/azure-log-analytics-data-collector-python) | [Python Package](https://pypi.python.org/pypi/azure-log-analytics-data-collector-api) |
+| PHP     |  [Source code](https://github.com/yokawasa/azure-log-analytics-data-collector-php) | NONE |
+| CSharp   |  [Sample Code](https://docs.microsoft.com/en-us/azure/log-analytics/log-analytics-data-collector-api#sample-requests) | NONE |
+| PowerShell  | [Sample Code](https://docs.microsoft.com/en-us/azure/log-analytics/log-analytics-data-collector-api#sample-requests) | NONE |

data/azure-loganalytics-datacollector-api.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |spec|
   spec.name          = "azure-loganalytics-datacollector-api"
   spec.version       = Azure::Loganalytics::Datacollectorapi::VERSION
   spec.authors       = ["Yoichi Kawasaki"]
-  spec.email         = ["yoichi.kawasaki@outlook.com"]
+  spec.email         = ["yokawasa@gmail.com"]
   spec.summary       = %q{Azure Log Analytics Data Collector API Ruby Client}
   spec.description   = spec.summary
   spec.homepage      = "https://github.com/yokawasa/azure-log-analytics-data-collector"
@@ -20,7 +20,6 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "rest-client"
-  spec.add_development_dependency "bundler", "~> 1.13"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", ">= 12.3.3"
   spec.add_development_dependency "rspec", "~> 3.0"
 end

data/lib/azure/loganalytics/datacollectorapi/client.rb

@@ -8,7 +8,10 @@ module Azure
 
       class Client
 
-        def initialize (customer_id, shared_key)
+        DEFAUT_MAX_RETRIES = 3.freeze
+        DEFAULT_RETRY_SLEEP_PERIOD = 5.freeze
+
+        def initialize (customer_id, shared_key, endpoint ='ods.opinsights.azure.com')
           require 'rest-client'
           require 'json'
           require 'openssl'
@@ -17,15 +20,20 @@ module Azure
 
           @customer_id = customer_id
           @shared_key = shared_key
+          @endpoint = endpoint
+          @default_azure_resource_id = ''
+
+          @max_retries = DEFAUT_MAX_RETRIES
+          @retry_sleep_period = DEFAULT_RETRY_SLEEP_PERIOD    
         end
 
-        def post_data(log_type, json_records, record_timestamp ='')
+        def post_data(log_type, json_records, record_timestamp ='', azure_resource_id ='' )
           raise ConfigError, 'no log_type' if log_type.empty?
-          raise ConfigError, 'log_type must be only alpha characters' if not is_alpha(log_type)
+          raise ConfigError, 'log_type must only contain alpha numeric and _, and not exceed 100 chars' if not is_valid_log_type(log_type)
           raise ConfigError, 'no json_records' if json_records.empty?
           body =  json_records.to_json
-          uri = sprintf("https://%s.ods.opinsights.azure.com/api/logs?api-version=%s",
-                        @customer_id, API_VERSION)
+          uri = sprintf("https://%s.%s/api/logs?api-version=%s",
+                        @customer_id, @endpoint, API_VERSION)
           date = rfc1123date()
           sig = signature(date, body.bytesize)
 
@@ -34,16 +42,39 @@ module Azure
               'Authorization' => sig,
               'Log-Type' => log_type,
               'x-ms-date' => date,
+              'x-ms-AzureResourceId' => azure_resource_id.empty? ? @default_azure_resource_id : azure_resource_id,
               'time-generated-field' => record_timestamp
           }
 
-          res = RestClient.post( uri, body, headers)
-          res
+          retries = 0 
+          begin
+            res = RestClient.post( uri, body, headers)
+            res
+          rescue => e
+            c = e.response.code.to_i
+            if c == 429 || c == 500 || c==503
+              if retries < @max_retries
+                retries += 1
+                sleep(@retry_sleep_period)
+                retry 
+              end
+            end
+            raise e
+          end
         end
 
         def set_proxy(proxy='')
           RestClient.proxy = proxy.empty? ? ENV['http_proxy'] : proxy
         end
+        
+        def set_default_azure_resoruce_id(azure_resource_id)
+          @default_azure_resource_id = azure_resource_id
+        end
+        
+        def set_retres(max_retries, retry_sleep_period)
+          @max_retries = max_retries
+          @retry_sleep_period = retry_sleep_period
+        end
 
         def self.is_success(res)
           return (res.code == 200) ? true : false
@@ -51,8 +82,8 @@ module Azure
 
         private
 
-        def is_alpha(s)
-          return (s.match(/^[[:alpha:]]+$/)) ? true : false
+        def is_valid_log_type(s)
+          return ( s.match(/^[a-zA-Z0-9_]+$/) && s.length <= 100 ) ? true : false
         end
 
         def rfc1123date()

data/lib/azure/loganalytics/datacollectorapi/version.rb

@@ -1,7 +1,7 @@
 module Azure
   module Loganalytics
     module Datacollectorapi
-      VERSION = "0.1.4"
+      VERSION = "0.4.0"
     end
   end
 end

data/spec/azure/loganalytics/datacollectorapi/client_spec.rb

@@ -5,15 +5,36 @@ describe Azure::Loganalytics::Datacollectorapi::Client do
     expect(Azure::Loganalytics::Datacollectorapi::VERSION).not_to be nil
   end
 
-  #it "does something useful" do
-  #  expect(false).to eq(true)
-  #end
+  customer_id = '<Customer ID aka WorkspaceID String>'
+  shared_key =  '<Primary Key String>'
+  log_type = "MyCustomLog"
 
-  it "posting data to datacollector api" do
-    customer_id = '<Customer ID aka WorkspaceID String>'
-    shared_key =  '<Primary Key String>'
-    log_type = "MyCustomLog"
+  it "log type validation" do
+    json_records = []
+    dummy= {
+      :string => "dummy title",
+      :number => 10
+    }
+    valid_log_type = "abcedefghijklmnopqrstuvwxyz1234567890_"
+    invalid_log_type1 = "*-|[]//"
+    invalid_log_type2 = "abcde__xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+    json_records.push(dummy)
+    client=Azure::Loganalytics::Datacollectorapi::Client::new( customer_id, shared_key)
+
+    res = client.post_data(valid_log_type, json_records)
+    expect(Azure::Loganalytics::Datacollectorapi::Client.is_success(res)).to eq(true)
 
+    expect{
+      client.post_data(invalid_log_type1, json_records)
+    }.to raise_error(Exception)
+
+    expect{
+      client.post_data(invalid_log_type2, json_records)
+    }.to raise_error(Exception)
+
+  end
+
+  it "posting data to datacollector api" do
     json_records = []
     record1= {
       :string => "MyText1",
@@ -33,4 +54,53 @@ describe Azure::Loganalytics::Datacollectorapi::Client do
     expect(Azure::Loganalytics::Datacollectorapi::Client.is_success(res)).to eq(true)
   end
 
+  it "posting data to datacollector api with time-generated-field" do
+    json_records = []
+    record1= {
+      :string => "MyText1",
+      :boolean => true,
+      :number => 100,
+      :timegen => "2020-05-05T11:13:35.576Z" # YYYY-MM-DDThh:mm:ssZ
+    }
+    record2= {
+      :string => "MyText2",
+      :boolean => false,
+      :number => 200,
+      :timegen => "2020-05-05T12:13:35.576Z" # YYYY-MM-DDThh:mm:ssZ
+    }
+    json_records.push(record1)
+    json_records.push(record2)
+
+    time_generated_field = "timegen"
+    client=Azure::Loganalytics::Datacollectorapi::Client::new( customer_id, shared_key)
+    res = client.post_data(log_type, json_records, time_generated_field)
+    expect(Azure::Loganalytics::Datacollectorapi::Client.is_success(res)).to eq(true)
+  end
+
+  it "posting data to datacollector api with azure-resource-id" do
+    json_records = []
+    record1= {
+      :string => "MyText1",
+      :boolean => true,
+      :number => 100
+    }
+    record2= {
+      :string => "MyText2",
+      :boolean => false,
+      :number => 200
+    }
+    json_records.push(record1)
+    json_records.push(record2)
+
+    time_generated_field = ""
+    # Azure Resource ID
+    # https://docs.microsoft.com/en-us/azure/azure-resource-manager/templates/template-functions-resource#resourceid
+    # /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}
+    azure_resource_id ="/subscriptions/11111111-1111-1111-1111-111111111111/resourceGroups/otherResourceGroup/providers/Microsoft.Storage/storageAccounts/examplestorage"
+
+    client=Azure::Loganalytics::Datacollectorapi::Client::new( customer_id, shared_key)
+    res = client.post_data(log_type, json_records, time_generated_field, azure_resource_id)
+    expect(Azure::Loganalytics::Datacollectorapi::Client.is_success(res)).to eq(true)
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: azure-loganalytics-datacollector-api
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.4.0
 platform: ruby
 authors:
 - Yoichi Kawasaki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-05-12 00:00:00.000000000 Z
+date: 2020-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -24,34 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.13'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.13'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -68,7 +54,7 @@ dependencies:
         version: '3.0'
 description: Azure Log Analytics Data Collector API Ruby Client
 email:
-- yoichi.kawasaki@outlook.com
+- yokawasa@gmail.com
 executables:
 - console
 - setup
@@ -82,6 +68,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- SampleCodes.md
 - azure-loganalytics-datacollector-api.gemspec
 - bin/console
 - bin/setup
@@ -108,7 +95,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.3
 signing_key: 
 specification_version: 4
 summary: Azure Log Analytics Data Collector API Ruby Client