azure-loganalytics-datacollector-api 0.1.5 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 130505bbd3cda48ee4330fba0c5fe76814a6414d20676a7691734cd38c7a7714
-  data.tar.gz: ff86325bc1df19f3c505efcb27c194684a9f2e2111149b2a1428ecfe044ea891
+  metadata.gz: 3d3abe715a08517a56495053c816fc36a814c4d373e82af5d96a108cc9ee5d31
+  data.tar.gz: db6499466afb04d793a80ca16ed91cc3654dbc7cb897c9e8e0a7dec3cce15d5f
 SHA512:
-  metadata.gz: 7c6998cb89da3454ab39605c4c801d96880beee4e80a2630d66d84ae8a5c5d2675b741f49cc5d61cc7ef0730582b3a67dfdc2f123c93959f01e0d4dddf9e87b4
-  data.tar.gz: 679d59d0c792db18c0be081f0baf450245d41ac70dc8c2778de52d9d58bbbdd22a32ef03e07f488fd034a3be85de384b6adde2df410eba23debd7d70bcde1d2f
+  metadata.gz: 76263c50638ec683371727e61240bb705242226757fb928c841896b6a0a22af48ce76cc9afefdf2a5271b2dee50e13ee2346d866ee35c3bfac6e985e7bc71cce
+  data.tar.gz: 87220e1418b867e5eb1a8e32a182c0a6e2ff813405d493e67473dc7f049bca382f14003c9548a2342a9f4b53563c8565ade2d79b393a7691f13c0f3580af61e7

data/ChangeLog.md

@@ -1,10 +1,38 @@
+# ChangeLog
+
+## 0.5.0
+
+* Check body size not to exceed data limit of 30MB in Azure Monitor Data Collection API - [issue #12](https://github.com/yokawasa/azure-log-analytics-data-collector/issues/12)
+
+## 0.4.0
+
+* restclient retries request on the following status code - [issue #10](https://github.com/yokawasa/azure-log-analytics-data-collector/issues/10)
+  * 429 - Too Many Requests
+  * 500 - Internal Server Error
+  * 503 -	Service Unavailable
+
+## 0.3.0
+
+* Enhance log type validation: check not only alpha but also numeric, underscore, and character length (may not exceed 100) - [issue #11](https://github.com/yokawasa/azure-log-analytics-data-collector/issues/11)
+
+## 0.2.0
+
+* Support setting the x-ms-AzureResourceId Header - [issue #8](https://github.com/yokawasa/azure-log-analytics-data-collector/issues/8)
+
+## 0.1.6
+
+* fix CVE-2020-8130 - [issue #7](https://github.com/yokawasa/azure-log-analytics-data-collector/issues/7)
+
 ## 0.1.5
+
 * Add endpoint parameter with the default value for Azure public - [PR#5](https://github.com/yokawasa/azure-log-analytics-data-collector/pull/5)
 
 ## 0.1.4
+
 * Add `set_proxy` method to client - [issue#3](https://github.com/yokawasa/azure-log-analytics-data-collector/issues/3)
 
 ## 0.1.2
+
 * fixedup bug - [issue #1](https://github.com/yokawasa/azure-log-analytics-data-collector/issues/1)
 
 ## 0.1.1

data/README.md

@@ -1,5 +1,23 @@
 # azure-log-analytics-data-collector Ruby Client
-Azure Log Analytics Data Collector API Ruby Client
+
+[Azure Log Analytics Data Collector API](https://docs.microsoft.com/en-us/azure/azure-monitor/platform/data-collector-api) Client Libraries for Ruby. The repository was originally created for multiple programming languages, but it was refactored as a dedicated one for Ruby client. Python and PHP client libraries were moved to [azure-log-analytics-data-colloector-python](https://github.com/yokawasa/azure-log-analytics-data-collector-python) and [azure-log-analytics-data-colloector-php](https://github.com/yokawasa/azure-log-analytics-data-collector-php) respectively.
+
+
+## Retry policy
+
+The client internal leverage [rest-client] to send HTTP request to the API. The client library retries request using the rest-client on the following status code (which is [recommended action](https://docs.microsoft.com/en-us/azure/azure-monitor/platform/data-collector-api)).
+  * `429` - Too Many Requests
+  * `500` - Internal Server Error
+  * `503` -	Service Unavailable
+
+By default, the client library retres for a total of `3` times, sleeping `5 sec` between retries. The number of retries and sleeping time between retries can be changed with `set_retries` in the client class.
+
+```ruby
+  def set_retries(max_retries, retry_sleep_period)
+    @max_retries = max_retries
+    @retry_sleep_period = retry_sleep_period
+  end
+```
 
 ## Installation
 ```bash
@@ -78,7 +96,51 @@ else
 end
 ```
 
-### Sample3 - use proxy to access the API
+### Sample3 - With time_generated_field and azure_resource_id option
+Supported setting azure_resource_id option from version [0.2.0](https://github.com/yokawasa/azure-log-analytics-data-collector/releases/tag/v0.2.0)
+```ruby
+require "azure/loganalytics/datacollectorapi/client"
+
+customer_id = '<Customer ID aka WorkspaceID String>'
+shared_key = '<The primary or the secondary Connected Sources client authentication key>'
+log_type = "MyCustomLog"
+
+posting_records = []
+record1= {
+  :string => "MyText1",
+  :boolean => true,
+  :number => 100,
+  :timegen => "2017-11-23T11:13:35.576Z" # YYYY-MM-DDThh:mm:ssZ
+}
+record2= {
+  :string => "MyText2",
+  :boolean => false,
+  :number => 200,
+  :timegen => "2017-11-23T12:13:35.576Z" # YYYY-MM-DDThh:mm:ssZ
+}
+posting_records.push(record1)
+posting_records.push(record2)
+
+time_generated_field = "timegen"
+
+# Azure Resource ID
+# [Azure Resource ID Format]
+# /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}
+azure_resource_id ="/subscriptions/11111111-1111-1111-1111-111111111111/resourceGroups/otherResourceGroup/providers/Microsoft.Storage/storageAccounts/examplestorage"
+
+client=Azure::Loganalytics::Datacollectorapi::Client::new( customer_id, shared_key)
+res = client.post_data(log_type, posting_records, time_generated_field, azure_resource_id)
+puts res
+puts "res code=#{res.code}"
+
+if Azure::Loganalytics::Datacollectorapi::Client.is_success(res)
+  puts "operation was succeeded!"
+else
+  puts "operation was failured!"
+end
+```
+
+### Sample4 - use proxy to access the API
 ```ruby
 require "azure/loganalytics/datacollectorapi/client"
 
@@ -114,4 +176,3 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/yokawa
 ## License
 
 The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
-

data/SampleCodes.md

@@ -0,0 +1,12 @@
+# Sample codes for azure-log-analytics-data-collector
+ [Azure Log Analytics HTTP Data Collecotr API](https://docs.microsoft.com/en-us/azure/log-analytics/log-analytics-data-collector-api) Client libraries
+
+## Client libraries and Sample code
+
+| Language | Source Code | Project URL | 
+| ------------- | ------------- |------------- |
+| Ruby     | This repository | [Rubygem Package](https://rubygems.org/gems/azure-loganalytics-datacollector-api) |
+| Python     |  [Source code](https://github.com/yokawasa/azure-log-analytics-data-collector-python) | [Python Package](https://pypi.python.org/pypi/azure-log-analytics-data-collector-api) |
+| PHP     |  [Source code](https://github.com/yokawasa/azure-log-analytics-data-collector-php) | NONE |
+| CSharp   |  [Sample Code](https://docs.microsoft.com/en-us/azure/log-analytics/log-analytics-data-collector-api#sample-requests) | NONE |
+| PowerShell  | [Sample Code](https://docs.microsoft.com/en-us/azure/log-analytics/log-analytics-data-collector-api#sample-requests) | NONE |

data/azure-loganalytics-datacollector-api.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |spec|
   spec.name          = "azure-loganalytics-datacollector-api"
   spec.version       = Azure::Loganalytics::Datacollectorapi::VERSION
   spec.authors       = ["Yoichi Kawasaki"]
-  spec.email         = ["yoichi.kawasaki@outlook.com"]
+  spec.email         = ["yokawasa@gmail.com"]
   spec.summary       = %q{Azure Log Analytics Data Collector API Ruby Client}
   spec.description   = spec.summary
   spec.homepage      = "https://github.com/yokawasa/azure-log-analytics-data-collector"
@@ -20,7 +20,6 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "rest-client"
-  spec.add_development_dependency "bundler", "~> 1.13"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", ">= 12.3.3"
   spec.add_development_dependency "rspec", "~> 3.0"
 end

data/lib/azure/loganalytics/datacollectorapi/client.rb

@@ -8,7 +8,13 @@ module Azure
 
       class Client
 
-        def initialize (customer_id, shared_key,endpoint ='ods.opinsights.azure.com')
+        DEFAUT_MAX_RETRIES = 3.freeze
+        DEFAULT_RETRY_SLEEP_PERIOD = 5.freeze
+        # API Data Limits
+        # https://docs.microsoft.com/en-us/azure/azure-monitor/platform/data-collector-api#data-limits
+        MAX_BODY_BYTE_SIZE = 31457280.freeze   # 30 MB
+
+        def initialize (customer_id, shared_key, endpoint ='ods.opinsights.azure.com')
           require 'rest-client'
           require 'json'
           require 'openssl'
@@ -18,13 +24,21 @@ module Azure
           @customer_id = customer_id
           @shared_key = shared_key
           @endpoint = endpoint
+          @default_azure_resource_id = ''
+
+          @max_retries = DEFAUT_MAX_RETRIES
+          @retry_sleep_period = DEFAULT_RETRY_SLEEP_PERIOD    
         end
 
-        def post_data(log_type, json_records, record_timestamp ='')
+        def post_data(log_type, json_records, record_timestamp ='', azure_resource_id ='' )
           raise ConfigError, 'no log_type' if log_type.empty?
-          raise ConfigError, 'log_type must be only alpha characters' if not is_alpha(log_type)
+          raise ConfigError, 'log_type must only contain alpha numeric and _, and not exceed 100 chars' if not is_valid_log_type(log_type)
           raise ConfigError, 'no json_records' if json_records.empty?
+
           body =  json_records.to_json
+          body_size = body.bytesize
+          raise "too large payload (#{body_size})! max post data size is #{MAX_BODY_BYTE_SIZE} bytes!" if body_size >= MAX_BODY_BYTE_SIZE
+
           uri = sprintf("https://%s.%s/api/logs?api-version=%s",
                         @customer_id, @endpoint, API_VERSION)
           date = rfc1123date()
@@ -35,16 +49,39 @@ module Azure
               'Authorization' => sig,
               'Log-Type' => log_type,
               'x-ms-date' => date,
+              'x-ms-AzureResourceId' => azure_resource_id.empty? ? @default_azure_resource_id : azure_resource_id,
               'time-generated-field' => record_timestamp
           }
 
-          res = RestClient.post( uri, body, headers)
-          res
+          retries = 0 
+          begin
+            res = RestClient.post( uri, body, headers)
+            res
+          rescue => e
+            c = e.response.code.to_i
+            if c == 429 || c == 500 || c==503
+              if retries < @max_retries
+                retries += 1
+                sleep(@retry_sleep_period)
+                retry 
+              end
+            end
+            raise e
+          end
         end
 
         def set_proxy(proxy='')
           RestClient.proxy = proxy.empty? ? ENV['http_proxy'] : proxy
         end
+        
+        def set_default_azure_resoruce_id(azure_resource_id)
+          @default_azure_resource_id = azure_resource_id
+        end
+        
+        def set_retres(max_retries, retry_sleep_period)
+          @max_retries = max_retries
+          @retry_sleep_period = retry_sleep_period
+        end
 
         def self.is_success(res)
           return (res.code == 200) ? true : false
@@ -52,8 +89,8 @@ module Azure
 
         private
 
-        def is_alpha(s)
-          return (s.match(/^[[:alpha:]]+$/)) ? true : false
+        def is_valid_log_type(s)
+          return ( s.match(/^[a-zA-Z0-9_]+$/) && s.length <= 100 ) ? true : false
         end
 
         def rfc1123date()

data/lib/azure/loganalytics/datacollectorapi/version.rb

@@ -1,7 +1,7 @@
 module Azure
   module Loganalytics
     module Datacollectorapi
-      VERSION = "0.1.5"
+      VERSION = "0.5.0"
     end
   end
 end

data/spec/azure/loganalytics/datacollectorapi/client_spec.rb

@@ -5,15 +5,36 @@ describe Azure::Loganalytics::Datacollectorapi::Client do
     expect(Azure::Loganalytics::Datacollectorapi::VERSION).not_to be nil
   end
 
-  #it "does something useful" do
-  #  expect(false).to eq(true)
-  #end
+  customer_id = '<Customer ID aka WorkspaceID String>'
+  shared_key =  '<Primary Key String>'
+  log_type = "MyCustomLog"
 
-  it "posting data to datacollector api" do
-    customer_id = '<Customer ID aka WorkspaceID String>'
-    shared_key =  '<Primary Key String>'
-    log_type = "MyCustomLog"
+  it "log type validation" do
+    json_records = []
+    dummy= {
+      :string => "dummy title",
+      :number => 10
+    }
+    valid_log_type = "abcedefghijklmnopqrstuvwxyz1234567890_"
+    invalid_log_type1 = "*-|[]//"
+    invalid_log_type2 = "abcde__xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+    json_records.push(dummy)
+    client=Azure::Loganalytics::Datacollectorapi::Client::new( customer_id, shared_key)
+
+    res = client.post_data(valid_log_type, json_records)
+    expect(Azure::Loganalytics::Datacollectorapi::Client.is_success(res)).to eq(true)
 
+    expect{
+      client.post_data(invalid_log_type1, json_records)
+    }.to raise_error(Exception)
+
+    expect{
+      client.post_data(invalid_log_type2, json_records)
+    }.to raise_error(Exception)
+
+  end
+
+  it "posting data to datacollector api" do
     json_records = []
     record1= {
       :string => "MyText1",
@@ -33,4 +54,53 @@ describe Azure::Loganalytics::Datacollectorapi::Client do
     expect(Azure::Loganalytics::Datacollectorapi::Client.is_success(res)).to eq(true)
   end
 
+  it "posting data to datacollector api with time-generated-field" do
+    json_records = []
+    record1= {
+      :string => "MyText1",
+      :boolean => true,
+      :number => 100,
+      :timegen => "2020-05-05T11:13:35.576Z" # YYYY-MM-DDThh:mm:ssZ
+    }
+    record2= {
+      :string => "MyText2",
+      :boolean => false,
+      :number => 200,
+      :timegen => "2020-05-05T12:13:35.576Z" # YYYY-MM-DDThh:mm:ssZ
+    }
+    json_records.push(record1)
+    json_records.push(record2)
+
+    time_generated_field = "timegen"
+    client=Azure::Loganalytics::Datacollectorapi::Client::new( customer_id, shared_key)
+    res = client.post_data(log_type, json_records, time_generated_field)
+    expect(Azure::Loganalytics::Datacollectorapi::Client.is_success(res)).to eq(true)
+  end
+
+  it "posting data to datacollector api with azure-resource-id" do
+    json_records = []
+    record1= {
+      :string => "MyText1",
+      :boolean => true,
+      :number => 100
+    }
+    record2= {
+      :string => "MyText2",
+      :boolean => false,
+      :number => 200
+    }
+    json_records.push(record1)
+    json_records.push(record2)
+
+    time_generated_field = ""
+    # Azure Resource ID
+    # https://docs.microsoft.com/en-us/azure/azure-resource-manager/templates/template-functions-resource#resourceid
+    # /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}
+    azure_resource_id ="/subscriptions/11111111-1111-1111-1111-111111111111/resourceGroups/otherResourceGroup/providers/Microsoft.Storage/storageAccounts/examplestorage"
+
+    client=Azure::Loganalytics::Datacollectorapi::Client::new( customer_id, shared_key)
+    res = client.post_data(log_type, json_records, time_generated_field, azure_resource_id)
+    expect(Azure::Loganalytics::Datacollectorapi::Client.is_success(res)).to eq(true)
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: azure-loganalytics-datacollector-api
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.5.0
 platform: ruby
 authors:
 - Yoichi Kawasaki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-06-06 00:00:00.000000000 Z
+date: 2020-07-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -24,34 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.13'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.13'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -68,7 +54,7 @@ dependencies:
         version: '3.0'
 description: Azure Log Analytics Data Collector API Ruby Client
 email:
-- yoichi.kawasaki@outlook.com
+- yokawasa@gmail.com
 executables:
 - console
 - setup
@@ -82,6 +68,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- SampleCodes.md
 - azure-loganalytics-datacollector-api.gemspec
 - bin/console
 - bin/setup
@@ -108,7 +95,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Azure Log Analytics Data Collector API Ruby Client