logstash-input-cloudflare 0.9.10 → 0.9.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 50f6a19fe24103c587edeff10c6d39bd4f7c3d3c
-  data.tar.gz: e40407c9b7d7d5e3c2c25d52e31837fa34e9fc77
+  metadata.gz: 94f95e1f9294b1290f75f3746bc1388c4eeea9d2
+  data.tar.gz: b9cad63d34e29be682d81ae49029ebec572f6ab7
 SHA512:
-  metadata.gz: de7fe10c67e3186317f3cf85640a08e8d4421d8c864e3604f6c5c8e817e51f79cd99947d12f83c76bce92df083ac802f0c9f4782f58d8614e16c716f7b860d0d
-  data.tar.gz: b66187a1dc38f25305191abc199ed7730bffc990421197a7208ee9057b75971765ddcd2006f0c9af9b27b8f3467cd19910d132d208f19302a75ae3e22106f9b2
+  metadata.gz: 94ba03576fa49b97f0b9e760ba3a1ef33d2e465cfc0baae77503d582fc13c2f7ea4891f0f2bfa9012c6afd98abc81b5a5b03249630996d3d1e251d3fb2bc960b
+  data.tar.gz: 3d4883a7ccae03efd7bc64935daf7e24cf8b09cbe18229e63c8111d48b7f8de3ea0abed4ffcdff718b0907df73ae549bb0175747b8944bfd597e241d22175c6b

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.9.11
+- added option to set open and read timeouts for API calls.
+
 ## 0.9.10
 - fixed an issue when Cloudflare wouldn't return a value for a field, Elasticsearch would return
   an error and fail to process the message. Error in question was:

data/README.md

@@ -2,10 +2,53 @@
 
 [![Circle CI](https://circleci.com/gh/iserko/logstash-input-cloudflare/tree/master.svg?style=svg&circle-token=78044d92053ebb2ad4ca3b45cdf3cbd271d71ac1)](https://circleci.com/gh/iserko/logstash-input-cloudflare/tree/master)
 
-This is a plugin for [Logstash](https://github.com/elastic/logstash).
+This is a plugin for [Logstash](https://github.com/elastic/logstash) and it allows Logstash to read web request logs from the Cloudflare ELS API. Then Logstash can parse those logs and store them into your store (ElasticSearch if you use the ELK stack).
+
+The repository provides a sample logstash configuration which you can use and should just work (as long as you fill in `CF_AUTH_EMAIL`, `CF_AUTH_KEY` and `CF_DOMAIN`). Take the [example config file](https://github.com/iserko/logstash-input-cloudflare/blob/master/logstash.conf.m4) and play around.
+
+Take a look at [Cloudflare Support page for Enterprise Log Share REST API](https://support.cloudflare.com/hc/en-us/articles/216672448-Enterprise-Log-Share-REST-API) for more information about the  feature. **You are required to be a Cloudflare's Enterprise customer in order to use this plugin**
+
+## Configuration
+
+```
+input {
+    cloudflare {
+        auth_email => "CF_AUTH_EMAIL"
+        auth_key => "CF_AUTH_KEY"
+        domain => "CF_DOMAIN"
+        type => "cloudflare_logs"
+        poll_time => 15
+        poll_interval => 120
+        metadata_filepath => "/logstash-input-cloudflare/cf_metadata.json"
+        fields => [
+          'timestamp', 'zoneId', 'ownerId', 'zoneName', 'rayId', 'securityLevel',
+          'client.ip', 'client.country', 'client.sslProtocol', 'client.sslCipher',
+          'client.deviceType', 'client.asNum', 'clientRequest.bytes',
+          'clientRequest.httpHost', 'clientRequest.httpMethod', 'clientRequest.uri',
+          'clientRequest.httpProtocol', 'clientRequest.userAgent', 'cache.cacheStatus',
+          'edge.cacheResponseTime', 'edge.startTimestamp', 'edge.endTimestamp',
+          'edgeResponse.status', 'edgeResponse.bytes', 'edgeResponse.bodyBytes',
+          'originResponse.status', 'origin.responseTime'
+        ]
+    }
+}
+```
+
+Setting | Description | Default Value | Required
+------- | ----------- | ------------- | --------
+auth_email | Email used to login to Cloudflare (suggest creating a new user with only the permissions to access the ELS API | - | true
+auth_key | API key user to login to Cloudflare | - | true
+domain | The domain you watch to read logs for (since Cloudflare works on top level domains, that usually means something like `example.com`) | - | true
+poll_time | The time in seconds between different API calls | 15 | false
+poll_interval | The time in seconds which determines how many web request logs we pull down from the API (only used when there is no state) | 60 | false
+start_from_secs_ago | The time in seconds which determines how far back in the past you want to start processing logs from | 1200 | false
+batch_size | Number of events per API call to get. Helps reduce memory overhead | 1000 | false
+fields | List of fields you want to process from the API (read the [ELS schema](https://support.cloudflare.com/hc/en-us/article_attachments/205413947/els_schema.json)) | See [fields](https://github.com/iserko/logstash-input-cloudflare/blob/master/lib/logstash/inputs/cloudflare.rb#L54-L60) | false
 
 ## Running in isolation (for testing)
 
+**You need to be running Docker locally in order to use this!**
+
 ```
 export CF_AUTH_EMAIL=<email>
 export CF_AUTH_KEY=<api_key>
@@ -14,4 +57,4 @@ make
 ```
 
 Logstash will run in verbose mode, so you will see some messages coming through. In order to verify you're getting results you can open up your browser to http://&lt;IP&gt;:5601 and check Kibana.
-Value for the IP address is whatever `docker-machine ip default` says.
+Value for the IP address is whatever `docker-machine ip default` says or if you use Docker For Mac ... it's just 127.0.0.1.

data/lib/logstash/inputs/cloudflare.rb

@@ -72,6 +72,8 @@ class LogStash::Inputs::Cloudflare < LogStash::Inputs::Base
          validate: :string, default: '/var/lib/logstash/cf_metadata.json', required: false
   config :poll_time, validate: :number, default: 15, required: false
   config :poll_interval, validate: :number, default: 60, required: false
+  config :open_timeout, validate: :number, default: 60, required: false
+  config :read_timeout, validate: :number, default: 60, required: false
   config :start_from_secs_ago, validate: :number, default: 1200, required: false
   config :batch_size, validate: :number, default: 1000, required: false
   config :fields, validate: :array, default: DEFAULT_FIELDS, required: false
@@ -130,7 +132,10 @@ class LogStash::Inputs::Cloudflare < LogStash::Inputs::Base
   def cloudflare_api_call(endpoint, params, multi_line = false)
     uri = _build_uri(endpoint, params)
     @logger.info('Sending request to Cloudflare')
-    Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
+    Net::HTTP.start(uri.hostname, uri.port,
+                    use_ssl: true,
+                    open_timeout: @open_timeout,
+                    read_timeout: @read_timeout) do |http|
       request = Net::HTTP::Get.new(
         uri.request_uri,
         'Accept-Encoding' => 'gzip',
@@ -144,7 +149,11 @@ class LogStash::Inputs::Cloudflare < LogStash::Inputs::Base
 
   def cloudflare_zone_id(domain)
     params = { status: 'active', name: domain }
-    response = cloudflare_api_call('/zones', params)
+    begin
+      response = cloudflare_api_call('/zones', params)
+    rescue Timeout::Error
+      raise 'Cloudflare API timed out. Consider adjusting the read_timeout.'
+    end
     response['result'].each do |zone|
       return zone['id'] if zone['name'] == domain
     end
@@ -208,6 +217,9 @@ class LogStash::Inputs::Cloudflare < LogStash::Inputs::Base
                       "#{error['message']}")
       end
       entries = []
+    rescue Timeout::Error
+      @logger.error('Cloudflare API timed out. Consider adjusting the read_timeout.')
+      entries = []
     end
     return entries unless entries.empty?
     @logger.info('No entries returned from Cloudflare')

data/logstash-input-cloudflare.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-input-cloudflare'
-  s.version = '0.9.10'
+  s.version = '0.9.11'
   s.licenses = ['Apache License (2.0)']
   s.summary = 'This logstash input plugin fetches logs from Cloudflare using'\
               'their API'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-cloudflare
 version: !ruby/object:Gem::Version
-  version: 0.9.10
+  version: 0.9.11
 platform: ruby
 authors:
 - Igor Serko
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-02-23 00:00:00.000000000 Z
+date: 2017-06-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement