logstash-input-cloudflare_logs 0.1.10 → 0.1.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b2e7718a2db8c32d7b87f6a6952b26558069b1df
-  data.tar.gz: 719dbb75a6a5088e2b3ad06784ff143350e7726f
+  metadata.gz: 774d459f36cd5d6aa1c570f3c6bf7f900a14fa84
+  data.tar.gz: 5704d3a50ee2c3baddb254af75dbb3bcd7569df5
 SHA512:
-  metadata.gz: 29b431afd92e56a57385c574793cdcdeacca6939c76dcf0ce22196f5d53638db49b01139827290051c1a108d172b69dd1d7eef1491c704a3cbfc203ad1b37940
-  data.tar.gz: 023d9bb7b455afe0e69a4ee679748312a9a86bfd86c807462a4732b0195d1d712502cc5a20540e978726e9956c894d4b987c21084b899d6d827b5e4744a21d7a
+  metadata.gz: 9c18dcc7c4d292d7c8162c8b96674a7bcc21b105919705b3828fc9c205688aaa81c7eae8304c7bfaeee48f63612c0ee9013ca036bf667009126fe5850470eea8
+  data.tar.gz: d4e3f3193c31c13b0729e0d7449e0b7932cb5061c51920336f6eb9580b8de830eab54fde10f5e4ce530ecd9ad57d61b4f4a1c50516909a9e70c3852f7f2e72e5

data/lib/logstash/inputs/cloudflare_logs.rb

@@ -54,10 +54,13 @@ class CloudflareAccess
   end
 
   def logs
-    @logger.warn('starting to get logs')
-    results = RestClient.get("https://api.cloudflare.com/client/v4/zones/#{domain}/logs/received?start=#{start_time}&end=#{end_time}&fields=#{fields.join(',')}", 'X-Auth-Email' => auth_email, 'X-Auth-Key' => auth_key)
-    @logger.warn('got logs')
-    results.body.split("\n").collect { |raw_log| JSON.parse(raw_log) }
+    begin
+      results = RestClient.get("https://api.cloudflare.com/client/v4/zones/#{domain}/logs/received?start=#{start_time}&end=#{end_time}&fields=#{fields.join(',')}", 'X-Auth-Email' => auth_email, 'X-Auth-Key' => auth_key)
+      results.body.split("\n").collect { |raw_log| JSON.parse(raw_log) }
+    rescue RestClient::BadRequest => error
+      @logger.error(error.response.strip)
+      raise error
+    end
   end
 end
 
@@ -92,12 +95,13 @@ class LogStash::Inputs::CloudflareLogs < LogStash::Inputs::Base
       log['fields.type'] = 'cloudflare'
       log['fields.env'] = @environment_name
       event = LogStash::Event.new(log)
-      event.timestamp=  LogStash::Timestamp.at(log['EdgeStartTimestamp'].to_i/1_000_000_000)
+      event.timestamp= LogStash::Timestamp.at(log['EdgeStartTimestamp'].to_i / 1_000_000_000)
       decorate(event)
       queue << event
     end
 
     cloudflare_access.update_metadata_file('start_time', cloudflare_access.end_time)
+    @access = nil
   end
 
   def run(queue)

data/logstash-input-cloudflare-logs.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-input-cloudflare_logs'
-  s.version       = '0.1.10'
+  s.version       = '0.1.11'
   s.licenses      = ['Apache License (2.0)']
   s.summary       = 'write cloudflare logs to logstash'
   s.description   = 'Write cloudflare lgos to logstash.' \
@@ -39,4 +39,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'webmock'
   s.add_development_dependency 'vcr', '> 3.0.0', '< 4.0.0'
   s.add_development_dependency 'timecop'
+  s.add_development_dependency 'pry'
 end

data/spec/fixtures/vcr_cassettes/error.yml

@@ -0,0 +1,52 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://api.cloudflare.com/client/v4/zones/11/logs/received?end=2017-10-9T23:55:00Z&fields=CacheCacheStatus,CacheResponseBytes,CacheResponseStatus,ClientASN,ClientCountry,ClientDeviceType,ClientIP,ClientIPClass,ClientRequestBytes,ClientRequestHost,ClientRequestMethod,ClientRequestProtocol,ClientRequestReferer,ClientRequestURI,ClientRequestUserAgent,ClientSSLCipher,ClientSSLProtocol,ClientSrcPort,EdgeColoID,EdgeEndTimestamp,EdgePathingStatus,EdgeResponseBytes,EdgeResponseCompressionRatio,EdgeResponseStatus,EdgeStartTimestamp,OriginIP,OriginResponseBytes,OriginResponseHTTPExpires,OriginResponseHTTPLastModified,OriginResponseStatus,OriginResponseTime,RayID,WAFAction,WAFRuleID,ZoneID&start=2017-10-10T23:54:57Z
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      X-Auth-Email:
+      - test@test.com
+      X-Auth-Key:
+      - secret
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 400
+      message: Bad Request
+    headers:
+      Date:
+      - Tue, 21 Nov 2017 15:06:13 GMT
+      Content-Type:
+      - application/json
+      Transfer-Encoding:
+      - chunked
+      Connection:
+      - keep-alive
+      Set-Cookie:
+      - __cfduid=dbe9fc98db8aa0e054f739756baf7f9c51511276772; expires=Wed, 21-Nov-18
+        15:06:12 GMT; path=/; domain=.cloudflare.com; HttpOnly
+      Strict-Transport-Security:
+      - max-age=15780000; includeSubDomains
+      Served-In-Seconds:
+      - '0.112'
+      Server:
+      - cloudflare-nginx
+      Cf-Ray:
+      - 3c148d370c0838ac-ATL
+    body:
+      encoding: UTF-8
+      string: '{"success":false,"errors":[{"code":10000,"message":"Authentication
+        error"}]}
+
+        '
+    http_version:
+  recorded_at: Tue, 21 Nov 2017 15:06:13 GMT
+recorded_with: VCR 3.0.3

data/spec/inputs/cloudflare-logs_spec.rb

@@ -2,6 +2,7 @@
 
 require "#{File.expand_path(File.dirname(__FILE__))}/../spec_helper"
 require 'logstash/inputs/cloudflare_logs'
+require 'pry'
 
 describe LogStash::Inputs::CloudflareLogs do
   describe '#run' do
@@ -9,24 +10,29 @@ describe LogStash::Inputs::CloudflareLogs do
       allow_any_instance_of(LogStash::Inputs::CloudflareLogs).to receive(:process_logs).and_return([])
     end
     it_behaves_like 'an interruptible input plugin' do
-      let(:config) { { 'interval' => 100, 'auth_email' => 'test@test.com', 'auth_key' => 'test', 'domain_key' => 'asdf' } }
+      let(:config) { { 'metadata_file' => "#{File.expand_path(File.dirname(__FILE__))}/../tmp/metadata.json", 'interval' => 100, 'auth_email' => 'test@test.com', 'auth_key' => 'test', 'domain_key' => 'asdf' } }
     end
   end
 
   describe '#process_logs' do
-    let(:config) { { 'interval' => 100, 'auth_email' => 'test@test.com', 'auth_key' => 'test', 'domain_key' => 'asdf' } }
+    let(:config) { { 'metadata_file' => "#{File.expand_path(File.dirname(__FILE__))}/../tmp/metadata.json", 'interval' => 100, 'auth_email' => 'test@test.com', 'auth_key' => 'test', 'domain_key' => 'asdf' } }
     let(:queue) { double(:queue, :<< => 'test') }
     before(:each) { allow_any_instance_of(CloudflareAccess).to receive(:logs).and_return([{ a: 1 }]) }
     let(:plugin) { described_class.new(config) }
     subject { plugin.process_logs(queue) }
+    before :each do
+      event = double(LogStash::Event)
+      allow(event).to receive(:timestamp=)
+      # allow(LogStash::Event).to receive(:new).and_return(event)
+    end
 
     it 'should create each log event' do
-      expect(LogStash::Event).to receive(:new).with(a: 1)
+      expect(LogStash::Event).to receive(:new).with(a: 1, 'fields.type' => 'cloudflare', 'fields.env' => nil).and_return(LogStash::Event.new)
       subject
     end
 
     it 'should create each log event' do
-      expect(LogStash::Event).to receive(:new).with(a: 1)
+      expect(LogStash::Event).to receive(:new).with(a: 1, 'fields.type' => 'cloudflare', 'fields.env' => nil).and_return(LogStash::Event.new)
       subject
     end
 

data/spec/inputs/cloudflare_access_spec.rb

@@ -2,13 +2,17 @@
 require 'json'
 require "#{File.expand_path(File.dirname(__FILE__))}/../spec_helper"
 require 'logstash/inputs/cloudflare_logs'
+require 'pry'
 
 describe CloudflareAccess do
   let(:meta_filename) { "#{File.expand_path(File.dirname(__FILE__))}/../tmp/metadata.json" }
+  let(:logger) { logger = double() }
+
   let(:cloudflare_access) do
     CloudflareAccess.new(auth_email: 'test@test.com',
                          auth_key: 'secret',
                          domain: '11',
+                         logger: logger,
                          metadata_file: meta_filename)
   end
 
@@ -101,5 +105,27 @@ describe CloudflareAccess do
     it 'should return each item as a hash' do
       expect(subject.first).to be_instance_of(Hash)
     end
+
+    describe 'connection failure' do
+      before do
+        allow(cloudflare_access).to receive(:end_time).and_return('2017-10-9T23:55:00Z')
+      end
+
+      subject do
+        VCR.use_cassette(:error) do
+          cloudflare_access.logs
+        end
+      end
+
+      it 'should raise an error' do
+        allow(logger).to receive(:error)
+        expect { subject }.to raise_error(RestClient::BadRequest)
+      end
+
+      it 'should log a descriptive error' do
+        expect(logger).to receive(:error).with("{\"success\":false,\"errors\":[{\"code\":10000,\"message\":\"Authentication error\"}]}")
+        expect { subject }.to raise_error(RestClient::BadRequest)
+      end
+    end
   end
 end

data/spec/tmp/metadata.json

@@ -0,0 +1 @@
+{"start_time":"2017-11-20 10:37:08 -0500"}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-cloudflare_logs
 version: !ruby/object:Gem::Version
-  version: 0.1.10
+  version: 0.1.11
 platform: ruby
 authors:
 - Josh Moore
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-10-30 00:00:00.000000000 Z
+date: 2017-11-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -128,6 +128,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: pry
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Write cloudflare lgos to logstash. This requires an Enterprise account
   with cloudflare
 email: joshsmoore@gmail.com
@@ -143,10 +157,12 @@ files:
 - README.md
 - lib/logstash/inputs/cloudflare_logs.rb
 - logstash-input-cloudflare-logs.gemspec
+- spec/fixtures/vcr_cassettes/error.yml
 - spec/fixtures/vcr_cassettes/logs.yml
 - spec/inputs/cloudflare-logs_spec.rb
 - spec/inputs/cloudflare_access_spec.rb
 - spec/spec_helper.rb
+- spec/tmp/metadata.json
 homepage: https://github.com/resumecompanion/logstash-input-cloudflare-logs
 licenses:
 - Apache License (2.0)
@@ -174,7 +190,9 @@ signing_key:
 specification_version: 4
 summary: write cloudflare logs to logstash
 test_files:
+- spec/fixtures/vcr_cassettes/error.yml
 - spec/fixtures/vcr_cassettes/logs.yml
 - spec/inputs/cloudflare-logs_spec.rb
 - spec/inputs/cloudflare_access_spec.rb
 - spec/spec_helper.rb
+- spec/tmp/metadata.json