logstash-input-cloudflare 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 1092862dc49784086c7b33ee78df8cb6596b1b3a
4
+ data.tar.gz: ce657c567f7a9c2571c723cadb6f2ef598064ae5
5
+ SHA512:
6
+ metadata.gz: b983c8fbb702181e88324e6a2ffd00731ec9793d379d918600a84807fc787b697f2d584f6607ebd816432a49b18794cf6ce021ac60f411d4832b51ea417d4c9f
7
+ data.tar.gz: 4473dff4ebf3284cf66d242e939ea6693f6f84dfca74d3524924f46420eede05b4fbfea99274347ba177148959eb5d2f02fa36de1e53edc67fb6f316b699a53b
@@ -0,0 +1,2 @@
1
+ ## 0.1.0
2
+ - Initial version of the Cloudflare input plugin
data/Gemfile ADDED
@@ -0,0 +1,2 @@
1
+ source 'https://rubygems.org'
2
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,13 @@
1
+ Copyright (c) 2016 Igor Serko
2
+
3
+ Licensed under the Apache License, Version 2.0 (the "License");
4
+ you may not use this file except in compliance with the License.
5
+ You may obtain a copy of the License at
6
+
7
+ http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+ Unless required by applicable law or agreed to in writing, software
10
+ distributed under the License is distributed on an "AS IS" BASIS,
11
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ See the License for the specific language governing permissions and
13
+ limitations under the License.
@@ -0,0 +1,5 @@
1
+ Igor Serko
2
+ Copyright 2016 Igor Serko
3
+
4
+ This product includes software developed by The Apache Software
5
+ Foundation (http://www.apache.org/).
@@ -0,0 +1,17 @@
1
+ # Logstash Input Plugin for Cloudflare
2
+
3
+ [![Circle CI](https://circleci.com/gh/iserko/logstash-input-cloudflare/tree/master.svg?style=svg&circle-token=78044d92053ebb2ad4ca3b45cdf3cbd271d71ac1)](https://circleci.com/gh/iserko/logstash-input-cloudflare/tree/master)
4
+
5
+ This is a plugin for [Logstash](https://github.com/elastic/logstash).
6
+
7
+ ## Running in isolation (for testing)
8
+
9
+ ```
10
+ export CF_AUTH_EMAIL=<email>
11
+ export CF_AUTH_KEY=<api_key>
12
+ export CF_DOMAIN=<domain>
13
+ make
14
+ ```
15
+
16
+ Logstash will run in verbose mode, so you will see some messages coming through. In order to verify you're getting results you can open up your browser to http://&lt;IP&gt;:5601 and check Kibana.
17
+ Value for the IP address is whatever `docker-machine ip default` says.
@@ -0,0 +1,265 @@
1
+ # encoding: utf-8
2
+ require 'date'
3
+ require 'json'
4
+ require 'logstash/inputs/base'
5
+ require 'logstash/namespace'
6
+ require 'net/http'
7
+ require 'socket' # for Socket.gethostname
8
+
9
+ class CloudflareAPIError < StandardError
10
+ attr_accessor :url
11
+ attr_accessor :errors
12
+ attr_accessor :success
13
+ attr_accessor :status_code
14
+
15
+ def initialize(url, response, content)
16
+ begin
17
+ json_data = JSON.parse(content)
18
+ rescue JSON::ParserError
19
+ json_data = {}
20
+ end
21
+ @url = url
22
+ @success = json_data.fetch('success', false)
23
+ @errors = json_data.fetch('errors', [])
24
+ @status_code = response.code
25
+ end # def initialize
26
+ end # class CloudflareAPIError
27
+
28
+ def response_body(response)
29
+ return '' unless response.body
30
+ return response.body.strip unless response.header['Content-Encoding'].eql?('gzip')
31
+ sio = StringIO.new(response.body)
32
+ gz = Zlib::GzipReader.new(sio)
33
+ gz.read.strip
34
+ end # def response_body
35
+
36
+ def parse_content(content)
37
+ return [] if content.empty?
38
+ lines = []
39
+ content.split("\n").each do |line|
40
+ line = line.strip
41
+ next if line.empty?
42
+ begin
43
+ lines << JSON.parse(line)
44
+ rescue JSON::ParserError
45
+ @logger.error("Couldn't parse JSON out of '#{line}'")
46
+ next
47
+ end
48
+ end
49
+ lines
50
+ end # def parse_content
51
+
52
+ # you can get the list of fields in the documentation provided
53
+ # by Cloudflare
54
+ DEFAULT_FIELDS = [
55
+ 'timestamp', 'zoneId', 'ownerId', 'zoneName', 'rayId', 'securityLevel',
56
+ 'client.ip', 'client.country', 'client.sslProtocol', 'client.sslCipher',
57
+ 'client.deviceType', 'client.asNum', 'clientRequest.bytes',
58
+ 'clientRequest.httpHost', 'clientRequest.httpMethod', 'clientRequest.uri',
59
+ 'clientRequest.httpProtocol', 'clientRequest.userAgent',
60
+ 'edgeResponse.status', 'edgeResponse.bytes'
61
+ ].freeze
62
+
63
+ class LogStash::Inputs::Cloudflare < LogStash::Inputs::Base
64
+ config_name 'cloudflare'
65
+
66
+ default :codec, 'json'
67
+
68
+ config :auth_email, validate: :string, required: true
69
+ config :auth_key, validate: :string, required: true
70
+ config :domain, validate: :string, required: true
71
+ config :metadata_filepath,
72
+ validate: :string, default: '/tmp/cf_logstash_metadata.json', required: false
73
+ config :poll_time, validate: :number, default: 15, required: false
74
+ config :start_from_secs_ago, validate: :number, default: 1200, required: false
75
+ config :fields, validate: :array, default: DEFAULT_FIELDS, required: false
76
+
77
+ public
78
+
79
+ def register
80
+ @host = Socket.gethostname
81
+ end # def register
82
+
83
+ def read_metadata
84
+ # read the ray_id of the message which was parsed last
85
+ metadata = {}
86
+ if File.exist?(@metadata_filepath)
87
+ content = File.read(@metadata_filepath).strip
88
+ unless content.empty?
89
+ begin
90
+ metadata = JSON.parse(content)
91
+ rescue JSON::ParserError
92
+ metadata = {}
93
+ end
94
+ end
95
+ end
96
+ # make sure metadata contains all the keys we need
97
+ %w(first_ray_id last_ray_id first_timestamp
98
+ last_timestamp).each do |field|
99
+ metadata[field] = nil unless metadata.key?(field)
100
+ end
101
+ metadata['default_start_time'] = \
102
+ Time.now.getutc.to_i - @start_from_secs_ago
103
+ metadata
104
+ end # def read_metadata
105
+
106
+ def write_metadata(metadata)
107
+ File.open(@metadata_filepath, 'w') do |file|
108
+ file.write(JSON.generate(metadata))
109
+ end
110
+ end # def write_metadata
111
+
112
+ def cloudflare_api_call(endpoint, params, multi_line = false)
113
+ uri = URI("https://api.cloudflare.com/client/v4#{endpoint}")
114
+ uri.query = URI.encode_www_form(params)
115
+ @logger.info('Sending request to Cloudflare')
116
+ Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
117
+ request = Net::HTTP::Get.new(
118
+ uri.request_uri,
119
+ 'Accept-Encoding' => 'gzip',
120
+ 'X-Auth-Email' => @auth_email,
121
+ 'X-Auth-Key' => @auth_key
122
+ )
123
+ response = http.request(request)
124
+ content = response_body(response)
125
+ if response.code != '200'
126
+ raise CloudflareAPIError.new(uri.to_s, response, content),
127
+ 'Error calling Cloudflare API'
128
+ end
129
+ lines = parse_content(content)
130
+ return lines if multi_line
131
+ return lines[0]
132
+ end
133
+ end # def cloudflare_api_call
134
+
135
+ def cloudflare_zone_id(domain)
136
+ params = { status: 'active' }
137
+ response = cloudflare_api_call('/zones', params)
138
+ response['result'].each do |zone|
139
+ return zone['id'] if zone['name'] == domain
140
+ end
141
+ raise "No zone with domain #{domain} found"
142
+ end # def cloudflare_zone_id
143
+
144
+ def cf_params(metadata)
145
+ params = {}
146
+ # if we have ray_id, we use that as a starting point and and use
147
+ # timestamp + 120 seconds as end because the API doesn't support the
148
+ # `count` parameter
149
+ if metadata['last_ray_id'] && metadata['last_timestamp']
150
+ dt_tstamp = DateTime.strptime("#{metadata['last_timestamp']}", '%s')
151
+ @logger.info('last_ray_id from previous run detected: '\
152
+ "#{metadata['last_ray_id']}")
153
+ @logger.info('last_timestamp from previous run detected: '\
154
+ "#{metadata['last_timestamp']} #{dt_tstamp}")
155
+ params['start_id'] = metadata['last_ray_id']
156
+ params['end'] = metadata['last_timestamp'].to_i + 120
157
+ metadata['first_ray_id'] = metadata['last_ray_id']
158
+ metadata['first_timestamp'] = nil
159
+ # not supported by the API yet which is why it's commented out
160
+ # elsif ray_id
161
+ # @logger.info("Previous ray_id detected: #{ray_id}")
162
+ # params['start_id'] = ray_id
163
+ # params['count'] = 100 # not supported in the API yet
164
+ # metadata['first_ray_id'] = ray_id
165
+ # metadata['first_timestamp'] = nil
166
+ elsif metadata['last_timestamp']
167
+ dt_tstamp = DateTime.strptime(metadata['last_timestamp'], '%s')
168
+ @logger.info('last_timestamp from previous run detected: '\
169
+ "#{metadata['last_timestamp']} #{dt_tstamp}")
170
+ params['start'] = metadata['last_timestamp'].to_i
171
+ params['end'] = params['start'] + 120
172
+ metadata['first_ray_id'] = nil
173
+ metadata['first_timestamp'] = params['start']
174
+ else
175
+ @logger.info('last_timestamp or last_ray_id from previous run NOT set')
176
+ params['start'] = metadata['default_start_time']
177
+ params['end'] = params['start'] + 120
178
+ metadata['first_ray_id'] = nil
179
+ metadata['first_timestamp'] = params['start']
180
+ end
181
+ metadata['last_timestamp'] = nil
182
+ metadata['last_ray_id'] = nil
183
+ params
184
+ end # def cf_params
185
+
186
+ def cloudflare_data(zone_id, metadata)
187
+ @logger.info("cloudflare_data metadata: '#{metadata}'")
188
+ params = cf_params(metadata)
189
+ @logger.info("Using params #{params}")
190
+ begin
191
+ entries = cloudflare_api_call("/zones/#{zone_id}/logs/requests",
192
+ params, multi_line: true)
193
+ rescue CloudflareAPIError => err
194
+ err.errors.each do |error|
195
+ @logger.error("Cloudflare error code: #{error['code']}: "\
196
+ "#{error['message']}")
197
+ end
198
+ entries = []
199
+ end
200
+ return entries unless entries.empty?
201
+ @logger.info('No entries returned from Cloudflare')
202
+ entries
203
+ end # def cloudflare_data
204
+
205
+ def fill_cloudflare_data(event, data)
206
+ fields.each do |field|
207
+ value = Hash[data]
208
+ field.split('.').each do |field_part|
209
+ value = value.fetch(field_part, {})
210
+ end
211
+ event[field.tr('.', '_')] = value
212
+ end
213
+ end # def fill_cloudflare_data
214
+
215
+ def run(queue)
216
+ @logger.info('Starting cloudflare run')
217
+ zone_id = cloudflare_zone_id(@domain)
218
+ @logger.info("Resolved zone ID #{zone_id} for domain #{@domain}")
219
+ until stop?
220
+ begin
221
+ metadata = read_metadata
222
+ entries = cloudflare_data(zone_id, metadata)
223
+ @logger.info("Received #{entries.length} events")
224
+ entries.each do |entry|
225
+ # skip the first ray_id because we already processed it
226
+ # in the last run
227
+ next if metadata['first_ray_id'] && \
228
+ entry['rayId'] == metadata['first_ray_id']
229
+ event = LogStash::Event.new('host' => @host)
230
+ fill_cloudflare_data(event, entry)
231
+ decorate(event)
232
+ queue << event
233
+ metadata['last_ray_id'] = entry['rayId']
234
+ # Cloudflare provides the timestamp in nanoseconds
235
+ metadata['last_timestamp'] = entry['timestamp'] / 1_000_000_000
236
+ end
237
+ @logger.info(metadata)
238
+ if !metadata['last_timestamp'] && metadata['first_timestamp']
239
+ # we need to increment the timestamp by 2 minutes as we haven't
240
+ # received any results in the last batch ... also make sure we
241
+ # only do this if the end date is more than 10 minutes from the
242
+ # current time
243
+ mod_tstamp = metadata['first_timestamp'].to_i + 120
244
+ unless mod_tstamp > metadata['default_start_time']
245
+ @logger.info('Incrementing start timestamp by 120 seconds')
246
+ metadata['last_timestamp'] = mod_tstamp
247
+ end
248
+ else # if
249
+ @logger.info("Waiting #{@poll_time} seconds before requesting data"\
250
+ 'from Cloudflare again')
251
+ (@poll_time * 2).times do
252
+ sleep(0.5)
253
+ end
254
+ end
255
+ write_metadata(metadata)
256
+ rescue => exception
257
+ break if stop?
258
+ @logger.error(exception.class)
259
+ @logger.error(exception.message)
260
+ @logger.error(exception.backtrace.join("\n"))
261
+ raise(exception)
262
+ end
263
+ end # until loop
264
+ end # def run
265
+ end # class LogStash::Inputs::Cloudflare
@@ -0,0 +1,32 @@
1
+ Gem::Specification.new do |s|
2
+ s.name = 'logstash-input-cloudflare'
3
+ s.version = '0.9.1'
4
+ s.licenses = ['Apache License (2.0)']
5
+ s.summary = 'This logstash input plugin fetches logs from Cloudflare using'\
6
+ 'their API'
7
+ s.description = 'This gem is a logstash plugin required to be installed on'\
8
+ 'top of the Logstash core pipeline using $LS_HOME/bin/plugin'\
9
+ ' install gemname. This gem is not a stand-alone program'
10
+ s.authors = ['Igor Serko']
11
+ s.email = 'igor.serko@gmail.com'
12
+ s.homepage = 'https://github.com/iserko/logstash-input-cloudflare/'
13
+ s.require_paths = ['lib']
14
+
15
+ # Files
16
+ s.files = Dir[
17
+ 'lib/**/*', 'spec/**/*', 'vendor/**/*', '*.gemspec', '*.md', 'CONTRIBUTORS',
18
+ 'Gemfile', 'LICENSE', 'NOTICE.TXT'
19
+ ]
20
+ # Tests
21
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
22
+
23
+ # Special flag to let us know this is actually a logstash plugin
24
+ s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'input' }
25
+
26
+ # Gem dependencies
27
+ s.add_runtime_dependency 'logstash-core', '>= 2.0.0', '< 3.0.0'
28
+ s.add_runtime_dependency 'logstash-codec-json', '>= 2.0.0', '< 3.0.0'
29
+ s.add_development_dependency 'logstash-devutils', '>= 0.0.16', '< 0.1.0'
30
+ s.add_development_dependency 'webmock', '>= 1.24.2', '< 1.25.0'
31
+ s.add_development_dependency 'rubocop', '>= 0.36.0', '< 0.40.0'
32
+ end
@@ -0,0 +1,45 @@
1
+ # encoding: utf-8
2
+ require 'json'
3
+ require 'logstash/devutils/rspec/spec_helper'
4
+ require 'logstash/inputs/cloudflare'
5
+ require 'webmock/rspec'
6
+
7
+ WebMock.disable_net_connect!(allow_localhost: true)
8
+
9
+ ZONE_LIST_RESPONSE = {
10
+ 'result' => [
11
+ 'id' => 'zoneid',
12
+ 'name' => 'example.com'
13
+ ]
14
+ }.freeze
15
+
16
+ LOGS_RESPONSE = {
17
+ }.freeze
18
+
19
+ HEADERS = {
20
+ 'Accept' => '*/*', 'Accept-Encoding' => 'gzip',
21
+ 'User-Agent' => 'Ruby', 'X-Auth-Email' => 'test@test.com',
22
+ 'X-Auth-Key' => 'abcdefg'
23
+ }.freeze
24
+
25
+ RSpec.configure do |config|
26
+ config.before(:each) do
27
+ stub_request(:get, 'https://api.cloudflare.com/client/v4/zones?status=active')
28
+ .with(headers: HEADERS)
29
+ .to_return(status: 200, body: ZONE_LIST_RESPONSE.to_json, headers: {})
30
+ stub_request(:get, /api.cloudflare.com\/client\/v4\/zones\/zoneid\/logs\/requests.*/)
31
+ .with(headers: HEADERS)
32
+ .to_return(status: 200, body: LOGS_RESPONSE.to_json, headers: {})
33
+ end
34
+ end
35
+
36
+ RSpec.describe LogStash::Inputs::Cloudflare do
37
+ let(:config) do
38
+ {
39
+ 'auth_email' => 'test@test.com',
40
+ 'auth_key' => 'abcdefg',
41
+ 'domain' => 'example.com'
42
+ }
43
+ end
44
+ it_behaves_like 'an interruptible input plugin'
45
+ end
metadata ADDED
@@ -0,0 +1,154 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: logstash-input-cloudflare
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.9.1
5
+ platform: ruby
6
+ authors:
7
+ - Igor Serko
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2016-04-11 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - ">="
17
+ - !ruby/object:Gem::Version
18
+ version: 2.0.0
19
+ - - "<"
20
+ - !ruby/object:Gem::Version
21
+ version: 3.0.0
22
+ name: logstash-core
23
+ prerelease: false
24
+ type: :runtime
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - ">="
28
+ - !ruby/object:Gem::Version
29
+ version: 2.0.0
30
+ - - "<"
31
+ - !ruby/object:Gem::Version
32
+ version: 3.0.0
33
+ - !ruby/object:Gem::Dependency
34
+ requirement: !ruby/object:Gem::Requirement
35
+ requirements:
36
+ - - ">="
37
+ - !ruby/object:Gem::Version
38
+ version: 2.0.0
39
+ - - "<"
40
+ - !ruby/object:Gem::Version
41
+ version: 3.0.0
42
+ name: logstash-codec-json
43
+ prerelease: false
44
+ type: :runtime
45
+ version_requirements: !ruby/object:Gem::Requirement
46
+ requirements:
47
+ - - ">="
48
+ - !ruby/object:Gem::Version
49
+ version: 2.0.0
50
+ - - "<"
51
+ - !ruby/object:Gem::Version
52
+ version: 3.0.0
53
+ - !ruby/object:Gem::Dependency
54
+ requirement: !ruby/object:Gem::Requirement
55
+ requirements:
56
+ - - ">="
57
+ - !ruby/object:Gem::Version
58
+ version: 0.0.16
59
+ - - "<"
60
+ - !ruby/object:Gem::Version
61
+ version: 0.1.0
62
+ name: logstash-devutils
63
+ prerelease: false
64
+ type: :development
65
+ version_requirements: !ruby/object:Gem::Requirement
66
+ requirements:
67
+ - - ">="
68
+ - !ruby/object:Gem::Version
69
+ version: 0.0.16
70
+ - - "<"
71
+ - !ruby/object:Gem::Version
72
+ version: 0.1.0
73
+ - !ruby/object:Gem::Dependency
74
+ requirement: !ruby/object:Gem::Requirement
75
+ requirements:
76
+ - - ">="
77
+ - !ruby/object:Gem::Version
78
+ version: 1.24.2
79
+ - - "<"
80
+ - !ruby/object:Gem::Version
81
+ version: 1.25.0
82
+ name: webmock
83
+ prerelease: false
84
+ type: :development
85
+ version_requirements: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ version: 1.24.2
90
+ - - "<"
91
+ - !ruby/object:Gem::Version
92
+ version: 1.25.0
93
+ - !ruby/object:Gem::Dependency
94
+ requirement: !ruby/object:Gem::Requirement
95
+ requirements:
96
+ - - ">="
97
+ - !ruby/object:Gem::Version
98
+ version: 0.36.0
99
+ - - "<"
100
+ - !ruby/object:Gem::Version
101
+ version: 0.40.0
102
+ name: rubocop
103
+ prerelease: false
104
+ type: :development
105
+ version_requirements: !ruby/object:Gem::Requirement
106
+ requirements:
107
+ - - ">="
108
+ - !ruby/object:Gem::Version
109
+ version: 0.36.0
110
+ - - "<"
111
+ - !ruby/object:Gem::Version
112
+ version: 0.40.0
113
+ description: This gem is a logstash plugin required to be installed ontop of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
114
+ email: igor.serko@gmail.com
115
+ executables: []
116
+ extensions: []
117
+ extra_rdoc_files: []
118
+ files:
119
+ - CHANGELOG.md
120
+ - Gemfile
121
+ - LICENSE
122
+ - NOTICE.TXT
123
+ - README.md
124
+ - lib/logstash/inputs/cloudflare.rb
125
+ - logstash-input-cloudflare.gemspec
126
+ - spec/inputs/cloudflare_spec.rb
127
+ homepage: https://github.com/iserko/logstash-input-cloudflare/
128
+ licenses:
129
+ - Apache License (2.0)
130
+ metadata:
131
+ logstash_plugin: 'true'
132
+ logstash_group: input
133
+ post_install_message:
134
+ rdoc_options: []
135
+ require_paths:
136
+ - lib
137
+ required_ruby_version: !ruby/object:Gem::Requirement
138
+ requirements:
139
+ - - ">="
140
+ - !ruby/object:Gem::Version
141
+ version: '0'
142
+ required_rubygems_version: !ruby/object:Gem::Requirement
143
+ requirements:
144
+ - - ">="
145
+ - !ruby/object:Gem::Version
146
+ version: '0'
147
+ requirements: []
148
+ rubyforge_project:
149
+ rubygems_version: 2.4.8
150
+ signing_key:
151
+ specification_version: 4
152
+ summary: This logstash input plugin fetches logs from Cloudflare usingtheir API
153
+ test_files:
154
+ - spec/inputs/cloudflare_spec.rb