logstash-input-cloudflare 0.9.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1092862dc49784086c7b33ee78df8cb6596b1b3a
+  data.tar.gz: ce657c567f7a9c2571c723cadb6f2ef598064ae5
+SHA512:
+  metadata.gz: b983c8fbb702181e88324e6a2ffd00731ec9793d379d918600a84807fc787b697f2d584f6607ebd816432a49b18794cf6ce021ac60f411d4832b51ea417d4c9f
+  data.tar.gz: 4473dff4ebf3284cf66d242e939ea6693f6f84dfca74d3524924f46420eede05b4fbfea99274347ba177148959eb5d2f02fa36de1e53edc67fb6f316b699a53b

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+## 0.1.0
+ - Initial version of the Cloudflare input plugin

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2016 Igor Serko
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/NOTICE.TXT

@@ -0,0 +1,5 @@
+Igor Serko
+Copyright 2016 Igor Serko
+
+This product includes software developed by The Apache Software
+Foundation (http://www.apache.org/).

data/README.md

@@ -0,0 +1,17 @@
+# Logstash Input Plugin for Cloudflare
+
+[![Circle CI](https://circleci.com/gh/iserko/logstash-input-cloudflare/tree/master.svg?style=svg&circle-token=78044d92053ebb2ad4ca3b45cdf3cbd271d71ac1)](https://circleci.com/gh/iserko/logstash-input-cloudflare/tree/master)
+
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
+
+## Running in isolation (for testing)
+
+```
+export CF_AUTH_EMAIL=<email>
+export CF_AUTH_KEY=<api_key>
+export CF_DOMAIN=<domain>
+make
+```
+
+Logstash will run in verbose mode, so you will see some messages coming through. In order to verify you're getting results you can open up your browser to http://&lt;IP&gt;:5601 and check Kibana.
+Value for the IP address is whatever `docker-machine ip default` says.

data/lib/logstash/inputs/cloudflare.rb

@@ -0,0 +1,265 @@
+# encoding: utf-8
+require 'date'
+require 'json'
+require 'logstash/inputs/base'
+require 'logstash/namespace'
+require 'net/http'
+require 'socket' # for Socket.gethostname
+
+class CloudflareAPIError < StandardError
+  attr_accessor :url
+  attr_accessor :errors
+  attr_accessor :success
+  attr_accessor :status_code
+
+  def initialize(url, response, content)
+    begin
+      json_data = JSON.parse(content)
+    rescue JSON::ParserError
+      json_data = {}
+    end
+    @url = url
+    @success = json_data.fetch('success', false)
+    @errors = json_data.fetch('errors', [])
+    @status_code = response.code
+  end # def initialize
+end # class CloudflareAPIError
+
+def response_body(response)
+  return '' unless response.body
+  return response.body.strip unless response.header['Content-Encoding'].eql?('gzip')
+  sio = StringIO.new(response.body)
+  gz = Zlib::GzipReader.new(sio)
+  gz.read.strip
+end # def response_body
+
+def parse_content(content)
+  return [] if content.empty?
+  lines = []
+  content.split("\n").each do |line|
+    line = line.strip
+    next if line.empty?
+    begin
+      lines << JSON.parse(line)
+    rescue JSON::ParserError
+      @logger.error("Couldn't parse JSON out of '#{line}'")
+      next
+    end
+  end
+  lines
+end # def parse_content
+
+# you can get the list of fields in the documentation provided
+# by Cloudflare
+DEFAULT_FIELDS = [
+  'timestamp', 'zoneId', 'ownerId', 'zoneName', 'rayId', 'securityLevel',
+  'client.ip', 'client.country', 'client.sslProtocol', 'client.sslCipher',
+  'client.deviceType', 'client.asNum', 'clientRequest.bytes',
+  'clientRequest.httpHost', 'clientRequest.httpMethod', 'clientRequest.uri',
+  'clientRequest.httpProtocol', 'clientRequest.userAgent',
+  'edgeResponse.status', 'edgeResponse.bytes'
+].freeze
+
+class LogStash::Inputs::Cloudflare < LogStash::Inputs::Base
+  config_name 'cloudflare'
+
+  default :codec, 'json'
+
+  config :auth_email, validate: :string, required: true
+  config :auth_key, validate: :string, required: true
+  config :domain, validate: :string, required: true
+  config :metadata_filepath,
+         validate: :string, default: '/tmp/cf_logstash_metadata.json', required: false
+  config :poll_time, validate: :number, default: 15, required: false
+  config :start_from_secs_ago, validate: :number, default: 1200, required: false
+  config :fields, validate: :array, default: DEFAULT_FIELDS, required: false
+
+  public
+
+  def register
+    @host = Socket.gethostname
+  end # def register
+
+  def read_metadata
+    # read the ray_id of the message which was parsed last
+    metadata = {}
+    if File.exist?(@metadata_filepath)
+      content = File.read(@metadata_filepath).strip
+      unless content.empty?
+        begin
+          metadata = JSON.parse(content)
+        rescue JSON::ParserError
+          metadata = {}
+        end
+      end
+    end
+    # make sure metadata contains all the keys we need
+    %w(first_ray_id last_ray_id first_timestamp
+       last_timestamp).each do |field|
+      metadata[field] = nil unless metadata.key?(field)
+    end
+    metadata['default_start_time'] = \
+      Time.now.getutc.to_i - @start_from_secs_ago
+    metadata
+  end # def read_metadata
+
+  def write_metadata(metadata)
+    File.open(@metadata_filepath, 'w') do |file|
+      file.write(JSON.generate(metadata))
+    end
+  end # def write_metadata
+
+  def cloudflare_api_call(endpoint, params, multi_line = false)
+    uri = URI("https://api.cloudflare.com/client/v4#{endpoint}")
+    uri.query = URI.encode_www_form(params)
+    @logger.info('Sending request to Cloudflare')
+    Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
+      request = Net::HTTP::Get.new(
+        uri.request_uri,
+        'Accept-Encoding' => 'gzip',
+        'X-Auth-Email' => @auth_email,
+        'X-Auth-Key' => @auth_key
+      )
+      response = http.request(request)
+      content = response_body(response)
+      if response.code != '200'
+        raise CloudflareAPIError.new(uri.to_s, response, content),
+              'Error calling Cloudflare API'
+      end
+      lines = parse_content(content)
+      return lines if multi_line
+      return lines[0]
+    end
+  end # def cloudflare_api_call
+
+  def cloudflare_zone_id(domain)
+    params = { status: 'active' }
+    response = cloudflare_api_call('/zones', params)
+    response['result'].each do |zone|
+      return zone['id'] if zone['name'] == domain
+    end
+    raise "No zone with domain #{domain} found"
+  end # def cloudflare_zone_id
+
+  def cf_params(metadata)
+    params = {}
+    # if we have ray_id, we use that as a starting point and and use
+    # timestamp + 120 seconds as end because the API doesn't support the
+    # `count` parameter
+    if metadata['last_ray_id'] && metadata['last_timestamp']
+      dt_tstamp = DateTime.strptime("#{metadata['last_timestamp']}", '%s')
+      @logger.info('last_ray_id from previous run detected: '\
+                   "#{metadata['last_ray_id']}")
+      @logger.info('last_timestamp from previous run detected: '\
+                   "#{metadata['last_timestamp']} #{dt_tstamp}")
+      params['start_id'] = metadata['last_ray_id']
+      params['end'] = metadata['last_timestamp'].to_i + 120
+      metadata['first_ray_id'] = metadata['last_ray_id']
+      metadata['first_timestamp'] = nil
+    # not supported by the API yet which is why it's commented out
+    # elsif ray_id
+    #   @logger.info("Previous ray_id detected: #{ray_id}")
+    #   params['start_id'] = ray_id
+    #   params['count'] = 100 # not supported in the API yet
+    #   metadata['first_ray_id'] = ray_id
+    #   metadata['first_timestamp'] = nil
+    elsif metadata['last_timestamp']
+      dt_tstamp = DateTime.strptime(metadata['last_timestamp'], '%s')
+      @logger.info('last_timestamp from previous run detected: '\
+                   "#{metadata['last_timestamp']} #{dt_tstamp}")
+      params['start'] = metadata['last_timestamp'].to_i
+      params['end'] = params['start'] + 120
+      metadata['first_ray_id'] = nil
+      metadata['first_timestamp'] = params['start']
+    else
+      @logger.info('last_timestamp or last_ray_id from previous run NOT set')
+      params['start'] = metadata['default_start_time']
+      params['end'] = params['start'] + 120
+      metadata['first_ray_id'] = nil
+      metadata['first_timestamp'] = params['start']
+    end
+    metadata['last_timestamp'] = nil
+    metadata['last_ray_id'] = nil
+    params
+  end # def cf_params
+
+  def cloudflare_data(zone_id, metadata)
+    @logger.info("cloudflare_data metadata: '#{metadata}'")
+    params = cf_params(metadata)
+    @logger.info("Using params #{params}")
+    begin
+      entries = cloudflare_api_call("/zones/#{zone_id}/logs/requests",
+                                    params, multi_line: true)
+    rescue CloudflareAPIError => err
+      err.errors.each do |error|
+        @logger.error("Cloudflare error code: #{error['code']}: "\
+                      "#{error['message']}")
+      end
+      entries = []
+    end
+    return entries unless entries.empty?
+    @logger.info('No entries returned from Cloudflare')
+    entries
+  end # def cloudflare_data
+
+  def fill_cloudflare_data(event, data)
+    fields.each do |field|
+      value = Hash[data]
+      field.split('.').each do |field_part|
+        value = value.fetch(field_part, {})
+      end
+      event[field.tr('.', '_')] = value
+    end
+  end # def fill_cloudflare_data
+
+  def run(queue)
+    @logger.info('Starting cloudflare run')
+    zone_id = cloudflare_zone_id(@domain)
+    @logger.info("Resolved zone ID #{zone_id} for domain #{@domain}")
+    until stop?
+      begin
+        metadata = read_metadata
+        entries = cloudflare_data(zone_id, metadata)
+        @logger.info("Received #{entries.length} events")
+        entries.each do |entry|
+          # skip the first ray_id because we already processed it
+          # in the last run
+          next if metadata['first_ray_id'] && \
+                  entry['rayId'] == metadata['first_ray_id']
+          event = LogStash::Event.new('host' => @host)
+          fill_cloudflare_data(event, entry)
+          decorate(event)
+          queue << event
+          metadata['last_ray_id'] = entry['rayId']
+          # Cloudflare provides the timestamp in nanoseconds
+          metadata['last_timestamp'] = entry['timestamp'] / 1_000_000_000
+        end
+        @logger.info(metadata)
+        if !metadata['last_timestamp'] && metadata['first_timestamp']
+          # we need to increment the timestamp by 2 minutes as we haven't
+          # received any results in the last batch ... also make sure we
+          # only do this if the end date is more than 10 minutes from the
+          # current time
+          mod_tstamp = metadata['first_timestamp'].to_i + 120
+          unless mod_tstamp > metadata['default_start_time']
+            @logger.info('Incrementing start timestamp by 120 seconds')
+            metadata['last_timestamp'] = mod_tstamp
+          end
+        else # if
+          @logger.info("Waiting #{@poll_time} seconds before requesting data"\
+                       'from Cloudflare again')
+          (@poll_time * 2).times do
+            sleep(0.5)
+          end
+        end
+        write_metadata(metadata)
+      rescue => exception
+        break if stop?
+        @logger.error(exception.class)
+        @logger.error(exception.message)
+        @logger.error(exception.backtrace.join("\n"))
+        raise(exception)
+      end
+    end # until loop
+  end # def run
+end # class LogStash::Inputs::Cloudflare

data/logstash-input-cloudflare.gemspec

@@ -0,0 +1,32 @@
+Gem::Specification.new do |s|
+  s.name = 'logstash-input-cloudflare'
+  s.version = '0.9.1'
+  s.licenses = ['Apache License (2.0)']
+  s.summary = 'This logstash input plugin fetches logs from Cloudflare using'\
+              'their API'
+  s.description = 'This gem is a logstash plugin required to be installed on'\
+                  'top of the Logstash core pipeline using $LS_HOME/bin/plugin'\
+                  ' install gemname. This gem is not a stand-alone program'
+  s.authors = ['Igor Serko']
+  s.email = 'igor.serko@gmail.com'
+  s.homepage = 'https://github.com/iserko/logstash-input-cloudflare/'
+  s.require_paths = ['lib']
+
+  # Files
+  s.files = Dir[
+    'lib/**/*', 'spec/**/*', 'vendor/**/*', '*.gemspec', '*.md', 'CONTRIBUTORS',
+    'Gemfile', 'LICENSE', 'NOTICE.TXT'
+  ]
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'input' }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash-core', '>= 2.0.0', '< 3.0.0'
+  s.add_runtime_dependency 'logstash-codec-json', '>= 2.0.0', '< 3.0.0'
+  s.add_development_dependency 'logstash-devutils', '>= 0.0.16', '< 0.1.0'
+  s.add_development_dependency 'webmock', '>= 1.24.2', '< 1.25.0'
+  s.add_development_dependency 'rubocop', '>= 0.36.0', '< 0.40.0'
+end

data/spec/inputs/cloudflare_spec.rb

@@ -0,0 +1,45 @@
+# encoding: utf-8
+require 'json'
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/inputs/cloudflare'
+require 'webmock/rspec'
+
+WebMock.disable_net_connect!(allow_localhost: true)
+
+ZONE_LIST_RESPONSE = {
+  'result' => [
+    'id' => 'zoneid',
+    'name' => 'example.com'
+  ]
+}.freeze
+
+LOGS_RESPONSE = {
+}.freeze
+
+HEADERS = {
+  'Accept' => '*/*', 'Accept-Encoding' => 'gzip',
+  'User-Agent' => 'Ruby', 'X-Auth-Email' => 'test@test.com',
+  'X-Auth-Key' => 'abcdefg'
+}.freeze
+
+RSpec.configure do |config|
+  config.before(:each) do
+    stub_request(:get, 'https://api.cloudflare.com/client/v4/zones?status=active')
+      .with(headers: HEADERS)
+      .to_return(status: 200, body: ZONE_LIST_RESPONSE.to_json, headers: {})
+    stub_request(:get, /api.cloudflare.com\/client\/v4\/zones\/zoneid\/logs\/requests.*/)
+      .with(headers: HEADERS)
+      .to_return(status: 200, body: LOGS_RESPONSE.to_json, headers: {})
+  end
+end
+
+RSpec.describe LogStash::Inputs::Cloudflare do
+  let(:config) do
+    {
+      'auth_email' => 'test@test.com',
+      'auth_key' => 'abcdefg',
+      'domain' => 'example.com'
+    }
+  end
+  it_behaves_like 'an interruptible input plugin'
+end

metadata

@@ -0,0 +1,154 @@
+--- !ruby/object:Gem::Specification
+name: logstash-input-cloudflare
+version: !ruby/object:Gem::Version
+  version: 0.9.1
+platform: ruby
+authors:
+- Igor Serko
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2016-04-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  name: logstash-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  name: logstash-codec-json
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.16
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.16
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.24.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.25.0
+  name: webmock
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.24.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.25.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.36.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.40.0
+  name: rubocop
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.36.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.40.0
+description: This gem is a logstash plugin required to be installed ontop of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+email: igor.serko@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- Gemfile
+- LICENSE
+- NOTICE.TXT
+- README.md
+- lib/logstash/inputs/cloudflare.rb
+- logstash-input-cloudflare.gemspec
+- spec/inputs/cloudflare_spec.rb
+homepage: https://github.com/iserko/logstash-input-cloudflare/
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: input
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
+specification_version: 4
+summary: This logstash input plugin fetches logs from Cloudflare usingtheir API
+test_files:
+- spec/inputs/cloudflare_spec.rb