logstash-input-cloudflare 0.9.5 → 0.9.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 349c376b65705125d9abe47b5dffee4879af69dd
-  data.tar.gz: 5f989a24fe1d7fa7d250da37957c0af2d0845fb4
+  metadata.gz: ce272684c54290cb373b1b2f7632cacda4dc41c7
+  data.tar.gz: f9b2c0a8d0ec6e9ea3b97d4277992dab958c4188
 SHA512:
-  metadata.gz: 3e00297ceb83ed25366de92f91ff31e4f322c64671b02d62c36e107a9240fd24250de912c324824e91aaa005b135ba8f8bf0e5f917696e814d9a5ea142f9fb17
-  data.tar.gz: 044d256eff38371c6f13cc9244e8ec11acd97186a2b7d3510aea4b687bfe19b9df050a66bf2891ea5ab030410cdb4f05ce97d37db30ba616e51ba5b8372a601f
+  metadata.gz: 0d3a1855c767bbdaf4834b36650db2f443851340915ae814c6946699e13390a2e6ff35f48253ffeb00707a65aa4d3e04daa75d7141556978a37daae0b1bad08f
+  data.tar.gz: c28bc6bc2a081aa88248e0a941536ef5ace9bc1f1c714b9fefb643c2bb959ebabcbb458d13c4af403e48d0e3fd2b0a57b78885c59a229f133c2eebd6880f77e3

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.9.6
+- using rubocop to keep the code tidy
+- cleaned up code in order to make rubocop happy
+- worker will now catch-up if it was shut off for some time
+
 ## 0.9.5
 - moar bugfixes
 

data/lib/logstash/inputs/cloudflare.rb

@@ -71,12 +71,11 @@ class LogStash::Inputs::Cloudflare < LogStash::Inputs::Base
   config :metadata_filepath,
          validate: :string, default: '/tmp/cf_logstash_metadata.json', required: false
   config :poll_time, validate: :number, default: 15, required: false
+  config :poll_interval, validate: :number, default: 60, required: false
   config :start_from_secs_ago, validate: :number, default: 1200, required: false
   config :batch_size, validate: :number, default: 1000, required: false
   config :fields, validate: :array, default: DEFAULT_FIELDS, required: false
 
-  public
-
   def register
     @host = Socket.gethostname
   end # def register
@@ -110,9 +109,26 @@ class LogStash::Inputs::Cloudflare < LogStash::Inputs::Base
     end
   end # def write_metadata
 
-  def cloudflare_api_call(endpoint, params, multi_line = false)
+  def _build_uri(endpoint, params)
     uri = URI("https://api.cloudflare.com/client/v4#{endpoint}")
     uri.query = URI.encode_www_form(params)
+    uri
+  end
+
+  def _process_response(response, multi_line)
+    content = response_body(response)
+    if response.code != '200'
+      raise CloudflareAPIError.new(uri.to_s, response, content),
+            'Error calling Cloudflare API'
+    end
+    @logger.info("Received response from Cloudflare API (status_code: #{response.code})")
+    lines = parse_content(content)
+    return lines if multi_line
+    lines[0]
+  end # def _process_response
+
+  def cloudflare_api_call(endpoint, params, multi_line = false)
+    uri = _build_uri(endpoint, params)
     @logger.info('Sending request to Cloudflare')
     Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
       request = Net::HTTP::Get.new(
@@ -122,15 +138,7 @@ class LogStash::Inputs::Cloudflare < LogStash::Inputs::Base
         'X-Auth-Key' => @auth_key
       )
       response = http.request(request)
-      content = response_body(response)
-      if response.code != '200'
-        raise CloudflareAPIError.new(uri.to_s, response, content),
-              'Error calling Cloudflare API'
-      end
-      @logger.info("Received response from Cloudflare API (status_code: #{response.code})")
-      lines = parse_content(content)
-      return lines if multi_line
-      return lines[0]
+      return _process_response(response, multi_line)
     end
   end # def cloudflare_api_call
 
@@ -143,38 +151,53 @@ class LogStash::Inputs::Cloudflare < LogStash::Inputs::Base
     raise "No zone with domain #{domain} found"
   end # def cloudflare_zone_id
 
-  def cf_params(metadata)
+  def _from_ray_id(metadata, params)
+    # We have the previous ray ID so we use that and set the batch_size
+    # in order to fetch a certain number of events
+    @logger.info("Previous ray_id detected: #{metadata['last_ray_id']}")
+    params['start_id'] = metadata['last_ray_id']
+    params['count'] = @batch_size
+    metadata['first_ray_id'] = metadata['last_ray_id']
+    metadata['first_timestamp'] = nil
+  end # def _from_ray_id
+
+  def _from_timestamp(metadata, params)
+    # We have the last timestamp so we use it and use the poll_interval
+    dt_tstamp = DateTime.strptime(metadata['last_timestamp'], '%s')
+    @logger.info('last_timestamp from previous run detected: '\
+                 "#{metadata['last_timestamp']} #{dt_tstamp}")
+    params['start'] = metadata['last_timestamp'].to_i
+    params['end'] = params['start'] + @poll_interval
+    metadata['first_ray_id'] = nil
+    metadata['first_timestamp'] = params['start']
+  end # def _from_timestamp
+
+  def _from_neither(metadata, params)
+    @logger.info('last_timestamp or last_ray_id from previous run NOT set')
+    params['start'] = metadata['default_start_time']
+    params['end'] = params['start'] + @poll_interval
+    metadata['first_ray_id'] = nil
+    metadata['first_timestamp'] = params['start']
+  end # def _from_neither
+
+  def cloudflare_params(metadata)
     params = {}
     # if we have ray_id, we use that as a starting point
     if metadata['last_ray_id']
-      @logger.info("Previous ray_id detected: #{metadata['last_ray_id']}")
-      params['start_id'] = metadata['last_ray_id']
-      params['count'] = @batch_size
-      metadata['first_ray_id'] = metadata['last_ray_id']
-      metadata['first_timestamp'] = nil
+      _from_ray_id(metadata, params)
     elsif metadata['last_timestamp']
-      dt_tstamp = DateTime.strptime(metadata['last_timestamp'], '%s')
-      @logger.info('last_timestamp from previous run detected: '\
-                   "#{metadata['last_timestamp']} #{dt_tstamp}")
-      params['start'] = metadata['last_timestamp'].to_i
-      params['end'] = params['start'] + 120
-      metadata['first_ray_id'] = nil
-      metadata['first_timestamp'] = params['start']
+      _from_timestamp(metadata, params)
     else
-      @logger.info('last_timestamp or last_ray_id from previous run NOT set')
-      params['start'] = metadata['default_start_time']
-      params['end'] = params['start'] + 120
-      metadata['first_ray_id'] = nil
-      metadata['first_timestamp'] = params['start']
+      _from_neither(metadata, params)
     end
     metadata['last_timestamp'] = nil
     metadata['last_ray_id'] = nil
     params
-  end # def cf_params
+  end # def cloudflare_params
 
   def cloudflare_data(zone_id, metadata)
     @logger.info("cloudflare_data metadata: '#{metadata}'")
-    params = cf_params(metadata)
+    params = cloudflare_params(metadata)
     @logger.info("Using params #{params}")
     begin
       entries = cloudflare_api_call("/zones/#{zone_id}/logs/requests",
@@ -201,50 +224,74 @@ class LogStash::Inputs::Cloudflare < LogStash::Inputs::Base
     end
   end # def fill_cloudflare_data
 
+  def process_entry(queue, metadata, entry)
+    # skip the first ray_id because we already processed it
+    # in the last run
+    return if metadata['first_ray_id'] && \
+              entry['rayId'] == metadata['first_ray_id']
+    event = LogStash::Event.new('host' => @host)
+    fill_cloudflare_data(event, entry)
+    decorate(event)
+    queue << event
+    metadata['last_ray_id'] = entry['rayId']
+    # Cloudflare provides the timestamp in nanoseconds
+    metadata['last_timestamp'] = entry['timestamp'] / 1_000_000_000
+  end # def process_entry
+
+  def _sleep_time
+    @logger.info("Waiting #{@poll_time} seconds before requesting data"\
+                 'from Cloudflare again')
+    # We're staggering the poll_time so we don't block the worker for the whole 15s
+    (@poll_time * 2).times do
+      sleep(0.5)
+    end
+  end
+
+  def continue_or_sleep(metadata)
+    mod_tstamp = metadata['first_timestamp'].to_i + @poll_interval if metadata['first_timestamp']
+    if !metadata['last_timestamp'] && metadata['first_timestamp'] && \
+       mod_tstamp <= metadata['default_start_time']
+      # we need to increment the timestamp by 2 minutes as we haven't
+      # received any results in the last batch ... also make sure we
+      # only do this if the end date is more than 10 minutes from the
+      # current time
+      @logger.info("Incrementing start timestamp by #{@poll_interval} seconds")
+      metadata['last_timestamp'] = mod_tstamp
+    elsif metadata['last_timestamp'] < metadata['default_start_time']
+      # we won't need to sleep as we're trying to catch up
+      return
+    else
+      _sleep_time
+    end
+  end # def continue_or_sleep
+
+  def loop_worker(queue, zone_id)
+    metadata = read_metadata
+    entries = cloudflare_data(zone_id, metadata)
+    @logger.info("Received #{entries.length} events")
+    # if we only fetch one entry the odds are it's the one event that we asked for
+    if entries.length <= 1
+      @logger.info(
+        'Need more than 1 event to process all entries (usually because the 1 event contains the '\
+        'ray_id you asked for')
+      _sleep_time
+      return
+    end
+    entries.each do |entry|
+      process_entry(queue, metadata, entry)
+    end
+    @logger.info(metadata)
+    continue_or_sleep(metadata)
+    write_metadata(metadata)
+  end # def loop_worker
+
   def run(queue)
     @logger.info('Starting cloudflare run')
     zone_id = cloudflare_zone_id(@domain)
     @logger.info("Resolved zone ID #{zone_id} for domain #{@domain}")
     until stop?
       begin
-        metadata = read_metadata
-        entries = cloudflare_data(zone_id, metadata)
-        @logger.info("Received #{entries.length} events")
-        entries.each do |entry|
-          # skip the first ray_id because we already processed it
-          # in the last run
-          next if metadata['first_ray_id'] && \
-                  entry['rayId'] == metadata['first_ray_id']
-          event = LogStash::Event.new('host' => @host)
-          fill_cloudflare_data(event, entry)
-          decorate(event)
-          queue << event
-          metadata['last_ray_id'] = entry['rayId']
-          # Cloudflare provides the timestamp in nanoseconds
-          metadata['last_timestamp'] = entry['timestamp'] / 1_000_000_000
-        end
-        @logger.info(metadata)
-        if metadata['first_timestamp']
-          mod_tstamp = metadata['first_timestamp'].to_i + 120
-        else
-          mod_tstamp = nil
-        end
-        if !metadata['last_timestamp'] && metadata['first_timestamp'] && \
-           mod_tstamp <= metadata['default_start_time']
-          # we need to increment the timestamp by 2 minutes as we haven't
-          # received any results in the last batch ... also make sure we
-          # only do this if the end date is more than 10 minutes from the
-          # current time
-          @logger.info('Incrementing start timestamp by 120 seconds')
-          metadata['last_timestamp'] = mod_tstamp
-        else # if
-          @logger.info("Waiting #{@poll_time} seconds before requesting data"\
-                       'from Cloudflare again')
-          (@poll_time * 2).times do
-            sleep(0.5)
-          end
-        end
-        write_metadata(metadata)
+        loop_worker(queue, zone_id)
       rescue => exception
         break if stop?
         @logger.error(exception.class)

data/logstash-input-cloudflare.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-input-cloudflare'
-  s.version = '0.9.5'
+  s.version = '0.9.6'
   s.licenses = ['Apache License (2.0)']
   s.summary = 'This logstash input plugin fetches logs from Cloudflare using'\
               'their API'

data/spec/inputs/cloudflare_spec.rb

@@ -27,7 +27,9 @@ RSpec.configure do |config|
     stub_request(:get, 'https://api.cloudflare.com/client/v4/zones?status=active')
       .with(headers: HEADERS)
       .to_return(status: 200, body: ZONE_LIST_RESPONSE.to_json, headers: {})
-    stub_request(:get, /api.cloudflare.com\/client\/v4\/zones\/zoneid\/logs\/requests.*/)
+    stub_request(
+      :get,
+      %r{/api.cloudflare.com\/client\/v4\/zones\/zoneid\/logs\/requests.*/})
       .with(headers: HEADERS)
       .to_return(status: 200, body: LOGS_RESPONSE.to_json, headers: {})
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-cloudflare
 version: !ruby/object:Gem::Version
-  version: 0.9.5
+  version: 0.9.6
 platform: ruby
 authors:
 - Igor Serko