coverband-service-client 0.0.11 → 0.0.12.rc.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a0d326e9f18a89a72e6a251e92b1a2c4d99a44288c1e695570327675017e5328
-  data.tar.gz: 83efe304d0dac5797b63ad4dea23516df89ab211a0aa5809554c9b3fce5dfa82
+  metadata.gz: f8bd1387ac9c907dceb9a215b957d77bd30021615658f95163e627cd81239564
+  data.tar.gz: 303f7fa7b317f8b97bbb50f8a4f526c6e6bbf6860ef934fd3f23966b6f0611e9
 SHA512:
-  metadata.gz: 3588eb3a0fc1c70afa925b61feaf92ee95b7169476e0008abac2ee83413a7ad0a02726876fda490089fb2008d17861f71cebd297dc78744de049fc62a82c2022
-  data.tar.gz: ee29788c3d877f9cd0079615c30da437490cc3a4c6fb74ed7a0454d44b405ef041965c648b4e3d5162f775c13d8661165c578fa57993de090d39be253f34a139
+  metadata.gz: 5be62bec964696cb38174cc21b1422335621c6745befe4b60d1ca62375ab558db60c161175bf1049f3cb824429fbeac256350c0781bc867c67b45941b644a4fa
+  data.tar.gz: 1c3dc9904513616b21bc86d051905663603ebc816683c11cad86d9a60cca187f38a3cf5bda5c3b15c49bd390688adecd7049f2f9fc9a637609d63f7861ab36b8

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    coverband-service-client (0.0.11)
+    coverband-service-client (0.0.12.rc.4)
       coverband (~> 4.2.4)
 
 GEM

data/bin/stats

@@ -0,0 +1,42 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+
+ENV['COVERBAND_ENABLE_DEV_MODE'] = 'true'
+ENV['COVERBAND_ENABLE_TEST_MODE'] = 'true'
+ENV['PROCESS_TYPE'] = 'debug'
+ENV['COVERBAND_REPORT_PERIOD'] = '30'
+ENV['COVERBAND_API_KEY'] ||= 'set this'
+ENV['COVERBAND_STATS_KEY'] ||='set this'
+
+require "pry-byebug";
+require "dogapi";
+require "net/http/persistent";
+# require "httplog";
+require 'benchmark'
+
+require_relative "../lib/coverband-service-client"
+
+# HttpLog.configure do |config|
+#   config.url_SAFElist_pattern = /coverband/
+# end
+
+data = {
+  'app/helpers/posts_helper.rb' => [1, nil]
+}
+
+collector = Coverband::Collectors::Coverage.instance
+store = Coverband.configuration.store
+
+# What is the recommended timeout against the target, from the lib
+# puts store.recommended_timeout
+
+Benchmark.bmbm do |x|
+  x.report("connection") do
+    30.times do
+      store.save_report(data)
+    end
+  end
+end
+
+puts "done"

data/coverband-service-client.gemspec

@@ -28,6 +28,14 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "rake", "~> 13.0"
+
+  # For benchmarking stats
+  # spec.add_development_dependency "pry-byebug"
+  # spec.add_development_dependency "dogapi"
+  # spec.add_development_dependency "httplog"
+  # # to benchmark persistent connections
+  # spec.add_development_dependency "net-http-persistent"
+
   spec.add_development_dependency "minitest", "~> 5.0"
   spec.add_runtime_dependency "coverband", "~> 4.2.4"
 end

data/lib/coverband-service-client.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+require 'socket'
 
 COVERBAND_ORIGINAL_START = ENV['COVERBAND_DISABLE_AUTO_START']
 ENV['COVERBAND_DISABLE_AUTO_START'] = 'true'
@@ -9,11 +10,12 @@ require 'securerandom'
 module Coverband
   COVERBAND_ENV = ENV['RACK_ENV'] || ENV['RAILS_ENV'] || (defined?(Rails) ? Rails.env : 'unknown')
   COVERBAND_SERVICE_URL = ENV['COVERBAND_URL'] || 'https://coverband.io'
-  COVERBAND_TIMEOUT = (COVERBAND_ENV == 'development') ? 5 : 1
+  COVERBAND_TIMEOUT = (COVERBAND_ENV == 'development') ? 5 : 2
   COVERBAND_ENABLE_DEV_MODE = ENV['COVERBAND_ENABLE_DEV_MODE'] || false
   COVERBAND_ENABLE_TEST_MODE = ENV['COVERBAND_ENABLE_TEST_MODE'] || false
   COVERBAND_PROCESS_TYPE = ENV['PROCESS_TYPE'] || 'unknown'
   COVERBAND_REPORT_PERIOD = (ENV['COVERBAND_REPORT_PERIOD'] || 600).to_i
+  COVERBAND_PERSISTENT_HTTP = ENV['COVERBAND_PERSISTENT_HTTP'] || false
 
   def self.service_disabled_dev_test_env?
     (COVERBAND_ENV == 'test' && !COVERBAND_ENABLE_TEST_MODE) ||
@@ -35,16 +37,39 @@ module Coverband
       #
       # NOTES:
       # * uses net/http to avoid any dependencies
-      # * currently JSON, but likely better to move to something simpler / faster
+      # * currently JSON, but likely better to move to something faster
       ###
       class Service < Base
-        attr_reader :coverband_url, :process_type, :runtime_env
+        attr_reader :coverband_url, :process_type, :runtime_env, :hostname, :pid, :stats
 
         def initialize(coverband_url, opts = {})
           super()
           @coverband_url = coverband_url
-          @process_type = opts.fetch(:process_type) { COVERBAND_PROCESS_TYPE }
+          @process_type = opts.fetch(:process_type) { $PROGRAM_NAME&.split('/')&.last || COVERBAND_PROCESS_TYPE }
+          @hostname = opts.fetch(:hostname) { ENV["DYNO"] || Socket.gethostname.force_encoding('utf-8').encode }
+          @hostname = @hostname.gsub("'",'').gsub("’",'')
           @runtime_env = opts.fetch(:runtime_env) { COVERBAND_ENV }
+          @failed_coverage_reports = []
+          initialize_stats
+        end
+
+        def initialize_stats
+          return unless ENV['COVERBAND_STATS_KEY']
+          return unless defined?(Dogapi::Client)
+
+          @stats = Dogapi::Client.new(ENV['COVERBAND_STATS_KEY'])
+          @app_name = defined?(Rails) ? Rails.application.class.module_parent.to_s : "unknown"
+        end
+
+        def report_timing(timing)
+          return unless @stats
+
+          @stats.emit_point(
+            'coverband.save.time',
+            timing,
+            host: hostname,
+            device: "coverband_#{self.class.name.split("::").last}",
+            options: {tags: [runtime_env]})
         end
 
         def logger
@@ -68,6 +93,9 @@ module Coverband
           ENV['COVERBAND_API_KEY'] || Coverband.configuration.api_key
         end
 
+        ###
+        # Fetch coverband coverage via the API
+        ###
         def coverage(local_type = nil, opts = {})
           local_type ||= opts.key?(:override_type) ? opts[:override_type] : type
           env_filter = opts.key?(:env_filter) ? opts[:env_filter] : 'production'
@@ -85,8 +113,11 @@ module Coverband
         def save_report(report)
           return if report.empty?
 
+          # We set here vs initialize to avoid setting on the primary process vs child processes
+          @pid ||= ::Process.pid
+
           # TODO: do we need dup
-          # TODO: remove timestamps, server will track first_seen
+          # TODO: remove upstream timestamps, server will track first_seen
           Thread.new do
             data = expand_report(report.dup)
             full_package = {
@@ -95,12 +126,19 @@ module Coverband
                 tags: {
                   process_type: process_type,
                   app_loading: type == Coverband::EAGER_TYPE,
-                  runtime_env: runtime_env
+                  runtime_env: runtime_env,
+                  pid: pid,
+                  hostname: hostname,
                 },
                 file_coverage: data
               }
             }
+
+            starting = Process.clock_gettime(Process::CLOCK_MONOTONIC) if @stats
             save_coverage(full_package)
+            ending = Process.clock_gettime(Process::CLOCK_MONOTONIC) if @stats
+            report_timing((ending - starting)) if @stats
+            retry_failed_reports
           end&.join
         end
 
@@ -110,19 +148,48 @@ module Coverband
 
         private
 
+        def retry_failed_reports
+          retries = []
+          @failed_coverage_reports.any? do
+            report_body = arr.pop
+            send_report_body(report_body)
+          rescue StandardError
+            retries << report_body
+          end
+          retries.each do |report_body|
+            add_retry_message(report_body)
+          end
+        end
+
+        def add_retry_message(report_body)
+          if @failed_coverage_reports.length > 5
+            logger&.info "Coverband: The errored reporting queue has reached 5. Subsequent reports will not be transmitted"
+          else
+            @failed_coverage_reports << report_body
+          end
+        end
+
         def save_coverage(data)
           if api_key.nil?
             puts "Coverband: Error: no Coverband API key was found!"
+            return
           end
 
+          coverage_body = { remote_uuid: SecureRandom.uuid, data: data }.to_json
+          send_report_body(coverage_body)
+        rescue StandardError => e
+          add_retry_message(coverage_body)
+          logger&.info "Coverband: Error while saving coverage #{e}" if Coverband.configuration.verbose || COVERBAND_ENABLE_DEV_MODE
+        end
+
+        def send_report_body(coverage_body)
           uri = URI("#{coverband_url}/api/collector")
-          req = Net::HTTP::Post.new(uri,
+          req = ::Net::HTTP::Post.new(uri,
                                     'Content-Type' => 'application/json',
                                     'Coverband-Token' => api_key)
-          req.body = { remote_uuid: SecureRandom.uuid, data: data }.to_json
-
+          req.body = coverage_body
           logger&.info "Coverband: saving (#{uri}) #{req.body}" if Coverband.configuration.verbose
-          res = Net::HTTP.start(
+          res = ::Net::HTTP.start(
             uri.hostname,
             uri.port,
             open_timeout: COVERBAND_TIMEOUT,
@@ -132,6 +199,61 @@ module Coverband
             ) do |http|
             http.request(req)
           end
+          if res.code.to_i >= 500
+            add_retry_message(coverage_body)
+          end
+        end
+      end
+
+      class PersistentService < Service
+        attr_reader :http, :stats
+
+        def initialize(coverband_url, opts = {})
+          super
+          initiate_http
+        end
+
+        def recommended_timeout
+          puts Net::HTTP::Persistent.detect_idle_timeout URI("#{coverband_url}/api/collector")
+        end
+
+        private
+
+        def initiate_http
+          @http = Net::HTTP::Persistent.new name: 'coverband_persistent'
+          @http.headers['Content-Type'] = 'application/json'
+          @http.headers['Coverband-Token'] = api_key
+          @http.open_timeout = COVERBAND_TIMEOUT
+          @http.read_timeout = COVERBAND_TIMEOUT
+          # the two below seem inconsistent in terms of how they are set
+          # leaving off for now
+          # @http.ssl_timeout = COVERBAND_TIMEOUT
+          # @http.write_timeout = COVERBAND_TIMEOUT
+          # default is 5-10 seconds but we report ever few min, heroku kills them
+          # before our reporting period... ;(
+          # @http.idle_timeout = 1000
+        end
+
+        def save_coverage(data)
+          persistent_attempts = 0
+          begin
+            if api_key.nil?
+              puts "Coverband: Error: no Coverband API key was found!"
+              return
+            end
+
+            post_uri = URI("#{coverband_url}/api/collector")
+            post = Net::HTTP::Post.new post_uri.path
+            body = { remote_uuid: SecureRandom.uuid, data: data }.to_json
+            post.body = body
+            logger&.info "Coverband: saving (#{post_uri}) #{body}" if Coverband.configuration.verbose
+            res = http.request post_uri, post
+          rescue Net::HTTP::Persistent::Error => e
+            persistent_attempts += 1
+            http.shutdown
+            initiate_http
+            retry if persistent_attempts < 2
+          end
         rescue StandardError => e
           logger&.info "Coverband: Error while saving coverage #{e}" if Coverband.configuration.verbose || COVERBAND_ENABLE_DEV_MODE
         end
@@ -213,8 +335,12 @@ end
 
 ENV['COVERBAND_DISABLE_AUTO_START'] = COVERBAND_ORIGINAL_START
 Coverband.configure do |config|
-  # Use The Test Service Adapter
-  config.store = Coverband::Adapters::Service.new(Coverband::COVERBAND_SERVICE_URL)
+  # Use the Service Adapter
+  if Coverband::COVERBAND_PERSISTENT_HTTP && defined?(Net::HTTP::Persistent)
+    config.store = Coverband::Adapters::PersistentService.new(Coverband::COVERBAND_SERVICE_URL)
+  else
+    config.store = Coverband::Adapters::Service.new(Coverband::COVERBAND_SERVICE_URL)
+  end
 
   # default to tracking views true
   config.track_views = if ENV['COVERBAND_DISABLE_VIEW_TRACKER']

data/lib/coverband/service/client/version.rb

@@ -1,7 +1,7 @@
 module Coverband
   module Service
     module Client
-      VERSION = '0.0.11'
+      VERSION = '0.0.12.rc.4'
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: coverband-service-client
 version: !ruby/object:Gem::Version
-  version: 0.0.11
+  version: 0.0.12.rc.4
 platform: ruby
 authors:
 - Dan Mayer
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-05-26 00:00:00.000000000 Z
+date: 2020-06-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -84,6 +84,7 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- bin/stats
 - changelog.md
 - coverband-service-client.gemspec
 - lib/coverband-service-client.rb
@@ -106,9 +107,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.0.3
 signing_key: