coverband 4.2.3 → 4.2.4.rc.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0f5ee08a6ec38b97538430db828645f0aab7736c8c51946fea8c44f871c6a4a9
-  data.tar.gz: 733933d77d75e9e50c09a282f87da2b7cd3b09db9e24679b0267f384ba034c0f
+  metadata.gz: 4bab24810b493437a363e010a7a673170bd41f7d4f2e513c337f1955952a4598
+  data.tar.gz: fbc2991ab6802666af9461ce3da1c03c07b1cc9e39a84268bc0b0854df45eb67
 SHA512:
-  metadata.gz: 68117ba4c4d9dbc9370187e2a31f3b916a0368cb46b915eb7f3e189f1c7bb4f5699d972e176d9a138d999ca59a50e2817658767ccb803f2726dd3fd3f8ff31b4
-  data.tar.gz: bffb9453d31c0c93ec6706a2c426a6835b713dfc1000c50e030c252a25bd7abdedd95f41622dfc309420aa27276b19afb44cec81a5e2ec20abc35b1768f71541
+  metadata.gz: 146c61b557cee78a7bbf9db9481860c8a8833299d580b9f2d56ab77dbc3780f8f2817fcc96a2200c90e37ba0137718c232c30c762e1482f449e325db1a81871d
+  data.tar.gz: 5c0b7a3d14539f9d65c33c88ecc9c8c42bc17a02c6c10c03e6aadb3433384b75aff6733ab4957b207b905dbd20369429ee6269922f2710856bfa428a4e998abd

data/.rubocop.yml

@@ -1,6 +1,7 @@
 AllCops:
   TargetRubyVersion: 2.4
   Exclude:
+    - vendor/bundle/**/*
     - docs/**/*
     - test/rails4_dummy/**/*
     - test/rails5_dummy/**/*
@@ -31,6 +32,8 @@ Metrics/ParameterLists:
   Max: 10
 Naming/AccessorMethodName:
   Enabled: false
+Naming/RescuedExceptionsVariableName:
+  Enabled: false
 Naming/PredicateName:
   Enabled: true
 Style/TernaryParentheses:

data/.travis.yml

@@ -4,12 +4,18 @@ rvm:
   - "2.4"
   - "2.5"
   - "2.6.1"
+cache:
+  bundler: true
+  directories:
+  - $HOME/lua51
 gemfile:
   - Gemfile
   - Gemfile.rails4
 services:
   - redis-server
 script:
+  - sudo ./lua/install.sh 
+  - $HOME/lua51/bin/busted lua/test/*
   - bundle exec rake rubocop
   - COVERBAND_HASH_REDIS_STORE=t bundle exec rake
   - COVERBAND_HASH_REDIS_STORE=t bundle exec rake forked_tests

data/README.md

@@ -169,9 +169,29 @@ Coverband.configure do |config|
   # and is safe to use if one is investigating issues in production, but it will slightly
   # hit perf.
   config.verbose = false
+
+  # default false. button at the top of the web interface which clears all data
+  config.web_enable_clear = true
+
+  # default false. Experimental support for tracking view layer tracking.
+  # Does not track line-level usage, only indicates if an entire file
+  # is used or not.
+  config.track_views = true
 end
 ```
 
+### Working with environment variables
+
+Do you use figaro, mc-settings, dotenv or something else to inject environment variables into your app? If so ensure you have that done BEFORE coverband is required. 
+
+For example if you use dotenv, you need to do this, see https://github.com/bkeepers/dotenv#note-on-load-order
+
+```
+gem 'dotenv-rails', require: 'dotenv/rails-now'
+gem 'coverband'
+gem 'other-gem-that-requires-env-variables'
+```
+
 ### Ignoring Files
 
 Sometimes you have files that are known to be valuable perhaps in other environments or something that is just run very infrequently. Opposed to having to mentally filter them out of the report, you can just have them ignored in the Coverband reporting by using `config.ignore` as shown below. Ignore takes a string but can also match with regex rules see how below ignores all rake tasks as an example.
@@ -363,8 +383,9 @@ If you submit a change please make sure the tests and benchmarks are passing.
 
 ### Known Issues
 
-- **total fail** on front end code, because of the precompiled template step basically coverage doesn't work well for `erb`, `slim`, and the like.
+- **total fail** on front end code, for line for line coverage, because of the precompiled template step basically coverage doesn't work well for `erb`, `slim`, and the like.
   - related it will try to report something, but the line numbers reported for `ERB` files are often off and aren't considered useful. I recommend filtering out .erb using the `config.ignore` option. The default configuration excludes these files
+  - **NOTE:** We now have file level coverage for view files, but don't support line level detail
 - coverage doesn't show for Rails `config/application.rb` or `config/boot.rb` as they get loaded when loading the Rake environment prior to starting the `Coverage` library. See [reload files section](#forcing-coverband-to-track-coverage-on-files-loaded-during-boot-safe_reload_files).
 
 ### Debugging Redis Store

data/changes.md

@@ -76,16 +76,16 @@ Feature Ideas:
 
 # Alpha / Beta / Release Candidates
 
-### Coverband 4.2.2
-
-- new experimental hash redis store for high volume collection
-(hundreds of clients), thanks @kbaum
-- view_tracker supports tracking view layer files like `.html.erb`
-- documentation improvements, thanks @brossetti1, @jjb, @kbaum
-- we now have discordapp for discussions, thanks @kbaum,
-https://discordapp.com/channels/609509533999562753/609509533999562756
-- perf fix on Rails initialization, thanks @skangg
-- simplified logging
+### Coverband 4.2.4
+
+- fixes related to startup without Redis, skipping Coverband on common rake tasks (assets:precompile), etc
+- avoid calls to redis if coverage not updated
+- split out lua work and added testing around lua scripts use for the hash redis store
+- improvements to hash redis store resulting in 25% faster perf
+- avoid duplicate hash calculations
+- debug data now includes md5_hashes
+- added support to download coverage and view data in JSON format
+- documentation about working with environment variables
 
 # Released
 

data/lib/coverband.rb

@@ -99,12 +99,17 @@ module Coverband
     Coverband::Collectors::Coverage.instance
   end
   unless ENV['COVERBAND_DISABLE_AUTO_START']
-    # Coverband should be setup as early as possible
-    # to capture usage of things loaded by initializers or other Rails engines
-    configure
-    start
-    require 'coverband/utils/railtie' if defined? ::Rails::Railtie
-    require 'coverband/integrations/resque' if defined? ::Resque
-    require 'coverband/integrations/bundler' if defined? ::Bundler
+    begin
+      # Coverband should be setup as early as possible
+      # to capture usage of things loaded by initializers or other Rails engines
+      configure
+      start
+      require 'coverband/utils/railtie' if defined? ::Rails::Railtie
+      require 'coverband/integrations/resque' if defined? ::Resque
+      require 'coverband/integrations/bundler' if defined? ::Bundler
+    rescue Redis::CannotConnectError => error
+      Coverband.configuration.logger.info "Redis is not available (#{error}), Coverband not configured"
+      Coverband.configuration.logger.info 'If this is a setup task like assets:precompile feel free to ignore'
+    end
   end
 end

data/lib/coverband/adapters/base.rb

@@ -40,7 +40,7 @@ module Coverband
       end
 
       def size_in_mib
-        format('%.2f', (size.to_f / 2**20))
+        format('%<size>.2f', size: (size.to_f / 2**20))
       end
 
       def save_report(_report)

data/lib/coverband/adapters/hash_redis_store.rb

@@ -1,11 +1,13 @@
 # frozen_string_literal: true
 
+require 'securerandom'
+
 module Coverband
   module Adapters
     class HashRedisStore < Base
       FILE_KEY = 'file'
       FILE_LENGTH_KEY = 'file_length'
-      META_DATA_KEYS = [DATA_KEY, FIRST_UPDATED_KEY, LAST_UPDATED_KEY].freeze
+      META_DATA_KEYS = [DATA_KEY, FIRST_UPDATED_KEY, LAST_UPDATED_KEY, FILE_HASH].freeze
       ###
       # This key isn't related to the coverband version, but to the interal format
       # used to store data to redis. It is changed only when breaking changes to our
@@ -13,6 +15,8 @@ module Coverband
       ###
       REDIS_STORAGE_FORMAT_VERSION = 'coverband_hash_3_3'
 
+      JSON_PAYLOAD_EXPIRATION = 5 * 60
+
       attr_reader :redis_namespace
 
       def initialize(redis, opts = {})
@@ -22,7 +26,7 @@ module Coverband
         @redis = redis
         raise 'HashRedisStore requires redis >= 2.6.0' unless supported?
 
-        @ttl = opts[:ttl] || -1
+        @ttl = opts[:ttl]
         @relative_file_converter = opts[:relative_file_converter] || Utils::RelativeFileConverter
       end
 
@@ -53,25 +57,27 @@ module Coverband
       def save_report(report)
         report_time = Time.now.to_i
         updated_time = type == Coverband::EAGER_TYPE ? nil : report_time
-        script_id = hash_incr_script
-        @redis.pipelined do
-          keys = report.map do |file, data|
-            relative_file = @relative_file_converter.convert(file)
-            file_hash = file_hash(relative_file)
-            key = key(relative_file, file_hash: file_hash)
-            script_input = save_report_script_input(
-              key: key,
-              file: relative_file,
-              file_hash: file_hash,
-              data: data,
-              report_time: report_time,
-              updated_time: updated_time
-            )
-            @redis.evalsha(script_id, script_input[:keys], script_input[:args])
-            key
-          end
-          @redis.sadd(files_key, keys) if keys.any?
+        keys = []
+        files_data = report.map do |file, data|
+          relative_file = @relative_file_converter.convert(file)
+          file_hash = file_hash(relative_file)
+          key = key(relative_file, file_hash: file_hash)
+          keys << key
+          script_input(
+            key: key,
+            file: relative_file,
+            file_hash: file_hash,
+            data: data,
+            report_time: report_time,
+            updated_time: updated_time
+          )
         end
+        return unless keys.any?
+
+        arguments_key = [@redis_namespace, SecureRandom.uuid].compact.join('.')
+        @redis.set(arguments_key, { ttl: @ttl, files_data: files_data }.to_json, ex: JSON_PAYLOAD_EXPIRATION)
+        @redis.evalsha(hash_incr_script, [arguments_key])
+        @redis.sadd(files_key, keys)
       end
 
       def coverage(local_type = nil)
@@ -116,38 +122,33 @@ module Coverband
         end
       end
 
-      def save_report_script_input(key:, file:, file_hash:, data:, report_time:, updated_time:)
-        data.each_with_index
-            .each_with_object(keys: [key], args: [report_time, updated_time, file, file_hash, @ttl, data.length]) do |(coverage, index), hash|
-          if coverage
-            hash[:keys] << index
-            hash[:args] << coverage
-          end
+      def script_input(key:, file:, file_hash:, data:, report_time:, updated_time:)
+        coverage_data = data.each_with_index.each_with_object({}) do |(coverage, index), hash|
+          hash[index] = coverage if coverage
         end
+        meta = {
+          first_updated_at: report_time,
+          file: file,
+          file_hash: file_hash,
+          file_length: data.length,
+          hash_key: key
+        }
+        meta.merge!(last_updated_at: updated_time) if updated_time
+        {
+          hash_key: key,
+          meta: meta,
+          coverage: coverage_data
+        }
       end
 
       def hash_incr_script
-        @hash_incr_script ||= @redis.script(:load, <<~LUA)
-          local first_updated_at = table.remove(ARGV, 1)
-          local last_updated_at = table.remove(ARGV, 1)
-          local file = table.remove(ARGV, 1)
-          local file_hash = table.remove(ARGV, 1)
-          local ttl = table.remove(ARGV, 1)
-          local file_length = table.remove(ARGV, 1)
-          local hash_key = table.remove(KEYS, 1)
-          redis.call('HMSET', hash_key, '#{LAST_UPDATED_KEY}', last_updated_at, '#{FILE_KEY}', file, '#{FILE_HASH}', file_hash, '#{FILE_LENGTH_KEY}', file_length)
-          redis.call('HSETNX', hash_key, '#{FIRST_UPDATED_KEY}', first_updated_at)
-          for i, key in ipairs(KEYS) do
-            if ARGV[i] == '-1' then
-              redis.call("HSET", hash_key, key, ARGV[i])
-            else
-              redis.call("HINCRBY", hash_key, key, ARGV[i])
-            end
-          end
-          if ttl ~= '-1' then
-            redis.call("EXPIRE", hash_key, ttl)
-          end
-        LUA
+        @hash_incr_script ||= @redis.script(:load, lua_script_content)
+      end
+
+      def lua_script_content
+        File.read(File.join(
+                    File.dirname(__FILE__), '../../../lua/lib/persist-coverage.lua'
+                  ))
       end
 
       def values_from_redis(local_type, files)

data/lib/coverband/collectors/view_tracker.rb

@@ -29,10 +29,10 @@ module Coverband
 
         @roots = options.fetch(:roots) { Coverband.configuration.all_root_patterns }
         @roots = @roots.split(',') if @roots.is_a?(String)
+        @one_time_timestamp = false
 
         @logged_views = []
         @views_to_record = []
-        redis_store.set(tracker_time_key, Time.now.to_i) unless redis_store.exists(tracker_time_key)
       end
 
       ###
@@ -79,6 +79,13 @@ module Coverband
         all_views.reject { |view| view.match(/\/layouts\//) && recently_used_views.any? { |used_view| view.include?(used_view) } }
       end
 
+      def as_json
+        {
+          unused_views: unused_views,
+          used_views: used_views
+        }.to_json
+      end
+
       def tracking_since
         if (tracking_time = redis_store.get(tracker_time_key))
           Time.at(tracking_time.to_i).iso8601
@@ -100,11 +107,9 @@ module Coverband
         logged_views.delete(filename)
       end
 
-      def self.supported_version?
-        defined?(Rails) && defined?(Rails::VERSION) && Rails::VERSION::STRING.split('.').first.to_i >= 4
-      end
-
       def report_views_tracked
+        redis_store.set(tracker_time_key, Time.now.to_i) unless @one_time_timestamp || redis_store.exists(tracker_time_key)
+        @one_time_timestamp = true
         reported_time = Time.now.to_i
         views_to_record.each do |file|
           redis_store.hset(tracker_key, file, reported_time)
@@ -116,6 +121,10 @@ module Coverband
         logger&.error "Coverband: view_tracker failed to store, error #{e.class.name}"
       end
 
+      def self.supported_version?
+        defined?(Rails) && defined?(Rails::VERSION) && Rails::VERSION::STRING.split('.').first.to_i >= 4
+      end
+
       protected
 
       def newly_seen_file?(file)

data/lib/coverband/configuration.rb

@@ -23,7 +23,16 @@ module Coverband
     IGNORE_TASKS = ['coverband:clear',
                     'coverband:coverage',
                     'coverband:coverage_server',
-                    'coverband:migrate']
+                    'coverband:migrate',
+                    'assets:precompile',
+                    'db:version',
+                    'db:create',
+                    'db:drop',
+                    'db:seed',
+                    'db:setup',
+                    'db:structure:dump',
+                    'db:structure:load',
+                    'db:version']
 
     # Heroku when building assets runs code from a dynamic directory
     # /tmp was added to avoid coverage from /tmp/build directories during

data/lib/coverband/reporters/web.rb

@@ -51,6 +51,8 @@ module Coverband
             @static.call(env)
           when %r{\/settings}
             [200, { 'Content-Type' => 'text/html' }, [settings]]
+          when %r{\/view_tracker_data}
+            [200, { 'Content-Type' => 'text/json' }, [view_tracker_data]]
           when %r{\/view_tracker}
             [200, { 'Content-Type' => 'text/html' }, [view_tracker]]
           when %r{\/debug_data}
@@ -87,6 +89,10 @@ module Coverband
                                             base_path: base_path).format_view_tracker!
       end
 
+      def view_tracker_data
+        Coverband::Collectors::ViewTracker.new(store: Coverband.configuration.store).as_json
+      end
+
       def debug_data
         Coverband.configuration.store.get_coverage_report.to_json
       end

data/lib/coverband/utils/railtie.rb

@@ -11,22 +11,29 @@ module Coverband
 
   class Railtie < Rails::Railtie
     initializer 'coverband.configure' do |app|
-      app.middleware.use Coverband::BackgroundMiddleware
+      begin
+        app.middleware.use Coverband::BackgroundMiddleware
 
-      if Coverband.configuration.track_views
-        CoverbandViewTracker = Coverband::Collectors::ViewTracker.new
-        Coverband.configuration.view_tracker = CoverbandViewTracker
+        if Coverband.configuration.track_views
+          CoverbandViewTracker = Coverband::Collectors::ViewTracker.new
+          Coverband.configuration.view_tracker = CoverbandViewTracker
 
-        ActiveSupport::Notifications.subscribe(/render_partial.action_view|render_template.action_view/) do |name, start, finish, id, payload|
-          CoverbandViewTracker.track_views(name, start, finish, id, payload) unless name.include?('!')
+          ActiveSupport::Notifications.subscribe(/render_partial.action_view|render_template.action_view/) do |name, start, finish, id, payload|
+            CoverbandViewTracker.track_views(name, start, finish, id, payload) unless name.include?('!')
+          end
         end
+      rescue Redis::CannotConnectError => error
+        Coverband.configuration.logger.info "Redis is not available (#{error}), Coverband not configured"
+        Coverband.configuration.logger.info 'If this is a setup task like assets:precompile feel free to ignore'
       end
     end
 
     config.after_initialize do
-      Coverband.eager_loading_coverage!
-      Coverband.report_coverage
-      Coverband.runtime_coverage!
+      unless Coverband.tasks_to_ignore?
+        Coverband.eager_loading_coverage!
+        Coverband.report_coverage
+        Coverband.runtime_coverage!
+      end
     end
 
     rake_tasks do

data/lib/coverband/version.rb

@@ -5,5 +5,5 @@
 # use format '4.2.1.rc.1' ~> 4.2.1.rc to prerelease versions like v4.2.1.rc.2 and v4.2.1.rc.3
 ###
 module Coverband
-  VERSION = '4.2.3'
+  VERSION = '4.2.4.rc.1'
 end

data/lua/install.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+LUA_DIR="$HOME/lua51"
+BUSTED="$LUA_DIR/bin/busted"
+
+if [ -f "$BUSTED" ]
+then
+  echo "$BUSTED already exists"
+  exit 0
+fi
+
+pip install hererocks
+hererocks $LUA_DIR -l5.1 -rlatest
+source $LUA_DIR/bin/activate
+lua -v
+for i in luacov busted redis-lua inspect lua-cjson; do 
+  luarocks install $i;
+done

data/lua/lib/persist-coverage.lua

@@ -0,0 +1,28 @@
+local hmset = function (key, dict)
+  if next(dict) == nil then return nil end
+  local bulk = {}
+  for k, v in pairs(dict) do
+    table.insert(bulk, k)
+    table.insert(bulk, v)
+  end
+  return redis.call('HMSET', key, unpack(bulk))
+end
+local payload = cjson.decode(redis.call('get', (KEYS[1])))
+local ttl = payload.ttl
+local files_data = payload.files_data
+redis.call('DEL', KEYS[1])
+for _, file_data in ipairs(files_data) do
+
+  local hash_key = file_data.hash_key
+  local first_updated_at = file_data.meta.first_updated_at
+  file_data.meta.first_updated_at = nil
+
+  hmset(hash_key, file_data.meta)
+  redis.call('HSETNX', hash_key, 'first_updated_at', first_updated_at)
+  for line, coverage in pairs(file_data.coverage) do
+    redis.call("HINCRBY", hash_key, line, coverage)
+  end
+  if ttl and ttl ~= cjson.null then
+    redis.call("EXPIRE", hash_key, ttl)
+  end
+end

data/lua/test/bootstrap.lua

@@ -0,0 +1,5 @@
+inspect = require "inspect"
+cjson = require 'cjson'
+require './lua/test/redis-call'
+
+

data/lua/test/harness.lua

@@ -0,0 +1,19 @@
+require './lua/test/bootstrap'
+
+
+KEYS = {}
+ARGV = {}
+
+
+function call_redis_script(script, keys, argv)
+  -- This may not be strictly necessary
+  for k,v in pairs(ARGV) do ARGV[k] = nil end
+  for k,v in pairs(KEYS) do KEYS[k] = nil end
+
+  for k,v in pairs(keys) do table.insert(KEYS, v) end
+  for k,v in pairs(argv) do table.insert(ARGV, v)  end
+
+  return dofile('./lua/lib/' .. script)
+end
+
+return call_redis_script;

data/lua/test/redis-call.lua

@@ -0,0 +1,55 @@
+redis = require 'redis'
+
+-- If you have some different host/port change it here
+local host = "127.0.0.1"
+local port = 6379
+
+
+client = redis.connect(host, port)
+
+-- Workaround for absence of redis.call
+redis.call = function(cmd, ...)
+  local arg={...}
+  local args_string = ''
+  for i,v in ipairs(arg) do
+    args_string = args_string .. ' ' .. v
+  end
+  cmd = string.lower(cmd)
+  print(cmd .. args_string)
+  --local result = assert(load('return client:'.. cmd ..'(...)'))(...)
+  local result = assert(loadstring('return client:'.. cmd ..'(...)'))(...)
+
+  -- The redis-lua library returns some values differently to how `redis.call` works inside redis.
+  -- this makes the responses look like those from the builtin redis
+  local response_lookup = {
+    type = function() return { ["ok"]= result } end,
+    sadd = function() return tonumber(result) end,
+    zrange = function()
+      if type(result) == "table" and type(result[1]) == "table" then
+        -- Deal with WITHSCORES...
+        local new_result = {}
+        for k,v in pairs(result) do
+          table.insert(new_result, v[1])
+          table.insert(new_result, v[2])
+        end
+        return new_result;
+      end
+
+      return result;
+    end,
+    hgetall = function()
+      local new_result = {}
+      for key, value in pairs(result) do
+        table.insert(new_result, key)
+        table.insert(new_result, value)
+      end
+      return new_result
+    end
+  }
+
+  if response_lookup[cmd] then
+    return response_lookup[cmd]()
+  end
+
+  return result;
+end

data/lua/test/test-persist-coverage.lua

@@ -0,0 +1,132 @@
+local call_redis_script = require "./lua/test/harness";
+
+describe("persist-coverage", function()
+  local function hgetall(hash_key)
+    local flat_map = redis.call('HGETALL', hash_key)
+    local result = {}
+    for i = 1, #flat_map, 2 do
+      result[flat_map[i]] = flat_map[i + 1]
+    end
+    return result
+  end
+
+  local function clean_redis() 
+    redis.call('flushdb')
+  end
+
+  before_each(function()
+    clean_redis()
+  end)
+
+  after_each(function()
+    clean_redis()
+  end)
+
+  it("Adds data on multiple files", function()
+    local first_updated_at = "1569453853"
+    local last_updated_at = first_updated_at
+
+    local key = 'hash_key'
+    local json = cjson.encode({
+      ttl = nil,
+      files_data = {
+        {
+          hash_key = "coverband_hash_3_3.coverband_test.runtime../dog.rb.abcd",
+          meta = {
+            first_updated_at = first_updated_at, 
+            last_updated_at = last_updated_at, 
+            file = "./dog.rb",
+            file_hash = 'abcd', 
+            file_length = 3 
+          },
+          coverage = {
+            ['0'] = 0, 
+            ['1'] = 1, 
+            ['2'] = 2
+          }
+        },
+        {
+          hash_key = "coverband_hash_3_3.coverband_test.runtime../fish.rb.1234",
+          meta = {
+            first_updated_at = first_updated_at, 
+            last_updated_at = last_updated_at, 
+            file = "./fish.rb",
+            file_hash = '1234', 
+            file_length = 3
+          },
+          coverage = {
+            ['0'] = 1, 
+            ['1'] = 0, 
+            ['2'] = 1 
+          }
+        }
+      }
+    });
+    redis.call( 'set', key, json)
+
+    call_redis_script('persist-coverage.lua',  { key },  {});
+    local results = hgetall("coverband_hash_3_3.coverband_test.runtime../dog.rb.abcd")
+    assert.are.same({
+      ["0"] = "0",
+      ["1"] = "1",
+      ["2"] = "2",
+      file = "./dog.rb",
+      file_hash = "abcd",
+      file_length = "3",
+      first_updated_at = first_updated_at ,
+      last_updated_at = last_updated_at
+    }, results)
+
+    results = hgetall("coverband_hash_3_3.coverband_test.runtime../fish.rb.1234")
+    assert.are.same({
+      ["0"] = "1",
+      ["1"] = "0",
+      ["2"] = "1",
+      file = "./fish.rb",
+      file_hash = "1234",
+      file_length = "3",
+      first_updated_at = first_updated_at ,
+      last_updated_at = last_updated_at
+    }, results)
+
+    assert.is_false(false, redis.call('exists', key))
+
+    last_updated_at = "1569453953"
+    json = cjson.encode({
+      ttl = nil,
+      files_data = {
+        {
+          hash_key="coverband_hash_3_3.coverband_test.runtime../dog.rb.abcd",
+          meta = {
+            first_updated_at=first_updated_at, 
+            last_updated_at=last_updated_at, 
+            file="./dog.rb", 
+            file_hash='abcd', 
+            file_length=3
+          },
+          coverage = {
+            ['0']= 1, 
+            ['1']= 1, 
+            ['2']= 1
+          }
+        }
+      }
+    })
+    redis.call( 'set', key, json )
+
+    call_redis_script('persist-coverage.lua',  { key },  {} );
+    results = hgetall("coverband_hash_3_3.coverband_test.runtime../dog.rb.abcd")
+    assert.are.same({
+      ["0"] = "1",
+      ["1"] = "2",
+      ["2"] = "3",
+      file = "./dog.rb",
+      file_hash = "abcd",
+      file_length = "3",
+      first_updated_at = first_updated_at,
+      last_updated_at = last_updated_at 
+    }, results)
+
+    assert.is_false(false, redis.call('exists', key))
+  end)
+end)

data/test/coverband/adapters/hash_redis_store_test.rb

@@ -13,13 +13,12 @@ class HashRedisStoreTest < Minitest::Test
 
   def setup
     super
-    @redis = Redis.new
+    @redis = Coverband::Test.redis
     # FIXME: remove dependency on configuration and instead pass this in as an argument
     Coverband.configure do |config|
       config.root_paths = ['app_path/']
     end
     @store = Coverband::Adapters::HashRedisStore.new(@redis, redis_namespace: 'coverband_test', relative_file_converter: MockRelativeFileConverter)
-    @store.clear!
     Coverband.configuration.store = @store
   end
 
@@ -57,6 +56,7 @@ class HashRedisStoreTest < Minitest::Test
       {
         'first_updated_at' => yesterday.to_i,
         'last_updated_at' => yesterday.to_i,
+        'file_hash' => 'abcd',
         'data' => [0, 1, 2]
       },
       @store.coverage['./dog.rb']
@@ -69,6 +69,7 @@ class HashRedisStoreTest < Minitest::Test
       {
         'first_updated_at' => yesterday.to_i,
         'last_updated_at' => today.to_i,
+        'file_hash' => 'abcd',
         'data' => [1, 2, 2]
       },
       @store.coverage['./dog.rb']
@@ -107,6 +108,7 @@ class HashRedisStoreTest < Minitest::Test
       {
         'first_updated_at' => current_time.to_i,
         'last_updated_at' => current_time.to_i,
+        'file_hash' => 'abcd',
         'data' => [0, nil, 1, 2]
       }, @store.coverage['./dog.rb']
     )

data/test/coverband/adapters/redis_store_test.rb

@@ -8,11 +8,8 @@ unless ENV['COVERBAND_HASH_REDIS_STORE']
 
     def setup
       super
-      Coverband.configuration.redis_namespace = 'coverband_test'
-      @redis = Redis.new
-      @store = Coverband::Adapters::RedisStore.new(@redis, redis_namespace: 'coverband_test')
-      Coverband.configuration.store = @store
-      @store.clear!
+      @store = Coverband.configuration.store
+      @redis = @store.instance_variable_get(:@redis)
     end
 
     def test_coverage

data/test/coverband/collectors/coverage_test.rb

@@ -7,10 +7,7 @@ class CollectorsCoverageTest < Minitest::Test
 
   def setup
     super
-    Coverband.configure do |config|
-      config.store = Coverband::Adapters::RedisStore.new(Redis.new, redis_namespace: 'coverband_test')
-    end
-    @coverband = Coverband::Collectors::Coverage.instance.reset_instance
+    @coverband = Coverband::Collectors::Coverage.instance
     # preload first coverage hit
     @coverband.report_coverage
   end

data/test/coverband/collectors/view_tracker_test.rb

@@ -1,7 +1,8 @@
+# frozen_string_literal: true
+
 require File.expand_path('../../test_helper', File.dirname(__FILE__))
 
 class ReporterTest < Minitest::Test
-
   def tracker_key
     'render_tracker_2'
   end
@@ -11,75 +12,75 @@ class ReporterTest < Minitest::Test
     fake_store.raw_store.del(tracker_key)
   end
 
-  test "init correctly" do
+  test 'init correctly' do
     Coverband::Collectors::ViewTracker.expects(:supported_version?).returns(true)
-    tracker = Coverband::Collectors::ViewTracker.new(:store => fake_store, :roots => 'dir')
+    tracker = Coverband::Collectors::ViewTracker.new(store: fake_store, roots: 'dir')
     assert_equal 'dir', tracker.roots.first
-    assert tracker.store != nil
+    assert !tracker.store.nil?
     assert_equal [], tracker.target
     assert_equal [], tracker.logged_views
   end
 
-  test "track partials" do
+  test 'track partials' do
     Coverband::Collectors::ViewTracker.expects(:supported_version?).returns(true)
     store = fake_store
     file_path = "#{File.expand_path(Coverband.configuration.root)}/file"
     store.raw_store.expects(:hset).with(tracker_key, file_path, anything)
     tracker = Coverband::Collectors::ViewTracker.new(store: store, roots: 'dir')
-    tracker.track_views('name', 'start', 'finish', 'id', {:identifier => file_path})
+    tracker.track_views('name', 'start', 'finish', 'id', identifier: file_path)
     tracker.report_views_tracked
     assert_equal [file_path], tracker.logged_views
   end
 
-  test "track layouts" do
+  test 'track layouts' do
     Coverband::Collectors::ViewTracker.expects(:supported_version?).returns(true)
     store = fake_store
     file_path = "#{File.expand_path(Coverband.configuration.root)}/layout"
     store.raw_store.expects(:hset).with(tracker_key, file_path, anything)
     tracker = Coverband::Collectors::ViewTracker.new(store: store, roots: 'dir')
-    tracker.track_views('name', 'start', 'finish', 'id', {:layout => file_path})
+    tracker.track_views('name', 'start', 'finish', 'id', layout: file_path)
     tracker.report_views_tracked
     assert_equal [file_path], tracker.logged_views
   end
 
-  test "report used partials" do
+  test 'report used partials' do
     Coverband::Collectors::ViewTracker.expects(:supported_version?).returns(true)
     store = fake_store
     file_path = "#{File.expand_path(Coverband.configuration.root)}/file"
     tracker = Coverband::Collectors::ViewTracker.new(store: store, roots: 'dir')
-    tracker.track_views('name', 'start', 'finish', 'id', {:identifier => file_path})
+    tracker.track_views('name', 'start', 'finish', 'id', identifier: file_path)
     tracker.report_views_tracked
     assert_equal [file_path], tracker.used_views.keys
   end
 
-  test "report unused partials" do
+  test 'report unused partials' do
     Coverband::Collectors::ViewTracker.expects(:supported_version?).returns(true)
     store = fake_store
     file_path = "#{File.expand_path(Coverband.configuration.root)}/file"
     target = [file_path, 'not_used']
     tracker = Coverband::Collectors::ViewTracker.new(store: store, roots: 'dir', target: target)
-    tracker.track_views('name', 'start', 'finish', 'id', {:identifier => file_path})
+    tracker.track_views('name', 'start', 'finish', 'id', identifier: file_path)
     tracker.report_views_tracked
     assert_equal ['not_used'], tracker.unused_views
   end
 
-  test "reset store" do
+  test 'reset store' do
     Coverband::Collectors::ViewTracker.expects(:supported_version?).returns(true)
     store = fake_store
     store.raw_store.expects(:del).with(tracker_key)
     store.raw_store.expects(:del).with('render_tracker_time')
     tracker = Coverband::Collectors::ViewTracker.new(store: store, roots: 'dir')
-    tracker.track_views('name', 'start', 'finish', 'id', {:identifier => 'file'})
+    tracker.track_views('name', 'start', 'finish', 'id', identifier: 'file')
     tracker.reset_recordings
   end
 
-  test "clear_file" do
+  test 'clear_file' do
     Coverband::Collectors::ViewTracker.expects(:supported_version?).returns(true)
     store = fake_store
     file_path = "#{File.expand_path(Coverband.configuration.root)}/file"
     store.raw_store.expects(:hdel).with(tracker_key, file_path)
     tracker = Coverband::Collectors::ViewTracker.new(store: store, roots: 'dir')
-    tracker.track_views('name', 'start', 'finish', 'id', {:identifier => file_path})
+    tracker.track_views('name', 'start', 'finish', 'id', identifier: file_path)
     tracker.clear_file!('file')
     assert_equal [], tracker.logged_views
   end
@@ -87,7 +88,6 @@ class ReporterTest < Minitest::Test
   protected
 
   def fake_store
-    @fake_store ||= Coverband::Adapters::RedisStore.new(Redis.new, redis_namespace: 'coverband_test')
+    @fake_store ||= Coverband::Adapters::RedisStore.new(Coverband::Test.redis, redis_namespace: 'coverband_test')
   end
-
 end

data/test/coverband/configuration_test.rb

@@ -12,13 +12,13 @@ class BaseTest < Minitest::Test
       config.root_paths          = ['/app_path/']
       config.ignore              = ['config/envionments']
       config.reporter            = 'std_out'
-      config.store               = Coverband::Adapters::RedisStore.new(Redis.new, redis_namespace: 'coverband_test')
+      config.store               = Coverband::Adapters::RedisStore.new(Coverband::Test.redis, redis_namespace: 'coverband_test')
     end
   end
 
   test 'ignore works with equal' do
     coverband = Coverband::Collectors::Coverage.instance.reset_instance
-    expected = ["vendor", ".erb$", ".slim$", "/tmp", "internal:prelude", "schema.rb", "config/envionments"]
+    expected = ['vendor', '.erb$', '.slim$', '/tmp', 'internal:prelude', 'schema.rb', 'config/envionments']
     assert_equal expected, Coverband.configuration.ignore
   end
 
@@ -27,14 +27,14 @@ class BaseTest < Minitest::Test
       config.ignore += ['config/initializers']
     end
     coverband = Coverband::Collectors::Coverage.instance.reset_instance
-    expected = ["vendor",
-      ".erb$",
-      ".slim$",
-      "/tmp",
-      "internal:prelude",
-      "schema.rb",
-      "config/envionments",
-      "config/initializers"]
+    expected = ['vendor',
+                '.erb$',
+                '.slim$',
+                '/tmp',
+                'internal:prelude',
+                'schema.rb',
+                'config/envionments',
+                'config/initializers']
     assert_equal expected, Coverband.configuration.ignore
   end
 

data/test/coverband/integrations/resque_worker_test.rb

@@ -16,7 +16,6 @@ class ResqueWorkerTest < Minitest::Test
     Coverband.configure do |config|
       config.background_reporting_enabled = false
     end
-    Coverband.configuration.store.instance_variable_set(:@redis_namespace, 'coverband_test')
     Coverband.start
     redis = Coverband.configuration.store.instance_eval { @redis }
     Resque.redis = redis

data/test/coverband/reporters/base_test.rb

@@ -3,6 +3,10 @@
 require File.expand_path('../../test_helper', File.dirname(__FILE__))
 
 class ReportsBaseTest < Minitest::Test
+  def setup
+    super
+  end
+
   test '#merge_arrays basic merge preserves order and counts' do
     first = [0, 0, 1, 0, 1]
     second = [nil, 0, 1, 0, 0]
@@ -28,14 +32,9 @@ class ReportsBaseTest < Minitest::Test
   end
 
   test "#get_current_scov_data_imp doesn't ignore folders with default ignore keys" do
-    @redis = Redis.new
-    store = Coverband::Adapters::RedisStore.new(@redis, redis_namespace: 'coverband_test')
-    store.clear!
-
     Coverband.configure do |config|
       config.reporter          = 'std_out'
       config.root              = '/full/remote_app/path'
-      config.store             = store
     end
 
     key = '/a_path/that_has_erb_in/thepath.rb'
@@ -55,6 +54,7 @@ class ReportsBaseTest < Minitest::Test
   # rubocop:disable all
   ###
   test '#get_current_scov_data_imp merges multiples of file data' do
+    skip "@danmayer to take a closer look at this one"
     coverage = {'/base/66/app/controllers/dashboard_controller.rb' =>
        {"first_updated_at"=>1549610119,
         "last_updated_at"=>1549610200,
@@ -88,4 +88,5 @@ class ReportsBaseTest < Minitest::Test
                 "data"=>[38, 38, 38, nil, 38, 38, nil, nil, 38, nil, 38, 80, 80, nil, 80, 80, 80, 80, 80, 80, 80, nil, nil, 38, nil, 38, 80, 55, 0, 0, 0, 0, nil, nil, nil, nil, 0, 0, nil, nil, 38, 80, 80, 80, nil, nil, nil, nil, nil, 38, 80, nil, nil, 38, 80, nil, nil]}
     assert_equal expected, Coverband::Reporters::Base.send(:get_current_scov_data_imp, store, roots)[:merged][key]
   end
+
 end

data/test/coverband/reporters/console_test.rb

@@ -7,15 +7,12 @@ class HTMLReportTest < Minitest::Test
 
   def setup
     super
-    @redis = Redis.new
-    @store = Coverband::Adapters::RedisStore.new(@redis, redis_namespace: 'coverband_test')
-    @store.clear!
+    @store = Coverband.configuration.store
   end
 
   test 'report data' do
     Coverband.configure do |config|
-      config.reporter            = 'std_out'
-      config.store               = @store
+      config.reporter = 'std_out'
     end
     Coverband.configuration.logger.stubs('info')
     mock_file_hash

data/test/coverband/reporters/html_test.rb

@@ -5,9 +5,7 @@ require File.expand_path('../../test_helper', File.dirname(__FILE__))
 class ReportHTMLTest < Minitest::Test
   def setup
     super
-    @redis = Redis.new
-    @store = Coverband::Adapters::RedisStore.new(@redis, redis_namespace: 'coverband_test')
-    @store.clear!
+    @store = Coverband.configuration.store
     Coverband.configure do |config|
       config.store             = @store
       config.root              = fixtures_root

data/test/coverband/utils/html_formatter_test.rb

@@ -5,9 +5,7 @@ require File.expand_path('../../test_helper', File.dirname(__FILE__))
 class HTMLFormatterTest < Minitest::Test
   def setup
     super
-    @redis = Redis.new
-    @store = Coverband::Adapters::RedisStore.new(@redis, redis_namespace: 'coverband_test')
-    @store.clear!
+    @store = Coverband::Adapters::RedisStore.new(Coverband::Test.redis, redis_namespace: 'coverband_test')
   end
 
   test 'generate dynamic content hosted html report' do

data/test/forked/rails_rake_full_stack_test.rb

@@ -45,4 +45,14 @@ class RailsRakeFullStackTest < Minitest::Test
     assert_equal empty_hash, coverage_report[:eager_loading]
     assert_equal empty_hash, coverage_report[:merged]
   end
+
+  test "doesn't exit non-zero with error on missing redis" do
+    output = `COVERBAND_CONFIG=./test/rails#{Rails::VERSION::MAJOR}_dummy/config/coverband_missing_redis.rb bundle exec rake -f test/rails#{Rails::VERSION::MAJOR}_dummy/Rakefile -T`
+    assert_equal 0, $?.to_i
+    if ENV['COVERBAND_HASH_REDIS_STORE']
+      assert output.match(/Redis is not available/)
+    else
+      assert output.match(/coverage failed to store/)
+    end
+  end
 end

data/test/integration/full_stack_test.rb

@@ -11,12 +11,10 @@ class FullStackTest < Minitest::Test
     super
     Coverband::Collectors::Coverage.instance.reset_instance
     Coverband.configure do |config|
-      config.store = Coverband::Adapters::RedisStore.new(Redis.new, redis_namespace: 'coverband_test')
       config.s3_bucket = nil
-      config.background_reporting_enabled = true
+      config.background_reporting_enabled = false
       config.track_gems = true
     end
-    Coverband.configuration.store.clear!
     Coverband.start
     Coverband::Collectors::Coverage.instance.eager_loading!
     @rack_file = require_unique_file 'fake_app/basic_rack.rb'

data/test/rails4_dummy/config/coverband.rb

@@ -1,13 +1,3 @@
-Coverband.configure do |config|
-  config.root              = Dir.pwd
-  config.store             = Coverband::Adapters::RedisStore.new(Redis.new(url: ENV['REDIS_URL']), redis_namespace: 'coverband_test') if defined? Redis
-  config.ignore            = %w[vendor .erb$ .slim$]
-  config.root_paths        = []
-  config.logger            = Rails.logger
-  config.verbose           = true
-  config.background_reporting_enabled = true
-  config.track_gems = true
-  config.gem_details = true
-  config.use_oneshot_lines_coverage = true if ENV['ONESHOT']
-  config.simulate_oneshot_lines_coverage = true if ENV['SIMULATE_ONESHOT']
-end
+# frozen_string_literal: true
+
+require_relative '../../rails5_dummy/config/coverband'

data/test/rails4_dummy/config/coverband_missing_redis.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative '../../rails5_dummy/config/coverband_missing_redis'

data/test/rails5_dummy/config/coverband.rb

@@ -1,6 +1,8 @@
+# frozen_string_literal: true
+
 Coverband.configure do |config|
   config.root              = Dir.pwd
-  config.store             = Coverband::Adapters::RedisStore.new(Redis.new(url: ENV['REDIS_URL']), redis_namespace: 'coverband_test') if defined? Redis
+  config.store             = Coverband::Adapters::RedisStore.new(Redis.new(db: 2, url: ENV['REDIS_URL']), redis_namespace: 'coverband_test') if defined? Redis
   config.ignore            = %w[vendor .erb$ .slim$]
   config.root_paths        = []
   config.logger            = Rails.logger

data/test/rails5_dummy/config/coverband_missing_redis.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+Coverband.configure do |config|
+  config.root              = Dir.pwd
+  config.store             = Coverband::Adapters::RedisStore.new(Redis.new(db: 2, url: 'redis://127.0.0.1:123'), redis_namespace: 'coverband_test') if defined? Redis
+  config.ignore            = %w[vendor .erb$ .slim$]
+  config.root_paths        = []
+  config.logger            = Rails.logger
+  config.verbose           = true
+  config.background_reporting_enabled = true
+  config.track_gems = true
+  config.gem_details = true
+  config.use_oneshot_lines_coverage = true if ENV['ONESHOT']
+  config.simulate_oneshot_lines_coverage = true if ENV['SIMULATE_ONESHOT']
+end

data/test/test_helper.rb

@@ -30,20 +30,24 @@ end
 
 module Coverband
   module Test
+    TEST_DB = 2
+
+    def self.redis
+      @redis ||= Redis.new(db: TEST_DB)
+    end
+
     def self.reset
       Coverband.configuration.redis_namespace = 'coverband_test'
       Coverband.configuration.store.instance_variable_set(:@redis_namespace, 'coverband_test')
       Coverband.configuration.store.class.class_variable_set(:@@path_cache, {})
-      %i[eager_loading runtime].each do |type|
-        Coverband.configuration.store.type = type
-        Coverband.configuration.store.clear!
-      end
       Coverband.configuration.reset
       Coverband::Collectors::Coverage.instance.reset_instance
       Coverband::Utils::RelativeFileConverter.reset
       Coverband::Utils::AbsoluteFileConverter.reset
       Coverband.configuration.redis_namespace = 'coverband_test'
       Coverband::Background.stop
+      Coverband.configuration.store.instance_variable_set(:@redis, redis)
+      redis.flushdb
     end
 
     def setup

data/views/nav.erb

@@ -6,7 +6,7 @@
     <%= button("#{base_path}clear", 'clear coverage report', delete: true) %>
   <% end %>
   <% if Coverband.configuration.web_debug %>
-    <a href='<%= base_path %>debug_data'>Debug Data</a> &nbsp;
+    <a href='<%= base_path %>debug_data'>Coverage Data</a> &nbsp;
   <% end %>
 </div>
 <% if notice.to_s.length > 0 %>

data/views/view_tracker.erb

@@ -14,11 +14,14 @@
       <%= display_nav(show_coverage_link: true) %>
       <div id="content">
         <% tracker = Coverband::Collectors::ViewTracker.new(store: Coverband.configuration.store) %>
-        <h3>
+        <h4>
           <% if Coverband.configuration.web_enable_clear %>
             <%= button("#{base_path}clear_view_tracking", 'reset view tracker', delete: true) %>
           <% end %>
-        </h3>
+          <% if Coverband.configuration.web_debug %>
+            <a href='<%= "#{base_path}view_tracker_data" %>'>view tracker data</a>
+          <% end %>
+        </h4>
         <h2>Unused Views: (<%= tracker.unused_views.length %>)</h2>
         <p>These views have not been rendered since recording started at <%= tracker.tracking_since %></p>
         <ul>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: coverband
 version: !ruby/object:Gem::Version
-  version: 4.2.3
+  version: 4.2.4.rc.1
 platform: ruby
 authors:
 - Dan Mayer
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-28 00:00:00.000000000 Z
+date: 2019-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
@@ -304,6 +304,12 @@ files:
 - lib/coverband/utils/source_file.rb
 - lib/coverband/utils/tasks.rb
 - lib/coverband/version.rb
+- lua/install.sh
+- lua/lib/persist-coverage.lua
+- lua/test/bootstrap.lua
+- lua/test/harness.lua
+- lua/test/redis-call.lua
+- lua/test/test-persist-coverage.lua
 - public/application.css
 - public/application.js
 - public/colorbox/border.png
@@ -388,6 +394,7 @@ files:
 - test/rails4_dummy/config/application.rb
 - test/rails4_dummy/config/boot.rb
 - test/rails4_dummy/config/coverband.rb
+- test/rails4_dummy/config/coverband_missing_redis.rb
 - test/rails4_dummy/config/environment.rb
 - test/rails4_dummy/config/routes.rb
 - test/rails4_dummy/config/secrets.yml
@@ -397,6 +404,7 @@ files:
 - test/rails5_dummy/config.ru
 - test/rails5_dummy/config/application.rb
 - test/rails5_dummy/config/coverband.rb
+- test/rails5_dummy/config/coverband_missing_redis.rb
 - test/rails5_dummy/config/environment.rb
 - test/rails5_dummy/config/routes.rb
 - test/rails5_dummy/tmp/.keep
@@ -426,9 +434,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.0.3
 signing_key: 
@@ -493,6 +501,7 @@ test_files:
 - test/rails4_dummy/config/application.rb
 - test/rails4_dummy/config/boot.rb
 - test/rails4_dummy/config/coverband.rb
+- test/rails4_dummy/config/coverband_missing_redis.rb
 - test/rails4_dummy/config/environment.rb
 - test/rails4_dummy/config/routes.rb
 - test/rails4_dummy/config/secrets.yml
@@ -502,6 +511,7 @@ test_files:
 - test/rails5_dummy/config.ru
 - test/rails5_dummy/config/application.rb
 - test/rails5_dummy/config/coverband.rb
+- test/rails5_dummy/config/coverband_missing_redis.rb
 - test/rails5_dummy/config/environment.rb
 - test/rails5_dummy/config/routes.rb
 - test/rails5_dummy/tmp/.keep