betterlog 0.9.0 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 35b30ba42034bf3d4a09a4558f8a21b8e48b2a5145b2e56299b4f73f9e14ebd4
-  data.tar.gz: 03df9bb690c3fedb95696fa1c0708a08da9ed5f41246233115fa6444b761a1f5
+  metadata.gz: dade612b41a387e538c98ef623a79fdb483d4e230daed8d154cad2fa64465235
+  data.tar.gz: ac0ada91ced10e9d1cf64f4dfaecef4a7e9c24b63715fe7e7ddca231044646ed
 SHA512:
-  metadata.gz: 4982f6fb3cd975ef1069a8b36617e5494f346084069d3d6eed723152e854bde651d76859cb4e33681889c22821b8be19626dcc1fa396a4bdd166373e5def483e
-  data.tar.gz: 109c8fd2ff8c4f9217188450793aeacf32a580f47c82eafec33e6abea77fa152028207f82a05fbab91142fd73eaf8a34a2493bd1d85e2e3e770434fce12acf27
+  metadata.gz: 1b896f2288f53dedad255dccc578c072052fa6db07f6dfb2093db32ca8b38cfc1464ce5f26b2d94b4e8cf40b25889f1759920974cc0be6890f9cba026e8b83f9
+  data.tar.gz: 8dd4191451947ed9c9be2ff04ad2877b5902685599c1db9e32a25797a7393b5c9a49141565afa1216e2348fa90addab371664151cf01a973f9c8dc7a80f8628d

data/.travis.yml

@@ -1,5 +1,6 @@
 rvm:
   - 2.6
+  - 2.7
 sudo: false
 services:
   - redis-server

data/Makefile

@@ -1,4 +1,4 @@
-# Pass BASE_IMAGE=alpine:3.10.0
+# Pass BASE_IMAGE=alpine:3.13.3
 DOCKER_IMAGE_LATEST = betterlog
 DOCKER_IMAGE = $(DOCKER_IMAGE_LATEST):$(REVISION_SHORT)
 PROJECT_ID = betterplace-183212

data/VERSION

@@ -1 +1 @@
-0.9.0
+0.10.0

data/betterlog.gemspec

@@ -1,19 +1,19 @@
 # -*- encoding: utf-8 -*-
-# stub: betterlog 0.9.0 ruby lib
+# stub: betterlog 0.10.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "betterlog".freeze
-  s.version = "0.9.0"
+  s.version = "0.10.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["betterplace Developers".freeze]
-  s.date = "2020-01-17"
+  s.date = "2020-01-22"
   s.description = "This library provides structure json logging for our rails projects".freeze
   s.email = "developers@betterplace.org".freeze
   s.executables = ["betterlog".freeze, "betterlog_pusher".freeze, "betterlog_sink".freeze]
   s.extra_rdoc_files = ["README.md".freeze, "lib/betterlog.rb".freeze, "lib/betterlog/global_metadata.rb".freeze, "lib/betterlog/log.rb".freeze, "lib/betterlog/log/event.rb".freeze, "lib/betterlog/log/event_formatter.rb".freeze, "lib/betterlog/log/severity.rb".freeze, "lib/betterlog/log_event_formatter.rb".freeze, "lib/betterlog/logger.rb".freeze, "lib/betterlog/notifiers.rb".freeze, "lib/betterlog/railtie.rb".freeze, "lib/betterlog/version.rb".freeze]
-  s.files = [".gitignore".freeze, ".travis.yml".freeze, "Dockerfile".freeze, "Gemfile".freeze, "LICENSE".freeze, "Makefile".freeze, "README.md".freeze, "Rakefile".freeze, "TODO.md".freeze, "VERSION".freeze, "betterlog.gemspec".freeze, "betterlog/config.go".freeze, "betterlog/healthz.go".freeze, "betterlog/redis_cert_cache.go".freeze, "bin/betterlog".freeze, "bin/betterlog_pusher".freeze, "bin/betterlog_sink".freeze, "cmd/betterlog-server/LICENSE".freeze, "cmd/betterlog-server/main.go".freeze, "config/log.yml".freeze, "go.mod".freeze, "go.sum".freeze, "lib/betterlog.rb".freeze, "lib/betterlog/global_metadata.rb".freeze, "lib/betterlog/log.rb".freeze, "lib/betterlog/log/event.rb".freeze, "lib/betterlog/log/event_formatter.rb".freeze, "lib/betterlog/log/severity.rb".freeze, "lib/betterlog/log_event_formatter.rb".freeze, "lib/betterlog/logger.rb".freeze, "lib/betterlog/notifiers.rb".freeze, "lib/betterlog/railtie.rb".freeze, "lib/betterlog/version.rb".freeze, "spec/betterlog/global_metadata_spec.rb".freeze, "spec/betterlog/log/event_spec.rb".freeze, "spec/betterlog/log/severity_spec.rb".freeze, "spec/betterlog/log_spec.rb".freeze, "spec/betterlog/logger_spec.rb".freeze, "spec/spec_helper.rb".freeze]
+  s.files = [".gitignore".freeze, ".travis.yml".freeze, "Dockerfile".freeze, "Gemfile".freeze, "LICENSE".freeze, "Makefile".freeze, "README.md".freeze, "Rakefile".freeze, "TODO.md".freeze, "VERSION".freeze, "betterlog.gemspec".freeze, "betterlog/config.go".freeze, "betterlog/healthz.go".freeze, "betterlog/redis_cert_cache.go".freeze, "bin/betterlog".freeze, "bin/betterlog_pusher".freeze, "bin/betterlog_sink".freeze, "cloudbuild.yaml".freeze, "cmd/betterlog-server/LICENSE".freeze, "cmd/betterlog-server/main.go".freeze, "config/log.yml".freeze, "go.mod".freeze, "go.sum".freeze, "lib/betterlog.rb".freeze, "lib/betterlog/global_metadata.rb".freeze, "lib/betterlog/log.rb".freeze, "lib/betterlog/log/event.rb".freeze, "lib/betterlog/log/event_formatter.rb".freeze, "lib/betterlog/log/severity.rb".freeze, "lib/betterlog/log_event_formatter.rb".freeze, "lib/betterlog/logger.rb".freeze, "lib/betterlog/notifiers.rb".freeze, "lib/betterlog/railtie.rb".freeze, "lib/betterlog/version.rb".freeze, "spec/betterlog/global_metadata_spec.rb".freeze, "spec/betterlog/log/event_spec.rb".freeze, "spec/betterlog/log/severity_spec.rb".freeze, "spec/betterlog/log_spec.rb".freeze, "spec/betterlog/logger_spec.rb".freeze, "spec/spec_helper.rb".freeze]
   s.homepage = "http://github.com/betterplace/betterlog".freeze
   s.rdoc_options = ["--title".freeze, "Betterlog".freeze, "--main".freeze, "README.md".freeze]
   s.rubygems_version = "3.1.2".freeze

data/bin/betterlog_pusher

@@ -32,7 +32,7 @@ loop do
   lm.lock!(File.basename($0), lock_time) do
     print ?…
     logger.each_slice(lines).with_index do |batch, i|
-      count.zero? and print ?(
+      count.zero? and print ?○
       count += batch.sum(&:size)
       attempt(attempts: 10, sleep: -60, reraise: true) do
         print ?┄
@@ -44,7 +44,7 @@ loop do
   if count.zero?
     sleep 1
   else
-    print "→%s)" % Tins::Unit.format(count, format: '%.2f %U', prefix: 1024, unit: ?b)
+    print "→ %s sent.\n" % Tins::Unit.format(count, format: '%.2f %U', prefix: 1024, unit: ?b)
   end
 rescue Redlock::LockError => e
   STDERR.puts "Caught #{e.class}: #{e} => Retrying!"

data/cloudbuild.yaml

@@ -0,0 +1,9 @@
+steps:
+- name: gcr.io/cloud-builders/docker
+  args: ['pull', '${_BASE_IMAGE}']
+- name: gcr.io/cloud-builders/docker
+  args: ['build', '-t', 'eu.gcr.io/$PROJECT_ID/${_IMAGE_NAME}:$SHORT_SHA', '-t', 'eu.gcr.io/$PROJECT_ID/${_IMAGE_NAME}', '--build-arg=BASE_IMAGE=${_BASE_IMAGE}', '.']
+images:
+  - 'eu.gcr.io/$PROJECT_ID/${_IMAGE_NAME}:$SHORT_SHA'
+  - 'eu.gcr.io/$PROJECT_ID/${_IMAGE_NAME}'
+timeout: 600s

data/lib/betterlog/logger.rb

@@ -4,6 +4,7 @@ module Betterlog
   class Logger < ::Logger
     def initialize(redis, shift_age = 0, shift_size = 1048576, name: nil, buffer_size: nil, **opts)
       @redis       = redis
+      @fallback    = ::Logger.new(STDERR)
       @name        = name || self.class.name
       @buffer_size = determine_buffer_size(buffer_size)
       super(nil, shift_age, shift_size, **opts)
@@ -29,11 +30,8 @@ module Betterlog
     private def redis_write(msg)
       # Stop before reaching configured buffer_size limit, after warning a lot.
       if @redis.strlen(@name) > (@buffer_size * 96) / 100
-        return nil
-      end
-      if @redis.strlen(@name) > (@buffer_size * 95) / 100
-        @redis.append @name, "\nRedis memory limit will soon be reached =>"\
-          " Log output to redis stops now unless log data is pushed away!\n"
+        @fallback.error("Redis memory limit will soon be reached =>"\
+          " Log output to redis stops now unless log data is pushed away!")
         return nil
       end
       @redis.append @name, msg
@@ -60,10 +58,14 @@ module Betterlog
       redis_write(
         format_message(format_severity(severity), Time.now, progname, message))
       true
+    rescue Redis::BaseConnectionError
+      @fallback.add(severity, message, progname)
     end
 
     def <<(msg)
       redis_write(msg)
+    rescue Redis::BaseConnectionError
+      @fallback << msg
     end
 
     def clear
@@ -73,18 +75,40 @@ module Betterlog
 
     def each_chunk(chunk_size: 100 * 1024, &block)
       chunk_size > 0 or raise ArgumentError, 'chunk_size > 0 required'
+
+      # Delete any remaining temporary keys if we were interrtupted earlier
+      # (or in some other process.)
+      @redis.scan_each(match: "#{@name}_*") do |key|
+        @redis.del key
+      rescue Redis::BaseConnectionError
+      end
+
       @redis.exists(@name) or return Enumerator.new {}
+
       Enumerator.new do |y|
         name_tmp = "#{@name}_#{rand}"
         @redis.rename @name, name_tmp
+
         s = 0
         e = @redis.strlen(name_tmp) - 1
         until s > e
           y.yield @redis.getrange(name_tmp, s, s + chunk_size - 1)
           s += chunk_size
         end
-        @redis.del name_tmp
+
+      ensure
+        begin
+          @redis.del name_tmp
+        rescue Redis::BaseConnectionError
+          # We have to delete this later if del command failed here,
+          # see the beginning of this method.
+        end
       end.each(&block)
+
+    rescue Redis::BaseConnectionError => e
+      # Maybe it works again later, just log the error…
+      @fallback.error(e)
+      Enumerator.new {}
     end
 
     def each(chunk_size: 100 * 1024, &block)

data/lib/betterlog/version.rb

@@ -1,6 +1,6 @@
 module Betterlog
   # Betterlog version
-  VERSION         = '0.9.0'
+  VERSION         = '0.10.0'
   VERSION_ARRAY   = VERSION.split('.').map(&:to_i) # :nodoc:
   VERSION_MAJOR   = VERSION_ARRAY[0] # :nodoc:
   VERSION_MINOR   = VERSION_ARRAY[1] # :nodoc:

data/spec/betterlog/log_spec.rb

@@ -58,7 +58,7 @@ describe Betterlog::Log do
 
   describe '#info with internal logging error' do
     it 'should not crash ever, just log the problem to Rails.logger' do
-      expect_any_instance_of(instance.logger.class).to receive(:fatal).and_call_original
+      expect_any_instance_of(instance.logger.class).to receive(:fatal)
       expect(Log.info(BasicObject.new)).to eq Log.instance
     end
   end
@@ -77,6 +77,10 @@ describe Betterlog::Log do
       Betterlog::Notifiers.notifiers.clear
     end
 
+    before do
+      expect_any_instance_of(::Logger).to receive(:send).with(:info, any_args)
+    end
+
     it 'can send explicit notifications' do
       expect(notifier).to receive(:notify).with(
         'test',

data/spec/betterlog/logger_spec.rb

@@ -26,6 +26,14 @@ describe Betterlog::Logger do
         with('Betterlog::Logger', 'foo')
       logger << 'foo'
     end
+
+    it 'falls back if redis errors' do
+      allow(logger.instance_variable_get(:@redis)).to receive(:append).
+        and_raise(Redis::BaseConnectionError)
+      expect(logger.instance_variable_get(:@fallback)).to\
+        receive(:<<).with('foo')
+      logger << 'foo'
+    end
   end
 
   describe '#add' do
@@ -34,6 +42,14 @@ describe Betterlog::Logger do
         with('Betterlog::Logger', /INFO -- : foo/)
       logger.info 'foo'
     end
+
+    it 'falls back if redis errors' do
+      allow(logger.instance_variable_get(:@redis)).to receive(:append).
+        and_raise(Redis::BaseConnectionError)
+      expect(logger.instance_variable_get(:@fallback)).to\
+        receive(:add).with(::Logger::INFO, 'foo', nil)
+      logger.info 'foo'
+    end
   end
 
   describe '#each_chunk' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: betterlog
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.10.0
 platform: ruby
 authors:
 - betterplace Developers
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-17 00:00:00.000000000 Z
+date: 2020-01-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: gem_hadar
@@ -222,6 +222,7 @@ files:
 - bin/betterlog
 - bin/betterlog_pusher
 - bin/betterlog_sink
+- cloudbuild.yaml
 - cmd/betterlog-server/LICENSE
 - cmd/betterlog-server/main.go
 - config/log.yml