betterlog 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 975990d950f8eb2ee21cfbbf52fa8472f8a706a02a01728e76e40005ca5807d3
-  data.tar.gz: 74ec7d2ce187050e66d4ff5c424daffb99a18759e987a66a7340ec5edcdf16c1
+  metadata.gz: 4765c4c4583c00e53272425f3eba50890c1c1c9534bbcf0b9653befe1f5d59d0
+  data.tar.gz: 78523987e0a140b50aaee1e6d663b4c3821d5fab4e2dc750c17f5d3b5ca5b7d9
 SHA512:
-  metadata.gz: f38d883bf8935c90af49649ed0627b2dc600cb992d03d746be12586cca817efb04820b55455b4f209f405edf1dee8400a950f7a283fa1019719d71d629f48f9c
-  data.tar.gz: 1ac776bcbed6cafb804e25a6a1e4d41836bb232ef9b70b04d09fc6c6a80010c2bd187a88f3167495f2bff4fcd6fad34f55bada5e4e13801ed8dd5c3e2d1c505f
+  metadata.gz: 4cfd4bad70799e9e750439231bcc638e652f91e239f287688cab3895c0a6f72d143b48bb26297867860f1146c8837d1bdca595bd1a18bf601e24f467b869ddda
+  data.tar.gz: 76d29b2dd078b6ff14304851d2e62283434217d43667e62bad1b0f507ca729575b8927bdbd0d4e0264af377decf095377563fe6edd6660c287e1c4206a135405

data/README.md

@@ -2,17 +2,58 @@
 
 ## Description
 
-Logging tools for betterplace structured logging
+Logging tools for betterplace structured logging in rails applications.
+
+## Configuration
+
+Copy the example configuration in config/log.yml into your application to get
+you started. Then add this line to your Gemfile:
+
+```
+gem 'betterlog'
+```
 
 ## Usage
 
 ### `betterlog`
 
-TODO
+Use it to tail local logfiles:
+
+```
+$ betterlog -f
+```
+
+or filter from stdin with
+
+```
+$ cat log/development.log | betterlog
+```
+
+Search for GET in the last 1000 rails log lines:
+
+```
+$ betterlog -F rails -n 1000 -s GET
+```
+
+Display the help for more options with `betterlog -h`.
 
 ### `betterlog_pusher`
 
-TODO
+- `BETTERLOG_SERVER_URL` is the URL log information is ultimately posted to
+  in the form of `https://user:password@appname-prd-log.betterops.de/log`.
+
+- `BETTERLOG_LINES`, e. g. 1000, is the number of lines which are posted per
+  every request to the above URL.
+
+- `REDIS_URL` the redis server URL for the server where Log information is
+  stored before posted to the betterlog server.
+
+  The rails application should be configured like this to store log information
+  on this redis server:
+
+  ```
+  config.logger = Betterlog::Logger.new(Redis.new(url: ENV.fetch('REDIS_URL')))
+  ```
 
 ### `betterlog_sink`
 
@@ -23,12 +64,12 @@ To tail a log and prettify the output just call and pipe to the `betterlog`
 executable:
 
 ```
-  $ betterlog_sink --since=1m -f | betterlog
+$ betterlog_sink --since=1m -f | betterlog
 ```
 
 The sink always defaults to the production logfile, to switch the context to
 staging, prepend the command with the `LOG_ENV` env variable like so:
 
 ```
-  $ LOG_ENV=staging betterlog_sink -f | betterlog
+$ LOG_ENV=staging betterlog_sink -f | betterlog
 ```

data/Rakefile

@@ -24,6 +24,7 @@ GemHadar do
   dependency 'json',           '~>2.0'
   dependency 'term-ansicolor', '~>1.3'
   dependency 'redis',          '>=2.4'
+  dependency 'redlock'
   dependency 'excon'
 
   development_dependency 'rake'

data/VERSION

@@ -1 +1 @@
-0.3.0
+0.4.0

data/betterlog.gemspec

@@ -1,14 +1,14 @@
 # -*- encoding: utf-8 -*-
-# stub: betterlog 0.3.0 ruby lib
+# stub: betterlog 0.4.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "betterlog".freeze
-  s.version = "0.3.0"
+  s.version = "0.4.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["betterplace Developers".freeze]
-  s.date = "2019-07-15"
+  s.date = "2019-07-17"
   s.description = "This library provides structure json logging for our rails projects".freeze
   s.email = "developers@betterplace.org".freeze
   s.executables = ["betterlog".freeze, "betterlog_pusher".freeze, "betterlog_sink".freeze]
@@ -34,6 +34,7 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<json>.freeze, ["~> 2.0"])
       s.add_runtime_dependency(%q<term-ansicolor>.freeze, ["~> 1.3"])
       s.add_runtime_dependency(%q<redis>.freeze, [">= 2.4"])
+      s.add_runtime_dependency(%q<redlock>.freeze, [">= 0"])
       s.add_runtime_dependency(%q<excon>.freeze, [">= 0"])
     else
       s.add_dependency(%q<gem_hadar>.freeze, ["~> 1.10.0"])
@@ -46,6 +47,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<json>.freeze, ["~> 2.0"])
       s.add_dependency(%q<term-ansicolor>.freeze, ["~> 1.3"])
       s.add_dependency(%q<redis>.freeze, [">= 2.4"])
+      s.add_dependency(%q<redlock>.freeze, [">= 0"])
       s.add_dependency(%q<excon>.freeze, [">= 0"])
     end
   else
@@ -59,6 +61,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<json>.freeze, ["~> 2.0"])
     s.add_dependency(%q<term-ansicolor>.freeze, ["~> 1.3"])
     s.add_dependency(%q<redis>.freeze, [">= 2.4"])
+    s.add_dependency(%q<redlock>.freeze, [">= 0"])
     s.add_dependency(%q<excon>.freeze, [">= 0"])
   end
 end

data/bin/betterlog_pusher

@@ -3,26 +3,32 @@
 
 require 'betterlog'
 require 'excon'
+require 'redlock'
 
-lines  = Integer(ENV.fetch('BETTERLOG_LINES', 1_000))
-url    = ENV.fetch('BETTERLOG_SERVER_URL')
-name   = ENV['BETTERLOG_NAME']
-redis  = Redis.new(url: ENV.fetch('REDIS_URL'))
-logger = Betterlog::Logger.new(redis, name: name)
+lines     = Integer(ENV.fetch('BETTERLOG_LINES', 1_000))
+lock_time = Integer(ENV.fetch('BETTERLOG_LOCK_TIME', 60_000))
+url       = ENV.fetch('BETTERLOG_SERVER_URL')
+name      = ENV['BETTERLOG_NAME']
+redis_url = ENV.fetch('REDIS_URL')
+redis     = Redis.new(url: redis_url)
+lm        = Redlock::Client.new([ redis_url ])
+logger    = Betterlog::Logger.new(redis, name: name)
 
 quit = false
-
 [ :TERM, :INT, :QUIT ].each { |s| trap(s) { quit = true } }
 
 STDOUT.sync = true
 loop do
   count = 0
-  logger.each_slice(lines).with_index do |batch, i|
-    count.zero? and print ?(
-    count += batch.sum(&:size)
-    attempt(attempts: 10, sleep: -60, reraise: true) do
-      print ?┄
-      Excon.post(url, body: batch.join)
+  lm.lock!(File.basename($0), lock_time) do
+    print ?…
+    logger.each_slice(lines).with_index do |batch, i|
+      count.zero? and print ?(
+      count += batch.sum(&:size)
+      attempt(attempts: 10, sleep: -60, reraise: true) do
+        print ?┄
+        Excon.post(url, body: batch.join)
+      end
     end
   end
   quit and exit
@@ -31,4 +37,7 @@ loop do
   else
     print "→%s)" % Tins::Unit.format(count, format: '%.2f %U', prefix: 1024, unit: ?b)
   end
+rescue Redlock::LockError => e
+  STDERR.puts "Caught #{e.class}: #{e} => Retrying!"
+  exit
 end

data/config/log.yml

@@ -13,6 +13,7 @@ development: &development
     default: &default_format >
       {%lt%timestamp} {%5s%severity}
       "{%0.<%= ENV.fetch('COLUMNS', 80).to_i / 2 %>%message}"
+      {host}
       {location}
       {file}{-%O%backtrace}{-%O%meta}
     d: *default_format
@@ -23,6 +24,7 @@ development: &development
       metric: {metric} {value} {type}
       error_class: {error_class}
       backtrace: {%O%backtrace}
+      host: {host}
       location: {location}
       file: {file}
       meta: {%O%meta}\n
@@ -43,5 +45,21 @@ development: &development
       - /usr/local/var/log/elasticsearch.log
   legacy_supported: yes
 test: *development
-staging: *development
-production: *development
+staging:
+  <<: *development
+  sink:
+    namespace: betterlog
+    source: betterlog-app
+    cluster:
+      name: betterlog
+      zone: europe-west1-b
+      project: project-42
+production:
+  <<: *development
+  sink:
+    namespace: betterlog
+    source: betterlog-app
+    cluster:
+      name: betterlog
+      zone: europe-west1-b
+      project: project-666

data/lib/betterlog/log.rb

@@ -113,7 +113,7 @@ module Betterlog
     rescue => error
       e = Log::Event.ify(error)
       rest |= e.as_hash.subhash(:error_class, :backtrace, :message)
-      rest[:message] = "#{rest[:message]} while measuring metric #{metric}"
+      rest[:message] = "#{rest[:message].inspect} while measuring metric #{metric}"
       raise error
     ensure
       protect do

data/lib/betterlog/version.rb

@@ -1,6 +1,6 @@
 module Betterlog
   # Betterlog version
-  VERSION         = '0.3.0'
+  VERSION         = '0.4.0'
   VERSION_ARRAY   = VERSION.split('.').map(&:to_i) # :nodoc:
   VERSION_MAJOR   = VERSION_ARRAY[0] # :nodoc:
   VERSION_MINOR   = VERSION_ARRAY[1] # :nodoc:

data/spec/betterlog/log_spec.rb

@@ -154,7 +154,7 @@ describe Betterlog::Log do
 
     class MyEx < StandardError
       def backtrace
-        []
+        %w[ backtrace ]
       end
     end
 
@@ -162,18 +162,19 @@ describe Betterlog::Log do
       expected_event = Log::Event.new(
         metric:      'foo',
         type:        'seconds',
-        value:       10.0,
-        timestamp:   "2011-11-11T10:11:21.000Z",
-        message:     'MyEx: we were fucked while measuring metric foo',
+        value:       3.0,
+        timestamp:   "2011-11-11T10:11:14.000Z",
+        message:     '"MyEx: we were fucked" while measuring metric foo',
         error_class: 'MyEx',
-        backtrace:   [],
+        backtrace:   %w[ backtrace ]
       )
       expect(instance).to receive(:emit).with(expected_event)
       raised = false
       begin
         Log.measure(metric: 'foo') do
-          Time.dummy = Time.now + 10
+          Time.dummy = Time.now + 3
           raise MyEx, "we were fucked"
+          Time.dummy = Time.now + 7
         end
       rescue MyEx
         raised = true

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: betterlog
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - betterplace Developers
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-15 00:00:00.000000000 Z
+date: 2019-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: gem_hadar
@@ -156,6 +156,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '2.4'
+- !ruby/object:Gem::Dependency
+  name: redlock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: excon
   requirement: !ruby/object:Gem::Requirement