betterlog 0.1.0 → 0.2.0

data/bin/betterlog

@@ -2,239 +2,240 @@
 # vim: set ft=ruby et sw=2 ts=2:
 
 require 'betterlog'
-require 'tins/go'
-require 'file-tail'
 require 'complex_config/rude'
 require 'zlib'
+require 'file/tail'
+
+module Betterlog
+  class App
+    def initialize(args = ARGV.dup)
+      STDOUT.sync = true
+      @args = args
+      @opts = Tins::GO.go 'cfhp:e:s:S:n:F:', @args, defaults: { ?c => true, ?p => ?d }
+      filter_severities
+      @opts[?h] and usage
+    end
 
-class Betterlog
-  def initialize(args = ARGV.dup)
-    STDOUT.sync = true
-    @args = args
-    @opts = Tins::GO.go 'cfhp:e:s:S:n:F:', @args, defaults: { ?c => true, ?p => ?d }
-    filter_severities
-    @opts[?h] and usage
-  end
-
-  def usage
-    puts <<~end
-      Usage: #{prog} [OPTIONS] [LOGFILES]
+    def usage
+      puts <<~end
+        Usage: #{prog} [OPTIONS] [LOGFILES]
 
-      Options are
+        Options are
 
-        -c            to enable colors during pretty printing
-        -f            to follow the log files
-        -h            to display this help
-        -p FORMAT     to pretty print the log file if possible
-        -e EMITTER    only output events from these emitters
-        -s MATCH      only display events matching this search string
-        -S SEVERITY   only output events with severity, e. g. -S '>=warn'
-        -n NUMBER     rewind this many lines backwards before tailing log file
-        -F SHORTCUT   to open the config files with SHORTCUT
+          -c            to enable colors during pretty printing
+          -f            to follow the log files
+          -h            to display this help
+          -p FORMAT     to pretty print the log file if possible
+          -e EMITTER    only output events from these emitters
+          -s MATCH      only display events matching this search string
+          -S SEVERITY   only output events with severity, e. g. -S '>=warn'
+          -n NUMBER     rewind this many lines backwards before tailing log file
+          -F SHORTCUT   to open the config files with SHORTCUT
 
-      FORMAT values are: #{Array(cc.log.formats?&.attribute_names) * ?,}
+        FORMAT values are: #{Array(cc.log.formats?&.attribute_names) * ?,}
 
-      SEVERITY values are: #{Log::Severity.all * ?|}
+        SEVERITY values are: #{Log::Severity.all * ?|}
 
-      Config file SHORTCUTs are: #{Array(cc.log.config_files?&.attribute_names) * ?,}
+        Config file SHORTCUTs are: #{Array(cc.log.config_files?&.attribute_names) * ?,}
 
-      Note, that you can use multiple SHORTCUTs via "-F foo -F bar".
+        Note, that you can use multiple SHORTCUTs via "-F foo -F bar".
 
-      Examples:
+        Examples:
 
-        - Follow rails log in long format with colors for errors or greater:
+          - Follow rails log in long format with colors for errors or greater:
 
-          $ betterlog -f -F rails -p long -c -S ">=error"
+            $ betterlog -f -F rails -p long -c -S ">=error"
 
-        - Follow rails AND redis logs with default format in colors
-          including the last 10 lines:
+          - Follow rails AND redis logs with default format in colors
+            including the last 10 lines:
 
-          $ betterlog -f -F rails -F redis -pd -c -n 10
+            $ betterlog -f -F rails -F redis -pd -c -n 10
 
-        - Filter stdin from file unicorn.log with default format in color:
+          - Filter stdin from file unicorn.log with default format in color:
 
-          $ betterlog -pd -c <unicorn.log
+            $ betterlog -pd -c <unicorn.log
 
-        - Filter the last 10 lines of file unicorn.log with default format
-          in color:
+          - Filter the last 10 lines of file unicorn.log with default format
+            in color:
 
-          $ betterlog -c -pd -n 10 unicorn.log
+            $ betterlog -c -pd -n 10 unicorn.log
 
-        - Filter the last 10 lines of file unicorn.log as JSON events:
+          - Filter the last 10 lines of file unicorn.log as JSON events:
 
-          $ betterlog -n 10 unicorn.log
+            $ betterlog -n 10 unicorn.log
 
+      end
+      exit(0)
     end
-    exit(0)
-  end
 
-  private\
-  def filter_severities
-    @severities = Log::Severity.all
-    if severity = @opts[?S]
-      severity.each do |s|
-        if s =~ /\A(>=?|<=?)(.+)/
-          gs = Log::Severity.new($2)
-          @severities.select! { |x| x.send($1, gs) }
-        else
-          gs = Log::Severity.new(s)
-          @severities.select! { |x| x == gs }
+    private\
+    def filter_severities
+      @severities = Log::Severity.all
+      if severity = @opts[?S]
+        severity.each do |s|
+          if s =~ /\A(>=?|<=?)(.+)/
+            gs = Log::Severity.new($2)
+            @severities.select! { |x| x.send($1, gs) }
+          else
+            gs = Log::Severity.new(s)
+            @severities.select! { |x| x == gs }
+          end
         end
       end
     end
-  end
-
-  def prog
-    File.basename($0)
-  end
 
-  def emitters
-    Array(@opts[?e])
-  end
+    def prog
+      File.basename($0)
+    end
 
-  def search_matched?(event)
-    case @opts[?s]
-    when /:\?\z/
-      event[$`].present?
-    when /:([^:]+)\z/
-      event[$`].full?(:include?, $1)
-    when String
-      event.to_json.include?(@opts[?s])
-    else
-      return true
+    def emitters
+      Array(@opts[?e])
     end
-  end
 
-  def output_log_event(prefix, event)
-    return unless @severities.include?(event.severity)
-    return if emitters.full? && !emitters.include?(event.emitter)
-    search_matched?(event) or return
-    if format = @opts[?p]
-      puts event.format(pretty: :format, color: @opts[?c], format: format)
-    else
-      puts "#{prefix}#{event}"
+    def search_matched?(event)
+      case @opts[?s]
+      when /:\?\z/
+        event[$`].present?
+      when /:([^:]+)\z/
+        event[$`].full?(:include?, $1)
+      when String
+        event.to_json.include?(@opts[?s])
+      else
+        return true
+      end
     end
-  end
 
-  def output_log_line(l, filename)
-    l.blank? and return
-    prefix =
-      if filename && @args.size > 1
-        "#{filename}: "
+    def output_log_event(prefix, event)
+      return unless @severities.include?(event.severity)
+      return if emitters.full? && !emitters.include?(event.emitter)
+      search_matched?(event) or return
+      if format = @opts[?p]
+        puts event.format(pretty: :format, color: @opts[?c], format: format)
+      else
+        puts "#{prefix}#{event}"
       end
-    if event = Log::Event.parse(l)
-      filename and event[:file] = filename
-      output_log_event(prefix, event)
-    elsif l =~ /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3})\d* (.*)/
-      event = Log::Event.new(
-        timestamp: $1,
-        message:   Term::ANSIColor.uncolor($2),
-        type:      'isoprefix',
-      )
-      filename and event[:file] = filename
-      output_log_event(prefix, event)
-    else
-      @opts[?e] or puts "#{prefix}#{l}"
     end
-  rescue
-    @opts[?e] or puts "#{prefix}#{l}"
-  end
 
-  def query_config_file_configuration
-    if @opts[?F]
-      if cfs = cc.log.config_files?
-        @opts[?F].each do |f|
-          @args.concat cfs[f]
+    def output_log_line(l, filename)
+      l.blank? and return
+      prefix =
+        if filename && @args.size > 1
+          "#{filename}: "
         end
+      if event = Log::Event.parse(l)
+        filename and event[:file] = filename
+        output_log_event(prefix, event)
+      elsif l =~ /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3})\d* (.*)/
+        event = Log::Event.new(
+          timestamp: $1,
+          message:   Term::ANSIColor.uncolor($2),
+          type:      'isoprefix',
+        )
+        filename and event[:file] = filename
+        output_log_event(prefix, event)
       else
-        fail "no config files for #{@opts[?F]} defined"
-      end
-    else
-      if @args.empty? and r = cc.log.config_files?&.rails?
-        @args.concat r
-      end
-      if @args.empty?
-        fail "filenames to follow needed"
+        @opts[?e] or puts "#{prefix}#{l}"
       end
+    rescue
+      @opts[?e] or puts "#{prefix}#{l}"
     end
-    @args.uniq!
-  end
 
-  def follow_files
-    group = File::Tail::Group.new
-    @args.each do |f|
-      if File.exist?(f)
-        group.add_filename f, @opts[?n].to_i
+    def query_config_file_configuration
+      if @opts[?F]
+        if cfs = cc.log.config_files?
+          @opts[?F].each do |f|
+            @args.concat cfs[f]
+          end
+        else
+          fail "no config files for #{@opts[?F]} defined"
+        end
       else
-        STDERR.puts "file #{f.inspect} does not exist, skip it!"
+        if @args.empty? and r = cc.log.config_files?&.rails?
+          @args.concat r
+        end
+        if @args.empty?
+          fail "filenames to follow needed"
+        end
       end
+      @args.uniq!
     end
-    group.each_file { |f| f.max_interval = 1 }
-    t = Thread.new do
-      group.tail { |l| output_log_line(l, l.file.path) }
-    end
-    t.join
-  rescue Interrupt
-  end
 
-  def filter_argv
-    for fn in @args
-      unless File.exist?(fn)
-        STDERR.puts "file #{fn.inspect} does not exist, skip it!"
-        next
+    def follow_files
+      group = File::Tail::Group.new
+      @args.each do |f|
+        if File.exist?(f)
+          group.add_filename f, @opts[?n].to_i
+        else
+          STDERR.puts "file #{f.inspect} does not exist, skip it!"
+        end
       end
-      if fn.end_with?('.gz')
-        Zlib::GzipReader.open(fn) do |f|
-          f.extend(File::Tail)
-          f.each_line do |l|
-            output_log_line(l, fn)
-          end
+      group.each_file { |f| f.max_interval = 1 }
+      t = Thread.new do
+        group.tail { |l| output_log_line(l, l.file.path) }
+      end
+      t.join
+    rescue Interrupt
+    end
+
+    def filter_argv
+      for fn in @args
+        unless File.exist?(fn)
+          STDERR.puts "file #{fn.inspect} does not exist, skip it!"
+          next
         end
-      else
-        File::Tail::Logfile.open(fn, backward: @opts[?n].to_i) do |f|
-          f.each_line do |l|
-            output_log_line(l, fn)
+        if fn.end_with?('.gz')
+          Zlib::GzipReader.open(fn) do |f|
+            f.extend(File::Tail)
+            f.each_line do |l|
+              output_log_line(l, fn)
+            end
+          end
+        else
+          File::Tail::Logfile.open(fn, backward: @opts[?n].to_i) do |f|
+            f.each_line do |l|
+              output_log_line(l, fn)
+            end
           end
         end
       end
     end
-  end
 
-  def filter_stdin
-    STDIN.each_line do |l|
-      output_log_line(l, nil)
+    def filter_stdin
+      STDIN.each_line do |l|
+        output_log_line(l, nil)
+      end
     end
-  end
 
-  def output_log_sources
-    if @args.empty?
-      STDERR.puts "#{prog} tracking stdin\nseverities: #{@severities * ?|}"
-    else
-      STDERR.puts "#{prog} tracking files:\n"\
-        "#{@args.map { |a| '  ' + a.inspect }.join(' ')}\n"\
-        "severities: #{@severities * ?|}\n"
+    def output_log_sources
+      if @args.empty?
+        STDERR.puts "#{prog} tracking stdin\nseverities: #{@severities * ?|}"
+      else
+        STDERR.puts "#{prog} tracking files:\n"\
+          "#{@args.map { |a| '  ' + a.inspect }.join(' ')}\n"\
+          "severities: #{@severities * ?|}\n"
+      end
     end
-  end
 
-  def run
-    if @opts[?f]
-      query_config_file_configuration
-      output_log_sources
-      follow_files
-    elsif @opts[?F] && @args.empty?
-      query_config_file_configuration
-      output_log_sources
-      filter_argv
-    elsif !@args.empty?
-      output_log_sources
-      filter_argv
-    else
-      output_log_sources
-      filter_stdin
+    def run
+      if @opts[?f]
+        query_config_file_configuration
+        output_log_sources
+        follow_files
+      elsif @opts[?F] && @args.empty?
+        query_config_file_configuration
+        output_log_sources
+        filter_argv
+      elsif !@args.empty?
+        output_log_sources
+        filter_argv
+      else
+        output_log_sources
+        filter_stdin
+      end
     end
   end
 end
 
 if File.basename($0) == File.basename(__FILE__)
-  Betterlog.new(ARGV).run
+  Betterlog::App.new(ARGV).run
 end

data/bin/betterlog_pusher

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+# vim: set ft=ruby et sw=2 ts=2:
+
+require 'betterlog'
+require 'excon'
+
+lines  = Integer(ENV.fetch('BETTERLOG_LINES', 1_000))
+url    = ENV.fetch('BETTERLOG_SERVER_URL')
+name   = ENV['BETTERLOG_NAME']
+redis  = Redis.new(url: ENV.fetch('REDIS_URL'))
+logger = Betterlog::Logger.new(redis, name: name)
+
+quit = false
+
+[ :TERM, :INT, :QUIT ].each { |s| trap(s) { quit = true } }
+
+STDOUT.sync = true
+loop do
+  count = 0
+  logger.each_slice(lines).with_index do |batch, i|
+    count.zero? and print ?(
+    count += batch.sum(&:size)
+    attempt(attempts: 10, sleep: -60, reraise: true) do
+      print ?┄
+      Excon.post(url, body: batch.join)
+    end
+  end
+  quit and exit
+  if count.zero?
+    sleep 1
+  else
+    print "→%s)" % Tins::Unit.format(count, format: '%.2f %U', prefix: 1024, unit: ?b)
+  end
+end

data/cmd/betterlog-server/LICENSE

@@ -0,0 +1,13 @@
+Copyright 2018 Florian Frank
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/cmd/betterlog-server/main.go

@@ -0,0 +1,165 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"strings"
+
+	betterlog "github.com/betterplace/betterlog/betterlog"
+	"github.com/go-redis/redis"
+	"github.com/kelseyhightower/envconfig"
+	"github.com/labstack/echo"
+	"github.com/labstack/echo/middleware"
+	"golang.org/x/crypto/acme/autocert"
+)
+
+type Config struct {
+	PORT         int    `default:"5514"`
+	HEALTHZ_PORT int    `default:"5513"`
+	HTTP_REALM   string `default:"betterlog"`
+	HTTP_AUTH    string
+	SSL          bool
+	REDIS_PREFIX string
+	REDIS_URL    string `default:"redis://localhost:6379"`
+}
+
+type RedisCertCache struct {
+	Redis  *redis.Client
+	PREFIX string
+}
+
+// Get reads certificate data from the specified key name.
+func (cache RedisCertCache) Get(ctx context.Context, name string) ([]byte, error) {
+	name = strings.Join([]string{cache.PREFIX, name}, "/")
+	done := make(chan struct{})
+	var (
+		err  error
+		data string
+	)
+	go func() {
+		defer close(done)
+		result := cache.Redis.Get(name)
+		err = result.Err()
+		if err == nil {
+			data, err = result.Result()
+		}
+	}()
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	case <-done:
+	}
+	if err == redis.Nil {
+		return nil, autocert.ErrCacheMiss
+	}
+	return []byte(data), err
+}
+
+// Put writes the certificate data to the specified redis key name.
+func (cache RedisCertCache) Put(ctx context.Context, name string, data []byte) error {
+	name = strings.Join([]string{cache.PREFIX, name}, "/")
+	done := make(chan struct{})
+	var err error
+	go func() {
+		defer close(done)
+		select {
+		case <-ctx.Done():
+			// Don't overwrite the key if the context was canceled.
+		default:
+			result := cache.Redis.Set(name, string(data), 0)
+			err = result.Err()
+		}
+	}()
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-done:
+	}
+	return err
+}
+
+// Delete removes the specified key name.
+func (cache RedisCertCache) Delete(ctx context.Context, name string) error {
+	name = strings.Join([]string{cache.PREFIX, name}, "/")
+	var (
+		err  error
+		done = make(chan struct{})
+	)
+	go func() {
+		defer close(done)
+		err = cache.Redis.Del(name).Err()
+	}()
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-done:
+	}
+	return err
+}
+
+func postLogHandler(c echo.Context) error {
+	body := c.Request().Body
+	data, err := ioutil.ReadAll(body)
+	if err == nil {
+		defer body.Close()
+		os.Stdout.Write(data)
+		return c.NoContent(http.StatusOK)
+	} else {
+		return c.String(http.StatusInternalServerError, err.Error())
+	}
+}
+
+func basicAuthConfig(config Config) middleware.BasicAuthConfig {
+	return middleware.BasicAuthConfig{
+		Realm: config.HTTP_REALM,
+		Validator: func(username, password string, c echo.Context) (bool, error) {
+			httpAuth := strings.Split(config.HTTP_AUTH, ":")
+			if username == httpAuth[0] && password == httpAuth[1] {
+				return true, nil
+			}
+			return false, nil
+		},
+	}
+}
+
+func initializeRedis(config Config) *redis.Client {
+	options, err := redis.ParseURL(config.REDIS_URL)
+	if err != nil {
+		log.Panic(err)
+	}
+	options.MaxRetries = 3
+	return redis.NewClient(options)
+}
+
+func main() {
+	var config Config
+	err := envconfig.Process("", &config)
+	if err != nil {
+		log.Fatal(err)
+	}
+	e := echo.New()
+	if config.HTTP_AUTH != "" {
+		fmt.Println("info: Configuring HTTP Auth access control")
+		e.Use(middleware.BasicAuthWithConfig(basicAuthConfig(config)))
+	}
+	e.POST("/log", postLogHandler)
+	if config.SSL {
+		log.Println("Starting SSL AutoTLS service.")
+		redis := initializeRedis(config)
+		e.AutoTLSManager.Cache = RedisCertCache{
+			Redis:  redis,
+			PREFIX: config.REDIS_PREFIX,
+		}
+		go betterlog.StartHealthzEcho(
+			betterlog.Health{
+				PORT: config.HEALTHZ_PORT,
+			})
+		e.Logger.Fatal(e.StartAutoTLS(fmt.Sprintf(":%d", config.PORT)))
+	} else {
+		e.Logger.Fatal(e.Start(fmt.Sprintf(":%d", config.PORT)))
+	}
+}

data/{log.yml → config/log.yml}

@@ -1,4 +1,4 @@
-default: &default
+development: &development
   styles:
     'timestamp': [ yellow, bold ]
     'file': [ blue, bold ]
@@ -32,9 +32,6 @@ default: &default
       {program}: {message}
     metric: >
       {%ft%timestamp} {metric} {value} {type}
-
-development:
-  <<: *default
   config_files:
     rails:
       - log/development.log
@@ -43,35 +40,8 @@ development:
     redis:
       - /usr/local/var/log/redis.log
     elasticsearch:
-      - /usr/local/var/log/elasticsearch@1.7.log
-    nginx:
-      - /usr/local/var/log/nginx/access.log
-      - /usr/local/var/log/nginx/error.log
-  legacy_supported: <%= ENV['LOG_LEGACY_SUPPORTED'].to_i == 1 %>
-
-test:
-  <<: *default
-  config_files:
-    test:
-      - log/test.log
-  legacy_supported: <%= ENV['LOG_LEGACY_SUPPORTED'].to_i == 1 %>
-
-production:
-  <<: *default
-  config_files:
-    rails:
-      - /var/apps/betterplace/current/log/production.log
-    cron:
-      - /var/apps/betterplace/current/log/cron.log
-    nginx:
-      - /var/apps/betterplace/current/log/nginx.assets.access.log
-      - /var/apps/betterplace/current/log/nginx.assets.error.log
-      - /var/apps/betterplace/current/log/nginx.betterplace.access.log
-      - /var/apps/betterplace/current/log/nginx.betterplace.error.log
-      - /var/apps/betterplace/current/log/nginx.default.access.log
-      - /var/apps/betterplace/current/log/nginx.default.error.log
-    memosig:
-      - /var/log/memosig/current
-    unicorn:
-      - /var/log/unicorn/current
+      - /usr/local/var/log/elasticsearch.log
   legacy_supported: yes
+test: *development
+staging: *development
+production: *development