logtail 0.1.19 → 0.1.21

data/spec/logtail/logger_flush_all_spec.rb

@@ -0,0 +1,82 @@
+require "spec_helper"
+
+# Rails flushes Rails.logger after every request (Rails::Rack::Logger calls
+# ActiveSupport::LogSubscriber.flush_all!), and Logger#flush returns right away for that call
+# instead of waiting for delivery. These run ActiveSupport's real flush_all! in another process, so
+# an ActiveSupport that calls flush differently fails here. The ingesting host holds its answers
+# until the direct flush, so a flush_all! that waited for delivery would take 5 seconds.
+describe Logtail::Logger, "#flush called by ActiveSupport::LogSubscriber.flush_all!" do
+  {
+    "a Logtail::Logger" => "logger",
+    "a Logtail::Logger in ActiveSupport::TaggedLogging" => "ActiveSupport::TaggedLogging.new(logger)",
+    "a Logtail::Logger in an ActiveSupport::BroadcastLogger, as Rails 7.1 and later wrap it" =>
+      "ActiveSupport::BroadcastLogger.new(logger)",
+    "a Logtail::Logger in ActiveSupport::TaggedLogging in an ActiveSupport::BroadcastLogger" =>
+      "ActiveSupport::BroadcastLogger.new(ActiveSupport::TaggedLogging.new(logger))"
+  }.each do |description, rails_logger|
+    it "returns right away for #{description}, and flushing it directly delivers" do
+      if rails_logger.include?("BroadcastLogger") && Gem.loaded_specs["activesupport"].version < Gem::Version.new("7.1")
+        skip "ActiveSupport::BroadcastLogger was added in ActiveSupport 7.1"
+      end
+
+      result = run_ruby(<<-RUBY)
+        require "logger" # ActiveSupport 6.1 needs it loaded first since concurrent-ruby 1.3.5
+        require "active_support"
+        require "active_support/log_subscriber"
+        require "active_support/tagged_logging"
+        require "json"
+        require "logtail"
+
+        # The ingesting host keeps the lines of every request, but answers only once `answering` is set
+        delivered = []
+        answering = false
+        server = TCPServer.new("127.0.0.1", 0)
+        Thread.new do
+          loop do
+            Thread.new(server.accept) do |socket|
+              while socket.gets
+                length = 0
+                while (header = socket.gets) != "\\r\\n"
+                  length = header.split(":")[1].to_i if header.downcase.start_with?("content-length:")
+                end
+                delivered.concat(MessagePack.unpack(Zlib::Inflate.inflate(socket.read(length))).map { |line| line["message"] })
+                sleep 0.01 until answering
+                socket.write("HTTP/1.1 202 Accepted\\r\\nContent-Length: 0\\r\\n\\r\\n")
+              end
+            end
+          end
+        end
+
+        # Logs a line for every request, like Rails' ActionController::LogSubscriber
+        class RequestLogSubscriber < ActiveSupport::LogSubscriber
+          def request(event)
+            info(event.payload[:message])
+          end
+        end
+        RequestLogSubscriber.attach_to(:app)
+
+        logger = Logtail::Logger.new(Logtail::LogDevices::HTTP.new("token", flush_interval: 60,
+          ingesting_host: "127.0.0.1", ingesting_port: server.addr[1], ingesting_scheme: "http"))
+        ActiveSupport::LogSubscriber.logger = #{rails_logger}
+        ActiveSupport::Notifications.instrument("request.app", message: "Completed 200 OK")
+
+        flushing = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        ActiveSupport::LogSubscriber.flush_all!
+        flush_all_seconds = Process.clock_gettime(Process::CLOCK_MONOTONIC) - flushing
+        delivered_by_flush_all = delivered.dup
+
+        answering = true
+        ActiveSupport::LogSubscriber.logger.flush
+        puts JSON.generate("flush_all_seconds" => flush_all_seconds, "delivered_by_flush_all" => delivered_by_flush_all,
+          "delivered_by_flush" => delivered.drop(delivered_by_flush_all.size))
+      RUBY
+
+      expect(result.status).to be_success, result.stderr
+      expect(JSON.parse(result.stdout)).to match(
+        "flush_all_seconds" => a_value < 1,
+        "delivered_by_flush_all" => [],
+        "delivered_by_flush" => ["Completed 200 OK"]
+      )
+    end
+  end
+end

data/spec/logtail/logger_spec.rb

@@ -78,6 +78,19 @@ describe Logtail::Logger do
       expect(io.string).to eq("")
     end
 
+    # Ruby's Logger formats these severities as "ANY"
+    it "should log a nil severity at the unknown level" do
+      logger.add(nil, "log message")
+      expect(io.string).to include("log message")
+      expect(io.string).to include('"level":"unknown"')
+    end
+
+    it "should log a severity above UNKNOWN at the unknown level" do
+      logger.add(99, "log message")
+      expect(io.string).to include("log message")
+      expect(io.string).to include('"level":"unknown"')
+    end
+
     it "should not lose message when logging hashes with multiple io devices" do
       io1 = StringIO.new
       io2 = StringIO.new
@@ -147,6 +160,20 @@ describe Logtail::Logger do
         expect(tags).to eq(["tag1", "tag2"])
       end
 
+      it "should allow a single String in :tags" do
+        logger.info("event complete", tags: "tag1")
+        expect(io.string).to include("\"tags\":[\"tag1\"]")
+      end
+
+      it "should skip nil tags" do
+        logger.info("event complete", tag: nil, tags: nil)
+        logger.info("event complete", tags: ["tag1", nil])
+        lines = io.string.lines
+        expect(lines[0]).to start_with("event complete @metadata")
+        expect(lines[0]).not_to include("\"tags\"")
+        expect(lines[1]).to include("\"tags\":[\"tag1\"]")
+      end
+
       it "should allow functions" do
         logger.info do
           {message: "payment rejected", payment_rejected: {customer_id: "abcde1234", amount: 100}}
@@ -179,6 +206,21 @@ describe Logtail::Logger do
     end
   end
 
+  describe "#<<" do
+    it "logs the line at the info level" do
+      http_device = Logtail::LogDevices::HTTP.new("MYKEY", flush_continuously: false)
+      logger = Logtail::Logger.new(http_device)
+
+      logger << "appended line\n"
+
+      http_device.send(:flush_async)
+      request = http_device.instance_variable_get(:@request_queue).deq.request
+      entries = MessagePack.unpack(Zlib::Inflate.inflate(request.body))
+      expect(entries.size).to eq(1)
+      expect(entries[0]).to include("level" => "info", "message" => "appended line")
+    end
+  end
+
   describe "#error" do
     let(:io) { StringIO.new }
     let(:logger) { Logtail::Logger.new(io) }
@@ -197,6 +239,41 @@ describe Logtail::Logger do
     end
   end
 
+  describe "#unknown" do
+    let(:io) { StringIO.new }
+    let(:logger) { Logtail::Logger.new(io) }
+
+    it "should allow default usage" do
+      logger.unknown("log message")
+      expect(io.string).to include("log message")
+      expect(io.string).to include('"level":"unknown"')
+    end
+  end
+
+  describe "#flush" do
+    let(:device) { Logtail::LogDevices::HTTP.new("my_source_token") }
+    let(:extra_device) { Logtail::LogDevices::HTTP.new("my_source_token") }
+    let(:logger) { Logtail::Logger.new(device, extra_device) }
+
+    it "flushes the log devices, so that what was logged is delivered" do
+      expect(device).to receive(:flush)
+      expect(extra_device).to receive(:flush)
+      logger.flush
+    end
+
+    it "returns right away when Rails flushes it after every request" do
+      # Stands in for ActiveSupport::LogSubscriber.flush_all!, which calls Rails.logger.flush
+      log_subscriber = Module.new do
+        def self.flush_all!(logger)
+          logger.flush
+        end
+      end
+      expect(device).not_to receive(:flush)
+      expect(extra_device).not_to receive(:flush)
+      log_subscriber.flush_all!(logger)
+    end
+  end
+
   describe "#formatter=" do
     it "should not allow changing the formatter when the device is HTTP" do
       http_device = Logtail::LogDevices::HTTP.new("source_token")

data/spec/spec_helper.rb

@@ -12,6 +12,7 @@ require File.join(File.dirname(__FILE__), 'support', 'socket_hostname')
 require File.join(File.dirname(__FILE__), 'support', 'timecop')
 require File.join(File.dirname(__FILE__), 'support', 'webmock')
 require File.join(File.dirname(__FILE__), 'support', 'logtail')
+require File.join(File.dirname(__FILE__), 'support', 'processes')
 
 RSpec::Support::ObjectFormatter.default_instance.max_formatted_output_length = 5_000
 

data/spec/support/processes.rb

@@ -0,0 +1,91 @@
+require "msgpack"
+require "rbconfig"
+require "socket"
+require "tmpdir"
+require "zlib"
+
+# A local stand-in for the Better Stack ingesting host, for tests that run logtail in a separate
+# process: it answers every request with 202 and records the message of every line it receives,
+# and how many lines each request held.
+class LocalIngestServer
+  def initialize
+    @server = TCPServer.new("127.0.0.1", 0)
+    @messages = []
+    @batch_sizes = []
+    @lock = Mutex.new
+    @connections = []
+    @thread = Thread.new do
+      loop { @connections << Thread.new(@server.accept) { |socket| serve(socket) } }
+    end
+  end
+
+  # The options for Logtail::LogDevices::HTTP.new that deliver to this server, as Ruby code.
+  def device_options
+    %(ingesting_host: "127.0.0.1", ingesting_port: #{@server.addr[1]}, ingesting_scheme: "http")
+  end
+
+  def messages
+    @lock.synchronize { @messages.dup }
+  end
+
+  def batch_sizes
+    @lock.synchronize { @batch_sizes.dup }
+  end
+
+  def stop
+    @thread.kill.join
+    @connections.each { |connection| connection.kill.join }
+    @server.close
+  end
+
+  private
+
+  def serve(socket)
+    while socket.gets
+      headers = {}
+      while (header = socket.gets) && header != "\r\n"
+        name, value = header.split(":", 2)
+        headers[name.downcase] = value.strip
+      end
+      lines = MessagePack.unpack(Zlib::Inflate.inflate(socket.read(headers["content-length"].to_i)))
+      @lock.synchronize do
+        @messages.concat(lines.map { |line| line["message"] })
+        @batch_sizes << lines.size
+      end
+      socket.write("HTTP/1.1 202 Accepted\r\nContent-Length: 0\r\n\r\n")
+    end
+  rescue IOError, SystemCallError
+  ensure
+    socket.close
+  end
+end
+
+module ProcessHelpers
+  ProcessResult = Struct.new(:stdout, :stderr, :status)
+
+  # Runs the Ruby script in a new process with logtail from lib/ and waits for it to exit, at
+  # most `timeout` seconds (generous: TruffleRuby starts slowly on CI).
+  def run_ruby(script, timeout: 120)
+    Dir.mktmpdir do |dir|
+      File.write(File.join(dir, "script.rb"), script)
+      lib = File.expand_path("../../lib", __dir__)
+      pid = Process.spawn(RbConfig.ruby, "-I", lib, File.join(dir, "script.rb"),
+        out: File.join(dir, "stdout"), err: File.join(dir, "stderr"))
+      deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
+      status = nil
+      until (status = Process.waitpid2(pid, Process::WNOHANG))
+        if Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
+          Process.kill("KILL", pid)
+          Process.wait(pid)
+          raise "The Ruby process didn't exit within #{timeout} seconds:\n#{File.read(File.join(dir, "stderr"))}"
+        end
+        sleep 0.05
+      end
+      ProcessResult.new(File.read(File.join(dir, "stdout")), File.read(File.join(dir, "stderr")), status[1])
+    end
+  end
+end
+
+RSpec.configure do |config|
+  config.include ProcessHelpers
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logtail
 version: !ruby/object:Gem::Version
-  version: 0.1.19
+  version: 0.1.21
 platform: ruby
 authors:
 - Better Stack
@@ -188,14 +188,20 @@ files:
 - lib/logtail/version.rb
 - logtail.gemspec
 - spec/README.md
+- spec/logtail/config_spec.rb
 - spec/logtail/current_context_spec.rb
 - spec/logtail/events/controller_call_spec.rb
 - spec/logtail/events/error_spec.rb
+- spec/logtail/json_encoding_spec.rb
+- spec/logtail/log_devices/http_exit_spec.rb
+- spec/logtail/log_devices/http_fork_spec.rb
 - spec/logtail/log_devices/http_spec.rb
 - spec/logtail/log_entry_spec.rb
+- spec/logtail/logger_flush_all_spec.rb
 - spec/logtail/logger_spec.rb
 - spec/spec_helper.rb
 - spec/support/logtail.rb
+- spec/support/processes.rb
 - spec/support/socket_hostname.rb
 - spec/support/timecop.rb
 - spec/support/webmock.rb