logtail 0.1.0

data/lib/logtail/logtail.rb

@@ -0,0 +1,36 @@
+# Base (must come first, order matters)
+require "logtail/version"
+require "logtail/config"
+require "logtail/util"
+
+# Load frameworks
+
+# Other (sorted alphabetically)
+require "logtail/contexts"
+require "logtail/current_context"
+require "logtail/events"
+require "logtail/integration"
+require "logtail/log_devices"
+require "logtail/log_entry"
+require "logtail/logger"
+require "logtail/timer"
+require "logtail/integrator"
+require "logtail/integration"
+
+module Logtail
+  # Access the main configuration object. Please see {{Logtail::Config}} for more details.
+  def self.config
+    Config.instance
+  end
+
+  # Starts a timer for timing events. Please see {{Logtail::Logtail.start}} for more details.
+  def self.start_timer
+    Timer.start
+  end
+
+  # Adds context to all logs written within the passed block. Please see
+  # {{Logtail::CurrentContext.with}} for a more detailed description with examples.
+  def self.with_context(context, &block)
+    CurrentContext.with(context, &block)
+  end
+end

data/lib/logtail/timer.rb

@@ -0,0 +1,21 @@
+module Logtail
+  # This is an ultra-simple abstraction for timing code. This provides a little
+  # more control around how Logtail automatically processes "timers".
+  #
+  # @example
+  #   timer = Logtail::Timer.start
+  #   # ... code to time
+  #   logger.info("My log message", my_event: {time_ms: timer})
+  module Timer
+    # Abstract for starting a logtail. Currently this is simply calling `Time.now`.
+    def self.start
+      Time.now
+    end
+
+    # Get the duration in milliseconds from the object returned in {#start}
+    def self.duration_ms(timer)
+      now = Time.now
+      (now - timer) * 1000.0
+    end
+  end
+end

data/lib/logtail/util.rb

@@ -0,0 +1,7 @@
+require "logtail/util/non_nil_hash_builder"
+
+module Logtail
+  # @private
+  module Util
+  end
+end

data/lib/logtail/util/non_nil_hash_builder.rb

@@ -0,0 +1,40 @@
+require 'json'
+
+module Logtail
+  module Util
+    # @private
+    #
+    # The purpose of this class is to efficiently build a hash that does not
+    # include nil values. It's proactive instead of reactive, avoiding the
+    # need to traverse and reduce a new hash dropping blanks.
+    class NonNilHashBuilder
+      class << self
+        def build(&block)
+          builder = new
+          yield builder
+          builder.target
+        end
+      end
+
+      attr_reader :target
+
+      def initialize
+        @target = {}
+      end
+
+      def add(k, v, options = {})
+        if !v.nil?
+          if options[:json_encode]
+            v = v.to_json
+          end
+
+          if options[:limit]
+            v = v.byteslice(0, options[:limit])
+          end
+
+          @target[k] = v
+        end
+      end
+    end
+  end
+end

data/lib/logtail/version.rb

@@ -0,0 +1,3 @@
+module Logtail
+  VERSION = "0.1.0"
+end

data/logtail.gemspec

@@ -0,0 +1,43 @@
+# -*- encoding: utf-8 -*-
+$LOAD_PATH.push File.expand_path("../lib", __FILE__)
+require "logtail/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "logtail"
+  spec.version       = Logtail::VERSION
+  spec.platform      = Gem::Platform::RUBY
+  spec.authors       = ["Logtail"]
+  spec.email         = ["hi@logtail.com"]
+  spec.homepage      = "https://github.com/logtail/logtail-ruby"
+  spec.license       = "ISC"
+
+  spec.summary       = "Query logs like you query your database. https://logtail.com"
+
+  spec.metadata["bug_tracker_uri"] = "#{spec.homepage}/issues"
+  spec.metadata["changelog_uri"] = "#{spec.homepage}/tree/master/CHANGELOG.md"
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.2.0")
+
+  spec.files         = `git ls-files`.split("\n")
+  spec.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  spec.executables   = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency('msgpack', '~> 1.0')
+
+  spec.add_development_dependency('bundler-audit', '>= 0')
+  spec.add_development_dependency('rails_stdout_logging', '>= 0')
+  spec.add_development_dependency('rake', '>= 0')
+  spec.add_development_dependency('rspec', '~> 3.4')
+  spec.add_development_dependency('rspec-its', '>= 0')
+  spec.add_development_dependency('timecop', '>= 0')
+  spec.add_development_dependency('webmock', '~> 2.3')
+
+  if RUBY_PLATFORM == "java"
+    spec.add_development_dependency('activerecord-jdbcsqlite3-adapter', '>= 0')
+  else
+    spec.add_development_dependency('sqlite3', '>= 0')
+  end
+end

data/spec/README.md

@@ -0,0 +1,13 @@
+# Testing
+
+To get started:
+
+```shell
+bundle install
+```
+
+You can run tests like this:
+
+```shell
+bundle exec rspec
+```

data/spec/logtail/current_context_spec.rb

@@ -0,0 +1,113 @@
+require "spec_helper"
+
+describe Logtail::CurrentContext do
+  describe ".initialize" do
+    it "should not set the release context" do
+      context = described_class.send(:new)
+      expect(context.send(:hash)[:release]).to be_nil
+    end
+
+    it "should set the system context" do
+      context = described_class.send(:new)
+      system_content = context.fetch(:system)
+      expect(system_content[:hostname]).to_not be_nil
+    end
+
+    it "should set the runtime context" do
+      context = described_class.send(:new)
+      runtime_context = context.fetch(:runtime)
+      expect(runtime_context[:thread_id]).to_not be_nil
+    end
+
+    context "with Heroku dyno metadata" do
+      around(:each) do |example|
+        ENV['HEROKU_SLUG_COMMIT'] = "2c3a0b24069af49b3de35b8e8c26765c1dba9ff0"
+        ENV['HEROKU_RELEASE_CREATED_AT'] = "2015-04-02T18:00:42Z"
+        ENV['HEROKU_RELEASE_VERSION'] = "v2.3.1"
+
+        example.run
+
+        ENV.delete('HEROKU_SLUG_COMMIT')
+        ENV.delete('HEROKU_RELEASE_CREATED_AT')
+        ENV.delete('HEROKU_RELEASE_VERSION')
+
+        described_class.reset
+      end
+
+      it "should automatically set the release context" do
+        context = described_class.send(:new)
+        expect(context.send(:hash)[:release]).to eq({:commit_hash=>"2c3a0b24069af49b3de35b8e8c26765c1dba9ff0", :created_at=>"2015-04-02T18:00:42Z", :version=>"v2.3.1"})
+      end
+    end
+
+    context "with generic env vars" do
+      around(:each) do |example|
+        ENV['RELEASE_COMMIT'] = "2c3a0b24069af49b3de35b8e8c26765c1dba9ff0"
+        ENV['RELEASE_CREATED_AT'] = "2015-04-02T18:00:42Z"
+        ENV['RELEASE_VERSION'] = "v2.3.1"
+
+        example.run
+
+        ENV.delete('RELEASE_COMMIT')
+        ENV.delete('RELEASE_CREATED_AT')
+        ENV.delete('RELEASE_VERSION')
+
+        described_class.reset
+      end
+
+      it "should automatically set the release context" do
+        context = described_class.send(:new)
+        expect(context.send(:hash)[:release]).to eq({:commit_hash=>"2c3a0b24069af49b3de35b8e8c26765c1dba9ff0", :created_at=>"2015-04-02T18:00:42Z", :version=>"v2.3.1"})
+      end
+    end
+  end
+
+  describe ".add" do
+    after(:each) do
+      described_class.reset
+    end
+
+    it "should add the context" do
+      expect(described_class.instance.send(:hash)[:build]).to be_nil
+
+      described_class.add({build: {version: "1.0.0"}})
+      expect(described_class.instance.send(:hash)[:build]).to eq({:version=>"1.0.0"})
+
+      described_class.add({testing: {key: "value"}})
+      expect(described_class.instance.send(:hash)[:build]).to eq({:version=>"1.0.0"})
+      expect(described_class.instance.send(:hash)[:testing]).to eq({:key=>"value"})
+    end
+  end
+
+  describe ".remove" do
+    it "should remove context by key" do
+      context = {:build=>{:version=>"1.0.0"}}
+      described_class.add(context)
+      expect(described_class.instance.send(:hash)[:build]).to eq({:version=>"1.0.0"})
+
+      described_class.remove(:build)
+      expect(described_class.instance.send(:hash)[:build]).to be_nil
+    end
+  end
+
+  describe ".with" do
+    it "should merge the context and cleanup on block exit" do
+      expect(described_class.instance.send(:hash)[:build]).to be_nil
+
+      described_class.with({build: {version: "1.0.0"}}) do
+        expect(described_class.instance.send(:hash)[:build]).to eq({:version=>"1.0.0"})
+
+        described_class.with({testing: {key: "value"}}) do
+          expect(described_class.instance.send(:hash)[:build]).to eq({:version=>"1.0.0"})
+          expect(described_class.instance.send(:hash)[:testing]).to eq({:key=>"value"})
+        end
+
+        expect(described_class.instance.send(:hash)[:build]).to eq({:version=>"1.0.0"})
+        expect(described_class.instance.send(:hash)[:testing]).to be_nil
+      end
+
+      expect(described_class.instance.send(:hash)[:build]).to be_nil
+      expect(described_class.instance.send(:hash)[:testing]).to be_nil
+    end
+  end
+end

data/spec/logtail/events/controller_call_spec.rb

@@ -0,0 +1,12 @@
+# encoding: UTF-8
+
+require "spec_helper"
+
+describe Logtail::Events::ControllerCall do
+  describe ".initialize" do
+    it "sanitizes the password param" do
+      # event = described_class.new(controller: 'controller', action: 'action', params: {password: 'password'})
+      # expect(event.params).to eq({'password' => '[sanitized]'})
+    end
+  end
+end

data/spec/logtail/events/error_spec.rb

@@ -0,0 +1,15 @@
+require "spec_helper"
+
+describe Logtail::Events::Error do
+  describe "#to_hash" do
+    it "should jsonify the stacktrace" do
+      backtrace = [
+        "/path/to/file1.rb:26:in `function1'",
+        "path/to/file2.rb:86:in `function2'"
+      ]
+
+      exception_event = described_class.new(name: "RuntimeError", error_message: "Boom", backtrace: backtrace)
+      expect(exception_event.backtrace_json).to eq("[\"/path/to/file1.rb:26:in `function1'\",\"path/to/file2.rb:86:in `function2'\"]")
+    end
+  end
+end

data/spec/logtail/log_devices/http_spec.rb

@@ -0,0 +1,185 @@
+require "spec_helper"
+
+# Note: these tests access instance variables and private methods as a means of
+# not muddying the public API. This object should expose a simple buffer like
+# API, tests should not alter that.
+describe Logtail::LogDevices::HTTP do
+  describe "#initialize" do
+    it "should initialize properly" do
+      http = described_class.new("MYKEY", flush_interval: 0.1)
+
+      # Ensure that threads have not started
+      thread = http.instance_variable_get(:@flush_thread)
+      expect(thread).to be_nil
+      thread = http.instance_variable_get(:@request_outlet_thread)
+      expect(thread).to be_nil
+    end
+  end
+
+  describe "#write" do
+    let(:http) { described_class.new("MYKEY") }
+    let(:msg_queue) { http.instance_variable_get(:@msg_queue) }
+
+    it "should buffer the messages" do
+      http.write("test log message")
+      expect(msg_queue.flush).to eq(["test log message"])
+      http.close
+    end
+
+    it "should start the flush threads" do
+      http.write("test log message")
+
+      thread = http.instance_variable_get(:@flush_thread)
+      expect(thread).to be_alive
+      thread = http.instance_variable_get(:@request_outlet_thread)
+      expect(thread).to be_alive
+      expect(http).to receive(:flush).exactly(1).times
+      http.close
+    end
+
+    context "with a low batch size" do
+      let(:http) { described_class.new("MYKEY", :batch_size => 2) }
+
+      it "should attempt a delivery when the limit is exceeded" do
+        http.write("test")
+        expect(http).to receive(:flush_async).exactly(1).times
+        http.write("my log message")
+        expect(http).to receive(:flush).exactly(1).times
+        http.close
+      end
+    end
+  end
+
+  describe "#close" do
+    let(:http) { described_class.new("MYKEY") }
+
+    it "should kill the threads" do
+      http.send(:ensure_flush_threads_are_started)
+      http.close
+      thread = http.instance_variable_get(:@flush_thread)
+      sleep 0.1 # too fast!
+      expect(thread).to_not be_alive
+      thread = http.instance_variable_get(:@request_outlet_thread)
+      sleep 0.1 # too fast!
+      expect(thread).to_not be_alive
+    end
+
+    it "should attempt a delivery" do
+      message = "a" * 19
+      http.write(message)
+      expect(http).to receive(:flush).exactly(1).times
+      http.close
+    end
+  end
+
+  # Testing a private method because it helps break down our tests
+  describe "#flush" do
+    let(:time) { Time.utc(2016, 9, 1, 12, 0, 0) }
+
+    it "should deliver the request" do
+      http = described_class.new("MYKEY", flush_continuously: false)
+      log_entry = Logtail::LogEntry.new("INFO", time, nil, "test log message 1", nil, nil)
+      http.write(log_entry)
+      log_entry = Logtail::LogEntry.new("INFO", time, nil, "test log message 2", nil, nil)
+      http.write(log_entry)
+      expect(http).to receive(:flush_async).exactly(2).times
+      http.send(:flush)
+      http.close
+    end
+  end
+
+  # Testing a private method because it helps break down our tests
+  describe "#flush_async" do
+    let(:time) { Time.utc(2016, 9, 1, 12, 0, 0) }
+
+    it "should add a request to the queue" do
+      http = described_class.new("MYKEY", flush_continuously: false)
+      log_entry = Logtail::LogEntry.new("INFO", time, nil, "test log message 1", nil, nil)
+      http.write(log_entry)
+      log_entry = Logtail::LogEntry.new("INFO", time, nil, "test log message 2", nil, nil)
+      http.write(log_entry)
+      http.send(:flush_async)
+      request_queue = http.instance_variable_get(:@request_queue)
+      request_attempt = request_queue.deq
+      expect(request_attempt.request).to be_kind_of(Net::HTTP::Post)
+      expect(request_attempt.request.body).to start_with("\x92\x83\xA5level\xA4INFO\xA2dt\xBB2016-09-01T12:00:00.000000Z\xA7message\xB2test log message 1".force_encoding("ASCII-8BIT"))
+
+      message_queue = http.instance_variable_get(:@msg_queue)
+      expect(message_queue.size).to eq(0)
+    end
+  end
+
+  # Testing a private method because it helps break down our tests
+  describe "#intervaled_flush" do
+    it "should start a intervaled flush thread and flush on an interval" do
+      http = described_class.new("MYKEY", flush_interval: 0.1)
+      http.send(:ensure_flush_threads_are_started)
+      expect(http).to receive(:flush_async).at_least(3).times
+      sleep 1.1 # iterations check every 0.5 seconds
+      http.close
+    end
+  end
+
+  # Outlet
+  describe "#request_outlet" do
+    let(:time) { Time.utc(2016, 9, 1, 12, 0, 0) }
+
+    it "should deliver requests on an interval" do
+      stub = stub_request(:post, "https://in.logtail.com/").
+        with(
+          :body => start_with("\x92\x83\xA5level\xA4INFO\xA2dt\xBB2016-09-01T12:00:00.000000Z\xA7message\xB2test log message 1".force_encoding("ASCII-8BIT")),
+          :headers => {
+            'Authorization' => 'Bearer MYKEY',
+            'Content-Type' => 'application/msgpack',
+            'User-Agent' => "Logtail Ruby/#{Logtail::VERSION} (HTTP)"
+          }
+        ).
+        to_return(:status => 200, :body => "", :headers => {})
+
+      http = described_class.new("MYKEY", flush_interval: 0.1)
+      log_entry1 = Logtail::LogEntry.new("INFO", time, nil, "test log message 1", nil, nil)
+      http.write(log_entry1)
+      log_entry2 = Logtail::LogEntry.new("INFO", time, nil, "test log message 2", nil, nil)
+      http.write(log_entry2)
+      sleep 2
+
+      expect(stub).to have_been_requested.times(1)
+
+      http.close
+    end
+  end
+
+  describe "#deliver_requests" do
+    it "should handle exceptions properly and return" do
+      allow_any_instance_of(Net::HTTP).to receive(:request).and_raise("boom")
+
+      http_device = described_class.new("MYKEY", flush_continuously: false)
+      req_queue = http_device.instance_variable_get(:@request_queue)
+
+      # Place a request on the queue
+      request = Net::HTTP::Post.new("/")
+      request_attempt = Logtail::LogDevices::HTTP::RequestAttempt.new(request)
+      request_attempt.attempted!
+      req_queue.enq(request_attempt)
+
+      # Start a HTTP connection to test the method directly
+      http = http_device.send(:build_http)
+      http.start do |conn|
+        result = http_device.send(:deliver_requests, conn)
+        expect(result).to eq(false)
+      end
+
+      expect(req_queue.size).to eq(1)
+
+      # Start a HTTP connection to test the method directly
+      http = http_device.send(:build_http)
+      http.start do |conn|
+        result = http_device.send(:deliver_requests, conn)
+        expect(result).to eq(false)
+      end
+
+      # Ensure the request gets discards after 3 attempts
+      expect(req_queue.size).to eq(0)
+    end
+  end
+end