peastash 0.0.8

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9736868c5166f30e722153e48ca37fdcee47d6bd
+  data.tar.gz: 99514ca3d88f07f2d0d7e72ab3354776b4243820
+SHA512:
+  metadata.gz: 92ca9c0ba61390259c47f8cd54085519f6a3be4f5c7cc65f30eddd14a2327b66e22bf1144cb94b438dc51e841ca82889923d9ab5e6065836994f3c91f49ef943
+  data.tar.gz: a0f1b7e7b12de3a15a3ba15a327675574843faf4bcfb61cf24614ffa51e2033b3549e4184ba6472c70025b60c37835f32bf930dd9ef52659210021b1ac18b0d0

data/.gitignore

@@ -0,0 +1,4 @@
+coverage
+pkg
+.DS_STORE
+*.log

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1.0
+  - jruby-19mode

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,129 @@
+PATH
+  remote: .
+  specs:
+    peastash (0.0.8)
+      logstash-event
+      thread_safe
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actionmailer (4.1.1)
+      actionpack (= 4.1.1)
+      actionview (= 4.1.1)
+      mail (~> 2.5.4)
+    actionpack (4.1.1)
+      actionview (= 4.1.1)
+      activesupport (= 4.1.1)
+      rack (~> 1.5.2)
+      rack-test (~> 0.6.2)
+    actionview (4.1.1)
+      activesupport (= 4.1.1)
+      builder (~> 3.1)
+      erubis (~> 2.7.0)
+    activemodel (4.1.1)
+      activesupport (= 4.1.1)
+      builder (~> 3.1)
+    activerecord (4.1.1)
+      activemodel (= 4.1.1)
+      activesupport (= 4.1.1)
+      arel (~> 5.0.0)
+    activesupport (4.1.1)
+      i18n (~> 0.6, >= 0.6.9)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.1)
+      tzinfo (~> 1.1)
+    arel (5.0.1.20140414130214)
+    builder (3.2.2)
+    coderay (1.1.0)
+    diff-lcs (1.2.5)
+    erubis (2.7.0)
+    hike (1.2.3)
+    i18n (0.6.9)
+    json (1.8.1)
+    logstash-event (1.2.02)
+    mail (2.5.4)
+      mime-types (~> 1.16)
+      treetop (~> 1.4.8)
+    method_source (0.8.2)
+    mime-types (1.25.1)
+    minitest (5.3.4)
+    multi_json (1.10.0)
+    polyglot (0.3.4)
+    pry (0.9.12.6)
+      coderay (~> 1.0)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+    rack (1.5.2)
+    rack-test (0.6.2)
+      rack (>= 1.0)
+    rails (4.1.1)
+      actionmailer (= 4.1.1)
+      actionpack (= 4.1.1)
+      actionview (= 4.1.1)
+      activemodel (= 4.1.1)
+      activerecord (= 4.1.1)
+      activesupport (= 4.1.1)
+      bundler (>= 1.3.0, < 2.0)
+      railties (= 4.1.1)
+      sprockets-rails (~> 2.0)
+    railties (4.1.1)
+      actionpack (= 4.1.1)
+      activesupport (= 4.1.1)
+      rake (>= 0.8.7)
+      thor (>= 0.18.1, < 2.0)
+    rake (10.3.1)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.8)
+    rspec-expectations (2.14.5)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.6)
+    rspec-rails (2.14.1)
+      actionpack (>= 3.0)
+      activemodel (>= 3.0)
+      activesupport (>= 3.0)
+      railties (>= 3.0)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    simplecov (0.7.1)
+      multi_json (~> 1.0)
+      simplecov-html (~> 0.7.1)
+    simplecov-html (0.7.1)
+    slop (3.5.0)
+    sprockets (2.12.1)
+      hike (~> 1.2)
+      multi_json (~> 1.0)
+      rack (~> 1.0)
+      tilt (~> 1.1, != 1.3.0)
+    sprockets-rails (2.1.3)
+      actionpack (>= 3.0)
+      activesupport (>= 3.0)
+      sprockets (~> 2.8)
+    thor (0.19.1)
+    thread_safe (0.3.4)
+    tilt (1.4.1)
+    timecop (0.7.1)
+    treetop (1.4.15)
+      polyglot
+      polyglot (>= 0.3.1)
+    tzinfo (1.2.0)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  peastash!
+  pry
+  rack-test
+  rails
+  rake
+  rspec (~> 2.14)
+  rspec-rails
+  simplecov (~> 0.7.1)
+  timecop

data/README.md

@@ -0,0 +1,183 @@
+# Peastash [![build status](https://travis-ci.org/elhu/peastash.png?branch=master)](https://travis-ci.org/elhu/peastash) [![Code Climate](https://codeclimate.com/github/elhu/peastash.png)](https://codeclimate.com/github/elhu/peastash)
+
+## Description
+
+Peastash helps you instrument your Ruby code with the ELK stack. It provides useful methods to easily write logs in a format that Logstash can understand, which will turn in (hopefully) beautiful dashboards using Elasticsearch and Kibana.
+
+The design philosophy behind Peastash is:
+* Be as unobstrusive as possible. Your code shouldn't be riddled with references to Peastash.
+* Keep the code-base small and well-tested
+* Provide sane default that will work for most people out of the box
+
+Any ruby application can benefit from having clear metrics, and now you have one less excuse to start instrumenting your code today.
+
+## Usage
+
+### Installing
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'peastash'
+```
+
+### Getting started
+``Peastash`` provides a simple interface to aggregate data to be sent to Logstash.
+The boundaries of a log entry are defined by a the ``Peastash.with_instance.log`` block.
+When the code inside the block is done running a log entry will be created and written.
+The most basic usage would look like:
+
+```ruby
+Peastash.with_instance.log do
+  Peastash.with_instance.store[:foo] = 'bar'
+end
+# This will produce a Logstash log entry looking like this:
+# {"@source":"peastash","@fields":{"foo":"bar"},"@tags":[],"@timestamp":"2014-05-27T15:18:29.824Z","@version":"1"}
+```
+
+### Configuration
+
+``Peastash`` ships with sane defaults, but can be easily configured:
+
+```ruby
+Peastash.with_instance.configure!({
+  source: 'peastash', # This value will be used for the @source field of the logstash event
+  # Any tag you wish to find in Logstash later (tags passed as argument to #log are added).
+  tags: [], # Defaults to [].
+  # Dumps to STDOUT by default. Any object answering to the dump method
+  output: Peastash::Outputs::IO.new('/tmp/peastash.log'),
+  store_name: 'peastash', # The key used to reference the store in the Thread.current
+  dump_if_empty: true # Whether or not Peastash should produce a log entry when the store is empty
+})
+```
+
+#### Safety
+
+``Peastash`` is safe by default. This means that if anything raises, Peastash will keep on going with the program's execution flow to be as unobtrusive as possible.
+If this behaviour is unsuitable for your needs (for example if you want to test the new Peastash output you've just developped), simply call:
+
+```ruby
+Peastash.unsafe!
+```
+Please note that this is a global setting and cannot be set per-instance.
+
+#### Outputs
+
+Peastash ships with a single output: ``Peastash::Outputs::IO``. It can be initialized either by passing it path or an IO object.
+Peastash can easily be extended to output to any target.
+Simply configure Peastash's output with an object that responds to the ``#dump`` method. This method will be called at the end of the ``#log`` block, with 1 argument : a ``LogStash::Event`` object, that you will probably need to serialize to json.
+
+### What if I want to use it in my rack app?
+
+There's a middleware for that! Simply add the following somewhere:
+
+```ruby
+require 'peastash/middleware'
+use Peastash::Middleware
+```
+
+By default, the middleware only adds the ``duration``, ``status`` and ``hostname`` (machine name) fields to the log entry.
+In addition to using ``Peastash.with_instance.store`` to add information, you can pass one or two block arguments to ``use``, that will be called with the request env and the Rack response in parameter, in the context of the Middleware's instance.
+The first block will be called **before** the request (with a ``response = nil``), while the second one will be called **after**, with the actual response. For example:
+
+```ruby
+require 'peastash/middleware'
+before_block = ->(env, response) { Peastash.with_instance.store[:path] = Rack::Request.new(env).path }
+after_block = ->(env, response) { Peastash.with_instance.store[:headers] = response[1] }
+use Peastash::Middleware, before_block, after_block
+# Will add 'path' and headers to the list of fields.
+```
+
+Any instance variable you set in ``before_block`` will be available in ``after_block``, but those instances variables **WILL NOT** be reset at the end of the request.
+
+### But I want to use it in Rails!
+
+It's easy! Simply add Peastash to your Gemfile, and add the following to the configure block of either ``config/environment.rb`` or ``config/<RAILS_ENV>.rb``:
+
+```ruby
+config.peastash.enabled = true
+# You can also configure Peastash from here, for example:
+config.peastash.output = Peastash::Outputs::IO.new(File.join(Rails.root, 'log', "logstash_#{Rails.env}.log"))
+config.peastash.source = Rails.application.class.parent_name
+```
+
+By default, Peastash's Rails integration will log the same parameters as the Middleware version, plus the fields in the payload of the [``process_action.action_controller``](http://edgeguides.rubyonrails.org/active_support_instrumentation.html#process_action.action_controller) notification (except the params).
+
+All the options for ``Peastash`` can be set using the ``config.peastash`` configuratio object.
+
+#### Logging request parameters
+
+To enable parameter logging, you must add the following to your configuration:
+
+```ruby
+config.peastash.log_parameters = true
+```
+
+Be careful, as this can significantly increase the size of the log entries, as well as causing problem if other Logstash entries have the same field with a different data type.
+
+#### Listening to ``ActiveSupport::Notifications``
+Additionaly, you can use Peastash to aggregate data from any ``ActiveSupport::Notifications``:
+
+```ruby
+# In config/initializers/peastash.rb
+if defined?(Peastash.with_instance.watch)
+  Peastash.with_instance.watch('request.rsolr', event_group: 'solr') do |name, start, finish, id, payload, event_store|
+    event_store[:queries] = event_store[:queries].to_i.succ
+    event_store[:duration] = event_store[:duration].to_f + ((finish - start) * 1000)
+  end
+end
+# This will add something like the following to the log entry fields
+# {'request.rsolr': {'queries': 1, 'duration': 42}}
+```
+
+The store exposed to the blocked passed to watch is thread-safe, and reset after each request. By default, the store is only shared between occurences of the same event. You can easily share the same store between different types of notifications by assigning them to the same event group:
+
+```ruby
+Peastash.with_instance.watch('foo.notification', event_group: 'notification') do |*args, store|
+  # Shared store with 'bar.notification'
+end
+
+Peastash.with_instance.watch('bar.notification', event_group: 'notification') do |*args, store|
+  # Shared store with 'foo.notification'
+end
+```
+
+### Playing with multiple instances
+
+Calling ``Peastash.with_instance`` is the same as calling ``Peastash.with_instance(:global)``.
+You can use different instances to log different things.
+Each instance as its own:
+
+* Configuration (inherited from the ``global`` instance)
+* Store
+* Tags
+
+Using several instances, you can nest ``log`` blocks without sharing the store, or simply use them for different purposes.
+For example, you can log your Rails query with the ``global`` instance, and your asynchronous jobs with a ``worker`` instance, and have those instances output to different files.
+
+### Playing with tags
+There are three ways to tag your log entries in Peastash.with_instance.
+
+* The first one is through configuration (see above).
+* The second one is by passing tags to the ``Peastash.with_instance.log`` method.
+* The last one is to call the ``Peastash.with_instance.tags`` method from within a ``Peastash.with_instance.log`` block. Tags defined with this method are not persistent and will disappear at the next call to ``Peastash.with_instance.log``
+
+For example:
+
+```ruby
+Peastash.with_instance.configure!(tags: 'foo')
+Peastash.with_instance.log(['bar']) { Peastash.with_instance.tags << 'baz' }
+# Tags are ['foo', 'bar', 'baz']
+Peastash.with_instance.log(['bar']) { Peastash.with_instance.tags << 'qux' }
+# Tags are ['foo', 'bar', 'qux']
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+Don't forget to run the tests with `rake`.

data/Rakefile

@@ -0,0 +1,17 @@
+$:.unshift File.expand_path('../lib', __FILE__)
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new
+
+task :default => :spec
+task :test => :spec
+
+task :console do
+  require 'irb'
+  require 'irb/completion'
+  require 'peastash'
+  ARGV.clear
+  IRB.start
+end

data/lib/peastash/middleware.rb

@@ -0,0 +1,37 @@
+require 'socket'
+
+class Peastash
+  class Middleware
+    def initialize(app, before_block = nil, after_block = nil)
+      @app = app
+      define_singleton_method(:before_block, before_block || ->(_, _) {})
+      define_singleton_method(:after_block, after_block || ->(_, _) {})
+    end
+
+    def call(env)
+      response = nil
+      Peastash.with_instance.log do
+        start = Time.now
+
+        Peastash.safely do
+          before_block(env, response)
+
+          # Setting this before calling the next middleware so it can be overriden
+          request = Rack::Request.new(env)
+          Peastash.with_instance.store[:ip] = request.ip
+        end
+
+        response = @app.call(env)
+
+        Peastash.safely do
+          Peastash.with_instance.store[:duration] = ((Time.now - start) * 1000.0).round(2)
+          Peastash.with_instance.store[:status] = response.first
+
+          after_block(env, response)
+        end
+      end
+
+      response
+    end
+  end
+end

data/lib/peastash/outputs/io.rb

@@ -0,0 +1,21 @@
+require 'logger'
+
+class Peastash
+  module Outputs
+    class IO
+      @@default_io = STDOUT
+
+      def self.default_io
+        @@default_io
+      end
+
+      def initialize(file)
+        @device = Logger::LogDevice.new(file)
+      end
+
+      def dump(event)
+        @device.write(event.to_json + "\n")
+      end
+    end
+  end
+end

data/lib/peastash/rails_ext/railtie.rb

@@ -0,0 +1,30 @@
+require 'peastash/middleware'
+require 'active_support/notifications'
+
+class Peastash
+  class Railtie < ::Rails::Railtie
+    config.peastash = ActiveSupport::OrderedOptions.new
+
+    initializer 'peastash.configure' do |app|
+      if app.config.peastash[:enabled]
+        Peastash.with_instance.configure!(app.config.peastash)
+        ActiveSupport::Notifications.subscribe('process_action.action_controller') do |name, started, finished, unique_id, data|
+          # Handle parameters and sanitize if need be
+          to_reject = [:db_runtime, :view_runtime, :exception]
+          payload = data.reject { |key, _| to_reject.include?(key) }
+          payload.merge!(db: data[:db_runtime], view: data[:view_runtime])
+          payload.merge!(exception: { class: data[:exception].first, message: data[:exception].last[0..200] }) if data.has_key?(:exception)
+          if Peastash.with_instance.configuration[:log_parameters]
+            payload[:params].reject { |k, _| ActionController::LogSubscriber::INTERNAL_PARAMS.include?(k) }
+          else
+            payload.delete(:params)
+          end
+          # Preserve explicitely set data
+          Peastash.with_instance.store.merge!(payload) { |key, old_val, new_val| old_val }
+        end
+        before_middleware = app.config.peastash[:insert_before] || ActionDispatch::ShowExceptions
+        app.config.middleware.insert_before before_middleware, Peastash::Middleware
+      end
+    end
+  end
+end

data/lib/peastash/rails_ext/watch.rb

@@ -0,0 +1,15 @@
+require 'active_support/notifications'
+
+class Peastash
+  module Watch
+    def watch(event, opts = {}, &block)
+      event_group = opts[:event_group] || event
+      ActiveSupport::Notifications.subscribe(event) do |*args|
+        # Calling the processing block with the Notification args and the store
+        block.call(*args, self.store[event_group])
+      end
+    end
+  end
+end
+
+Peastash.send :include, Peastash::Watch

data/lib/peastash/rails_ext.rb

@@ -0,0 +1,2 @@
+require 'peastash/rails_ext/watch'
+require 'peastash/rails_ext/railtie'

data/lib/peastash/version.rb

@@ -0,0 +1,3 @@
+class Peastash
+  VERSION = "0.0.8"
+end

data/lib/peastash.rb

@@ -0,0 +1,120 @@
+require 'logstash/event'
+require 'thread_safe'
+require 'peastash/outputs/io'
+require 'peastash/rails_ext' if defined?(Rails)
+
+class Peastash
+
+  STORE_NAME = 'peastash'
+
+  class << self
+
+    def configure!(conf = {})
+      with_instance.configure!(conf)
+    end
+
+    def with_instance(instance_name = :global)
+      @@instance_cache[instance_name] ||= Peastash.new(instance_name)
+    end
+
+    def safely
+      yield
+    rescue StandardError => e
+      STDERR.puts e
+      raise e unless safe?
+    end
+
+    def safe?
+      !@unsafe
+    end
+
+    def safe!
+      @unsafe = false
+    end
+
+    def unsafe!
+      @unsafe = true
+    end
+
+  end
+
+  attr_reader :instance_name
+  attr_accessor :configuration
+
+  @@instance_cache = ThreadSafe::Cache.new
+
+  def initialize(instance_name)
+    @instance_name = instance_name
+
+    @configuration = {
+      :source => STORE_NAME,
+      :tags => [],
+      :output => Outputs::IO.new(Outputs::IO::default_io),
+      :store_name => STORE_NAME,
+      :dump_if_empty => true
+    }
+
+    configure!(@@instance_cache[:global].configuration || {}) if @@instance_cache[:global]
+  end
+
+  def store
+    Thread.current[instance_name] ||= Hash.new
+    Thread.current[instance_name][@store_name] ||= Hash.new { |hash, key| hash[key] = {} }
+  end
+
+  def configure!(conf = {})
+    self.configuration.merge!(conf)
+    @source = configuration[:source]
+    @base_tags = configuration[:tags].flatten
+    @output = configuration[:output]
+    @store_name = configuration[:store_name]
+    @dump_if_empty = configuration[:dump_if_empty]
+    @configured = true
+  end
+
+  def log(additional_tags = [])
+    Peastash.safely do
+      configure! unless configured?
+      tags.replace(additional_tags)
+      store.clear
+    end
+
+    yield(instance)
+
+    Peastash.safely do
+      if enabled? && (!store.empty? || dump_if_empty?)
+        event = build_event(@source, tags)
+        @output.dump(event)
+      end
+    end
+  end
+
+  def tags
+    Peastash.safely do
+      configure! unless configured?
+      Thread.current[@store_name + ":tags"] ||= []
+    end
+  end
+
+  def enabled?
+    !!configuration[:enabled]
+  end
+
+  def instance
+    @@instance_cache[instance_name]
+  end
+
+  private
+
+  def configured?
+    @configured
+  end
+
+  def dump_if_empty?
+    @dump_if_empty
+  end
+
+  def build_event(source, tags)
+    LogStash::Event.new('@source' => source, '@fields' => store, '@tags' => @base_tags + tags)
+  end
+end

data/peastash.gemspec

@@ -0,0 +1,31 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'peastash/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "peastash"
+  gem.version       = Peastash::VERSION
+  gem.authors       = ["Vincent Boisard"]
+  gem.email         = ["boisard.v@gmail.com"]
+  gem.description   = %q{RequestStore gives you per-request global storage.}
+  gem.summary       = %q{RequestStore gives you per-request global storage.}
+  gem.homepage      = "http://github.com/elhu/peastash"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_runtime_dependency "logstash-event"
+  gem.add_runtime_dependency "thread_safe"
+
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "rspec", '~> 2.14'
+  gem.add_development_dependency "rspec-rails"
+  gem.add_development_dependency "rack-test"
+  gem.add_development_dependency "simplecov", '~> 0.7.1'
+  gem.add_development_dependency "timecop"
+  gem.add_development_dependency "pry"
+  gem.add_development_dependency "rails"
+end

data/spec/dummy/README.rdoc

@@ -0,0 +1,28 @@
+== README
+
+This README would normally document whatever steps are necessary to get the
+application up and running.
+
+Things you may want to cover:
+
+* Ruby version
+
+* System dependencies
+
+* Configuration
+
+* Database creation
+
+* Database initialization
+
+* How to run the test suite
+
+* Services (job queues, cache servers, search engines, etc.)
+
+* Deployment instructions
+
+* ...
+
+
+Please feel free to use a different markup language if you do not plan to run
+<tt>rake doc:app</tt>.

data/spec/dummy/Rakefile

@@ -0,0 +1,6 @@
+# Add your own tasks in files placed in lib/tasks ending in .rake,
+# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
+
+require File.expand_path('../config/application', __FILE__)
+
+Rails.application.load_tasks

data/spec/dummy/app/assets/javascripts/application.js

@@ -0,0 +1,13 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or vendor/assets/javascripts of plugins, if any, can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file.
+//
+// Read Sprockets README (https://github.com/sstephenson/sprockets#sprockets-directives) for details
+// about supported directives.
+//
+//= require_tree .