resque_log_error_parser 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 01983891b93d909cccf6c77177a2a3b4cbf042f8
+  data.tar.gz: 4d798bdd4a5ca70ab0d0e8d36556c286dd6015f2
+SHA512:
+  metadata.gz: 7096493461a534a1d626f1311336ce0a6894927d74353f0e3e939eb02f375ae8939d22f2a30ddb30f09fc03af8bb1475543fd6ffb76fa89757cf68b14d78902a
+  data.tar.gz: da05e6fd1edccf410696ae8b526431ef838b53fee57c018d7f258db3b8af03841bd4db44c8df11eec657a2b8fd05410b2e68ea1f3df0a12632a26a952d78ed69

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+# 1.0.0
+
+- Initial release

data/Gemfile

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+gemspec
+
+gem "pry"

data/Gemfile.lock

@@ -0,0 +1,56 @@
+PATH
+  remote: .
+  specs:
+    resque_log_error_parser (0.0.0)
+      resque
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    coderay (1.0.9)
+    diff-lcs (1.2.4)
+    method_source (0.8.2)
+    mono_logger (1.1.0)
+    multi_json (1.8.1)
+    pry (0.9.12.2)
+      coderay (~> 1.0.5)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+    rack (1.5.2)
+    rack-protection (1.5.0)
+      rack
+    redis (3.0.5)
+    redis-namespace (1.3.1)
+      redis (~> 3.0.0)
+    resque (1.25.1)
+      mono_logger (~> 1.0)
+      multi_json (~> 1.0)
+      redis-namespace (~> 1.2)
+      sinatra (>= 0.9.2)
+      vegas (~> 0.1.2)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.5)
+    rspec-expectations (2.14.3)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.3)
+    sinatra (1.4.3)
+      rack (~> 1.4)
+      rack-protection (~> 1.4)
+      tilt (~> 1.3, >= 1.3.4)
+    slop (3.4.6)
+    tilt (1.4.1)
+    timecop (0.6.1)
+    vegas (0.1.11)
+      rack (>= 1.0.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  pry
+  resque_log_error_parser!
+  rspec
+  timecop

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 G5
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,37 @@
+# Resque Log File Error Parser
+
+Resque worker that parses log files given filters, and executes a callback of your choice.
+
+## Install
+
+    gem "resque_log_file_error_parser"
+
+## Setup:
+
+    ResqueLogFileErrorParser.queue = :queue_the_worker_should_listen_to
+    ResqueLogFileErrorParser.on_error do |line_in_log_with_error|
+      # You can email yourself, or report to PagerDuty like:
+      p = Pagerduty.new("key here")
+      p.trigger("Error found: #{line_in_log_with_error}")
+    end
+
+    # If your logs are located in a place other than "log/*.log", then you need to override the log_paths:
+    ResqueLogFileErrorParser.log_paths = ["some/other/*.log"]
+
+## Scheduling
+
+ResqueLogFileErrorParser is intended to be run periodically (e.g. every hour) using `resque_scheduler` or `cron`.
+
+## Processing log entries
+
+Right now, the worker will only process log lines that match the following format (assuming the job was executed on Oct 7, 2013, between 6pm and 7pm):
+
+    E, [2013-10-07T18:01:04.194981 #28727] ERROR -- : Error line today
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,7 @@
+require "bundler/gem_tasks"
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end

data/lib/resque_log_error_parser/lines.rb

@@ -0,0 +1,19 @@
+module ResqueLogErrorParser
+  class Lines < Array
+
+    def filter_by(*filters)
+      filters = [filters].flatten
+      filtered_lines = self.dup
+      filters.each do |filter|
+        filtered_lines = filtered_lines.grep(/#{filter}/)
+      end
+      self.class.new(filtered_lines)
+    end
+
+    def strip_newlines
+      stripped_lines = self.map { |line| line.gsub("\n", "") }
+      self.class.new(stripped_lines)
+    end
+
+  end
+end

data/lib/resque_log_error_parser/version.rb

@@ -0,0 +1,3 @@
+module ResqueLogErrorParser
+  VERSION = "1.0.0"
+end

data/lib/resque_log_error_parser/worker.rb

@@ -0,0 +1,43 @@
+module ResqueLogErrorParser
+  class Worker
+
+    def self.queue
+      if ResqueLogErrorParser.queue.nil?
+        raise ArgumentError, "you must set ResqueLogErrorParser.queue"
+      end
+
+      ResqueLogErrorParser.queue
+    end
+
+    def self.perform
+      self.log_files.each do |file|
+        self.lines_for(file).each do |line|
+          ResqueLogErrorParser.error_handler.call(line)
+        end
+      end
+    end
+
+    private
+
+    def self.log_files
+      ResqueLogErrorParser.log_paths.map do |log_path|
+        Dir.glob(log_path)
+      end.flatten
+    end
+
+    def self.lines_for(file)
+      lines = Lines.new(File.open(file).lines.to_a)
+      lines.filter_by(self.filters).strip_newlines
+    end
+
+    def self.filters
+      [self.time_filter_regex, ResqueLogErrorParser.line_filter_keys].flatten
+    end
+
+    def self.time_filter_regex
+      time_filter_str = Time.now.strftime(ResqueLogErrorParser.line_date_format)
+      /#{time_filter_str}/
+    end
+
+  end
+end

data/lib/resque_log_error_parser.rb

@@ -0,0 +1,46 @@
+module ResqueLogErrorParser
+  DEFAULT_LOG_PATHS = ["log/*.log"]
+  DEFAULT_LINE_DATE_FORMAT = '%Y-%m-%dT%H'
+  DEFAULT_LINE_FILTER_KEYS = ["ERROR"]
+
+  def self.queue=(q)
+    @queue = q
+  end
+
+  def self.queue
+    @queue
+  end
+
+  def self.log_paths
+    return [@log_paths].flatten unless @log_paths.nil?
+    return DEFAULT_LOG_PATHS
+  end
+
+  def self.log_paths=(paths)
+    @log_paths = paths
+  end
+
+  def self.on_error(&block)
+    @on_error = block
+  end
+
+  def self.error_handler
+    @on_error
+  end
+
+  def self.line_date_format
+    DEFAULT_LINE_DATE_FORMAT
+  end
+
+  def self.line_filter_keys
+    DEFAULT_LINE_FILTER_KEYS
+  end
+
+  def self.restore_defaults!
+    self.log_paths = DEFAULT_LOG_PATHS
+  end
+
+end
+
+require "resque_log_error_parser/worker"
+require "resque_log_error_parser/lines"

data/resque_log_error_parser.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "resque_log_error_parser/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "resque_log_error_parser"
+  spec.version       = ResqueLogErrorParser::VERSION
+  spec.authors       = ["G5 Search Marketing"]
+  spec.email         = ["nestor@aelogica.com", "ramon@aelogica.com"]
+  spec.description   = %q{Resque worker that parses log files given filters, and executes a callback of your choice.}
+  spec.summary       = %q{Resque worker that parses log files given filters, and executes a callback of your choice.}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "resque"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "timecop"
+end

data/spec/fixtures/log/test.log

@@ -0,0 +1,3 @@
+E, [2013-10-05T17:01:04.194981 #28727] ERROR -- : Error line yesterday
+E, [2013-10-06T17:01:04.194981 #28727] ERROR -- : Error line today
+D, [2013-10-06T17:01:04.423205 #28727] DEBUG -- : No error

data/spec/resque_log_error_parser/lines_spec.rb

@@ -0,0 +1,45 @@
+require "spec_helper"
+
+describe ResqueLogErrorParser::Lines do
+
+  describe "#filter_by" do
+    let(:lines) do
+      described_class.new(["hello", "world", "helloworld"])
+    end
+
+    it "returns a Lines instance filtered by the given strings" do
+      filtered_lines = lines.filter_by("llo", /orld/)
+      expect(filtered_lines.count).to eq(1)
+      expect(filtered_lines).to include("helloworld")
+    end
+
+    it "returns an instance of Lines" do
+      filtered_lines = lines.filter_by(["llo"])
+      expect(filtered_lines).to be_kind_of(described_class)
+    end
+
+    context "argument is an array" do
+      it "filters the lines given the array elements" do
+        filtered_lines = lines.filter_by(["ell", /hello/])
+        expect(filtered_lines.count).to eq(2)
+        expect(filtered_lines).to include("hello")
+        expect(filtered_lines).to include("helloworld")
+      end
+    end
+
+    context "there are no filters" do
+      it "returns the original" do
+        filtered_lines = lines.filter_by([])
+        expect(filtered_lines).to eq(lines)
+      end
+    end
+  end
+
+  describe "#strip_newlines" do
+    let(:lines) { described_class.new(["hello\n"]) }
+    subject { lines.strip_newlines }
+    it { should eq(["hello"]) }
+    it { should be_kind_of(described_class) }
+  end
+
+end

data/spec/resque_log_error_parser/worker_spec.rb

@@ -0,0 +1,36 @@
+require "spec_helper"
+
+describe ResqueLogErrorParser::Worker do
+
+  describe ".queue" do
+    it "refers to the queue set in ResqueLogErrorParser" do
+      ResqueLogErrorParser.queue = :q
+      expect(described_class.queue).to eq(:q)
+      ResqueLogErrorParser.queue = :r
+      expect(described_class.queue).to eq(:r)
+    end
+
+    context "when queue has not been set on ResqueLogErrorParser" do
+      it "raises an error" do
+        ResqueLogErrorParser.queue = nil
+        expect { described_class.queue }.to raise_error(ArgumentError, "you must set ResqueLogErrorParser.queue")
+      end
+    end
+  end
+
+  describe ".perform" do
+    it "should call the error_handler block for each matching line in the logs for the given hour" do
+      Timecop.travel(2013, 10, 6, 17)
+
+      ResqueLogErrorParser.log_paths = "#{SPEC_DIR}/fixtures/log/*.log"
+      lines = []
+      ResqueLogErrorParser.on_error do |line|
+        lines << line
+      end
+      described_class.perform
+      expect(lines.count).to eq(1)
+      expect(lines).to include("E, [2013-10-06T17:01:04.194981 #28727] ERROR -- : Error line today")
+    end
+  end
+
+end

data/spec/resque_log_error_parser_spec.rb

@@ -0,0 +1,62 @@
+require "spec_helper"
+
+describe ResqueLogErrorParser do
+
+  describe ".queue=" do
+    it "sets the queue on the module" do
+      described_class.queue = :a
+      expect(described_class.queue).to eq(:a)
+      described_class.queue = :b
+      expect(described_class.queue).to eq(:b)
+    end
+  end
+
+  describe ".log_paths" do
+    it "defaults to ['log/*.log']" do
+      expect(described_class.log_paths).to eq(['log/*.log'])
+    end
+
+    context "when log_paths was set to nil" do
+      it "return the default log paths" do
+        described_class.log_paths = nil
+        expect(described_class.log_paths).to eq(["log/*.log"])
+      end
+    end
+  end
+
+  describe ".log_paths=" do
+    it "sets the log_paths" do
+      described_class.log_paths = ["SOMETHING"]
+      expect(described_class.log_paths).to eq(["SOMETHING"])
+    end
+
+    context "a non-enumerable was passed" do
+      it "should wrap with an array" do
+        described_class.log_paths = "SOMETHING"
+        expect(described_class.log_paths).to eq(["SOMETHING"])
+      end
+    end
+  end
+
+  describe ".on_error" do
+    let(:block) { lambda { |msg| msg } }
+
+    it "sets the callback function" do
+      described_class.on_error(&block)
+      expect(described_class.error_handler).to eq(block)
+    end
+  end
+
+  describe ".line_date_format" do
+    it "defaults to '%Y-%m-%dT%H'" do
+      expect(described_class.line_date_format).to eq('%Y-%m-%dT%H')
+    end
+  end
+
+  describe ".line_filter_keys" do
+    it "defaults to ['ERROR']" do
+      expect(described_class.line_filter_keys).to eq(['ERROR'])
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,14 @@
+SPEC_DIR = File.expand_path(File.dirname(__FILE__))
+
+require "bundler/setup"
+Bundler.require(:test, :development, :defaults)
+
+require "resque_log_error_parser"
+
+RSpec.configure do |config|
+  config.before(:each) do
+    ResqueLogErrorParser.restore_defaults!
+  end
+
+  config.order = 'random'
+end

metadata

@@ -0,0 +1,110 @@
+--- !ruby/object:Gem::Specification
+name: resque_log_error_parser
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- G5 Search Marketing
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-11-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: resque
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Resque worker that parses log files given filters, and executes a callback
+  of your choice.
+email:
+- nestor@aelogica.com
+- ramon@aelogica.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- lib/resque_log_error_parser.rb
+- lib/resque_log_error_parser/lines.rb
+- lib/resque_log_error_parser/version.rb
+- lib/resque_log_error_parser/worker.rb
+- resque_log_error_parser.gemspec
+- spec/fixtures/log/test.log
+- spec/resque_log_error_parser/lines_spec.rb
+- spec/resque_log_error_parser/worker_spec.rb
+- spec/resque_log_error_parser_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Resque worker that parses log files given filters, and executes a callback
+  of your choice.
+test_files:
+- spec/fixtures/log/test.log
+- spec/resque_log_error_parser/lines_spec.rb
+- spec/resque_log_error_parser/worker_spec.rb
+- spec/resque_log_error_parser_spec.rb
+- spec/spec_helper.rb