ruby_reduce 0.0.1

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+*.swp

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in ruby_reduce.gemspec
+gemspec

data/README.md

@@ -0,0 +1,71 @@
+# Ruby Reduce
+
+Ruby Reduce is a small library designed to run reduce commands on a Rails 3.x log without needing to setup a hadoop cluster to do it.  So it is basically designed to be a small scale reduce function.  The original idea was to make the gem plugable so that you could plugin different inputers to chunk non Rails 3 log files.  Also there was supposed to be a similar plugin design for the output but that has not happened yet.  So currently it only accepts rails 3 files for input and writes the results to MongoDB.
+
+
+## Installation
+
+`gem install ruby_reduce`
+
+
+## Usage
+
+The usage is also very straight forward and demostraited by this code snippet.
+
+```ruby
+require 'date' #only required because I parse a date
+require 'ruby_reduce' # 1) Require ruby reduce
+
+module RubyReduce
+  #return host and port and mongo_db (or nothing for default host and port)
+  def self.mongo_connection
+  end
+
+  #actual mongo db to connect to
+  def self.mongo_db
+    return 'test_db'
+  end
+end
+
+#actually running the reduce function
+r =  RubyReduce.reduce({
+    :input => '../production.log', #the location of the log file to reduce
+    # The map function can be anything that response to call and accepts 2 paramaters.  
+    # The first paramater is the log file name and the second are all the log statements 
+    # for a single rails request.  The only thing required here is that you **emit** a 
+    # key value pair to reduce later.  simple call the `emit(key, value)` function to accomplish this.
+    :map => Proc.new do |key, value|
+      processed_by = /Processing by (\w*#\w*)/.match(value)
+
+        unless processed_by.nil?
+          emit processed_by[1].gsub!("#", "_"), value
+        end
+      end,
+    # The reduce function can be anything that responds to call and takes one argument.  
+    # This argument will be the value emited in the map function (so reduce is called once 
+    # for every key value pair emited.  Also in the reduce function you need to **emit** 
+    # the result of the reduce.  Simply call `emit(result)` to tell RubyReduce what the result is.
+    :reduce => Proc.new do |log_statement|
+        date = /Started [GET|POST|DELETE|PUT|HEAD].* at (.*)/.match(log_statement) 
+        unless date.nil? 
+          processing_time = /Completed \d* .* in (\d*)/.match(log_statement)[1]
+          emit({'date' => Time.parse(date[1]), 'processed_time' => processing_time})
+        end
+      end,
+    #MongoDB collection name to write results to
+    :output => 'graph_data'
+  })
+```
+
+
+Once all of the key value pairs have been reduced the results will be collected by key and written into mongodb with one document for each key emited in the map function.  The id (_id) of this document is the emited key
+
+
+## Limitations
+
+Right now this is at best alpha software.  There are no tests and while I use it for my on projects it is not really been on a wide range of problems.  Your feedback is welcome.
+
+
+## Questions
+
+Contact me at joshsmoore@gmail.com with question, comments, or just that you are using the library and want me to continue work on it.

data/Rakefile

@@ -0,0 +1 @@
+require 'bundler/gem_tasks'

data/lib/ruby_reduce.rb

@@ -0,0 +1,25 @@
+require "ruby_reduce/version"
+require 'ruby_reduce/input_reader'
+require 'ruby_reduce/out_put'
+require 'ruby_reduce/reduce'
+require 'ruby_reduce/map'
+
+require 'mongo'
+
+module RubyReduce
+  # Your code goes here...
+
+  def self.reduce(options)
+    input = InputReader.new(options[:input]) 
+    read = input.read
+
+    map = Map.new(read, options[:map])
+    map.map
+
+    reduce = Reduce.new(map.map, options[:reduce])
+    result = reduce.reduce
+
+    OutputWriter.new(options[:output]).write(result)
+    
+  end
+end

data/lib/ruby_reduce/input_reader.rb

@@ -0,0 +1,42 @@
+module RubyReduce
+  class InputReader
+    def initialize(file_name)
+      @output = []
+      @file_name = file_name
+
+    end
+
+    def emit_chunk(key, chunk)
+      @output << [key, chunk]
+    end
+
+    def read
+      current_chunks = []
+      in_error = false
+
+      File.new(@file_name).each do |line|
+        if line =~ /^Started/ && line =~ /newrelic/
+        elsif line =~ /^Started/
+          current_chunks << [line]
+        elsif line =~ /^Completed \d/ 
+          current_chunks.first << line
+          emit_chunk(@file_name, current_chunks.first.join(''))
+          current_chunks.delete_at 0
+        elsif line =~ /^Completed(\s)*in/
+          in_error = true
+          current_chunks.first << line
+        elsif in_error && line =~ /within rescues\/layout/
+          in_error = false
+          current_chunks.first << line
+          emit_chunk(@file_name, current_chunks.first.join(''))
+          current_chunks.delete_at 0
+        elsif line == '' || line == "\n"
+        else
+          current_chunks.first << line if current_chunks.first
+        end
+      end
+
+      @output
+    end
+  end
+end

data/lib/ruby_reduce/map.rb

@@ -0,0 +1,22 @@
+module RubyReduce
+  class Map
+    def initialize(reader_output, function)
+      @reader_output = reader_output
+      @mapped_values = {}
+      @function = function
+    end
+
+    def emit(key, value)
+      @mapped_values[key] ||= []
+      @mapped_values[key] << value
+    end
+
+    def map
+      @reader_output.each do |chunk|
+        instance_exec chunk[0], chunk[1], &@function
+      end
+
+      @mapped_values
+    end
+  end
+end

data/lib/ruby_reduce/out_put.rb

@@ -0,0 +1,16 @@
+module RubyReduce
+  class OutputWriter
+    def initialize(options)
+      @collection_name = options
+
+      @mongo_db = Mongo::Connection.new(RubyReduce.mongo_connection).db(RubyReduce.mongo_db)
+    end
+
+    def write(data)
+      @mongo_db[@collection_name].remove()
+      data.each do |key, values|
+        @mongo_db[@collection_name].insert({:_id => key, :value => values})
+      end
+    end
+  end
+end

data/lib/ruby_reduce/reduce.rb

@@ -0,0 +1,26 @@
+module RubyReduce
+  class Reduce
+    def initialize(map_result, reduce)
+      @map_result = map_result
+      @reduce = reduce
+
+      @result = {}
+    end
+
+    def emit(result)
+      @result[@current_key] ||= []
+      @result[@current_key] << result
+    end
+
+    def reduce
+      @map_result.each do |key, values|
+        @current_key = key
+        values.each do |value|
+          instance_exec value, &@reduce
+        end
+      end
+
+      @result
+    end
+  end
+end

data/lib/ruby_reduce/version.rb

@@ -0,0 +1,3 @@
+module RubyReduce
+  VERSION = "0.0.1"
+end

data/ruby_reduce.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "ruby_reduce/version"
+
+Gem::Specification.new do |s|
+  s.name        = "ruby_reduce"
+  s.version     = RubyReduce::VERSION
+  s.authors     = ["Josh Moore"]
+  s.email       = ["joshsmoore@gmail.com"]
+  s.homepage    = ""
+  s.summary     = %q{Map reduce library for Ruby}
+  s.description = %q{Map reduce library for Ruby for Rails 3 log files}
+
+  s.rubyforge_project = "ruby_reduce"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  s.add_dependency 'mongo'
+end

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification 
+name: ruby_reduce
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Josh Moore
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-12-14 00:00:00 +08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: mongo
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+description: Map reduce library for Ruby for Rails 3 log files
+email: 
+- joshsmoore@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- README.md
+- Rakefile
+- lib/ruby_reduce.rb
+- lib/ruby_reduce/input_reader.rb
+- lib/ruby_reduce/map.rb
+- lib/ruby_reduce/out_put.rb
+- lib/ruby_reduce/reduce.rb
+- lib/ruby_reduce/version.rb
+- ruby_reduce.gemspec
+has_rdoc: true
+homepage: ""
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: ruby_reduce
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Map reduce library for Ruby
+test_files: []
+