jeremyf-comma_pile 0.1.0

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,6 @@
+*.sw?
+.DS_Store
+coverage
+rdoc
+pkg
+test/tmp/*

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Jeremy Friesen
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,52 @@
+CommaPile
+==========
+
+A simple gem for processing and aggregating CSV files.  CommaPile builds a table that count
+
+Config Options:
+
+  * **source** - What is the name of the source csv file [**REQUIRED**, **SINGUL**]
+  * **on** - Specify any number of columns [**REQUIRED**, **MULTIPLE**]
+  * **sum_on** - Specify a column that you want to accumulate given [**OPTIONAL**, **MULTIPLE**]
+  * **conditions** - Specify a lambda, key-value pair that must be met for line to be
+      part of the compilation [**OPTIONAL**, **MULTIPLE**]
+  * **output** - Specify a path to output the compiled data; In addition a raw file,
+      containing all rows that were used in the compilation, is generated. [**OPTIONAL**, **SINGLE**]
+  * **line_parser** - Specify a custom line parser to use; By default CommaPile::LineParser is used.
+      A custom line parser would allow for transformation of data during compilation.  See
+      test/example\_line\_parser.rb [**OPTIONAL**, **SINGLE**]
+  
+Example
+-------
+    File.open('/path/to/input.csv', 'w+') do |file|
+      file.puts %('Work', 'Build CommaPile',2009-09-12,2)
+      file.puts %('Work', 'Build CommaPile',2009-09-13,1)
+      file.puts %('Work', 'Chase Chickens', 2009-09-12,4)
+      file.puts %('Work', 'Read Developer Blogs',2009-09-13,1)
+      file.puts %('Home', 'Do Dishes', 2009-09-12, 0.5)
+    end
+
+    require 'comma_pile'
+    
+    report = CommaPile.new do |config|
+      config.source = '/path/to/input.csv'
+      config.on << 0
+      config.on << 1
+      config.sum_on << 3
+      config.output = '/path/to/output.csv'
+      config.conditions = lambda {|r| r[1] == /^Chase/ }
+    end
+    
+    report.generate!
+    
+    report['Work'].counter == 3
+    report['Work'].sum[3] == 4
+    report['Work']['Build CommaPile'].counter == 2
+    report['Work']['Build CommaPile'].sum[3] == 3
+    report['Home'].sum[3] == 0.5
+
+
+Copyright
+---------
+
+Copyright (c) 2009 Jeremy Friesen. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,56 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "comma_pile"
+    gem.summary = %Q{Video Stats for an onstreammedia.com log}
+    gem.email = "jeremy.n.friesen@gmail.com"
+    gem.homepage = "http://github.com/jeremyf/comma_pile"
+    gem.authors = ["Jeremy Friesen"]
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/*_test.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  if File.exist?('VERSION.yml')
+    config = YAML.load(File.read('VERSION.yml'))
+    version = "#{config[:major]}.#{config[:minor]}.#{config[:patch]}"
+  else
+    version = ""
+  end
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "comma_pile #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+

data/VERSION.yml

@@ -0,0 +1,4 @@
+--- 
+:major: 0
+:minor: 1
+:patch: 0

data/comma_pile.gemspec

@@ -0,0 +1,53 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{comma_pile}
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Jeremy Friesen"]
+  s.date = %q{2009-08-13}
+  s.email = %q{jeremy.n.friesen@gmail.com}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.markdown"
+  ]
+  s.files = [
+    ".document",
+     ".gitignore",
+     "LICENSE",
+     "README.markdown",
+     "Rakefile",
+     "VERSION.yml",
+     "comma_pile.gemspec",
+     "lib/comma_pile.rb",
+     "lib/comma_pile/config.rb",
+     "lib/comma_pile/line_parser.rb",
+     "lib/comma_pile/pivot_node.rb",
+     "lib/comma_pile/report.rb",
+     "test/comma_pile_test.rb",
+     "test/example_line_parser.rb",
+     "test/fixtures/report.csv",
+     "test/test_helper.rb"
+  ]
+  s.homepage = %q{http://github.com/jeremyf/comma_pile}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.4}
+  s.summary = %q{Video Stats for an onstreammedia.com log}
+  s.test_files = [
+    "test/comma_pile_test.rb",
+     "test/example_line_parser.rb",
+     "test/test_helper.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end

data/lib/comma_pile/config.rb

@@ -0,0 +1,40 @@
+module CommaPile
+  class Config
+    def initialize
+      @field_names = []
+      @sum = []
+    end
+    attr_writer :conditions, :input, :output, :line_parser
+
+    def sum_on; @sum; end
+    def sum_on=(value)
+      @sum = (@sum << value).flatten.uniq
+    end
+
+    def line_parser
+      @line_parser ||= CommaPile::LineParser
+      if @line_parser.respond_to?(:with) && @line_parser.method(:with).arity == 1
+        @line_parser
+      else
+        raise RuntimeError, "#{self.class.to_s}#line_parser must respond to :with and have an arity of 1.  The line parser will receive an array of fields"
+      end
+    end
+
+    def output; @output; end
+    
+    def input; @input || './file.csv'; end
+    alias_method :source, :input
+    alias_method :source=, :input=
+    
+    def conditions; @conditions; end
+    
+    def field_names; @field_names; end
+    def field_names=(value)
+      @field_names = (@field_names << value).flatten.uniq
+    end
+    
+    alias_method :on, :field_names
+    alias_method :on=, :field_names=
+  end
+  
+end

data/lib/comma_pile/line_parser.rb

@@ -0,0 +1,16 @@
+module CommaPile
+  class LineParser
+    def self.with(line)
+      yield(new(line))
+    end
+
+    attr_reader :line
+    def initialize(line)
+      @line = line
+    end
+    
+    def [](value)
+      line[value]
+    end
+  end
+end

data/lib/comma_pile/pivot_node.rb

@@ -0,0 +1,20 @@
+module CommaPile
+  class PivotNode < Hash
+    attr_reader :sum
+    def initialize
+      @sum = {}
+    end
+    
+    attr_writer :counter
+    def counter; @counter ||= 0; end
+    
+    def inspect
+      "(counter: #{counter}; hash:#{super})"
+    end
+    
+    def add_to(fieldname, value)
+      self.sum[fieldname] ||= 0
+      self.sum[fieldname] += value
+    end
+  end
+end

data/lib/comma_pile/report.rb

@@ -0,0 +1,111 @@
+if RUBY_VERSION =~ /^1\.8/
+  require 'fastercsv'
+  CSV = FCSV
+else
+  require 'csv'
+end
+require 'delegate'
+
+module CommaPile
+  class Report < DelegateClass(Hash)
+    attr_reader :results
+
+    def initialize(config)
+      @config = config
+      @results = {}
+      super(@results)
+    end
+
+
+    def summary(entry = nil, parent_keys = [])
+      collector = ''
+      (entry || @results).each do |key, value|
+        if value.nil? || value.empty?
+          cells = [value.counter] + parent_keys + [key] + value.sum.values
+          collector += CSV.generate_line(cells.flatten)
+        else
+          collector += summary(value, parent_keys + [key])
+        end
+      end
+      collector
+    end
+    
+    def to_stdout(index = 0, entry = nil)
+      (entry || @results).each do |key, value|
+        puts "#{"\t" * index}#{key}: #{value.counter}"
+        if value.respond_to?(:each)
+          to_stdout(index + 1, value) if value && !value.empty?
+        end
+      end
+    end
+    
+
+    def generate!
+      if output
+        CSV.open(output.sub(/\.(\w+)$/, '.raw.\1'), 'w+') do |raw_csv|
+          @raw_csv = raw_csv
+          CSV.open(output, 'w+') do |parsed_csv|
+            @parsed_csv = parsed_csv
+            @parsed_csv << field_names.collect {|f| f.to_s }
+            process_input
+          end
+        end
+      else
+        process_input
+      end
+    end
+
+    protected
+    def field_names; @config.field_names; end
+    def input; @config.input; end
+    def conditions; @config.conditions; end
+    def output; @config.output; end
+    def line_parser; @config.line_parser; end
+    def sum_on_field_names; @config.sum_on; end
+
+    def process_input
+      CSV.foreach(input) do |line|
+        parse_line(line) do |record|
+          with_conditions_met_for(record) do
+            render_output_for(record, line)
+            accumulate_entry_for(record)
+          end
+        end
+      end
+    end
+
+    def register(line)
+    end
+
+    def render_output_for(record, line)
+      return nil unless output
+      @parsed_csv << field_names.inject([]) {|m,v| m << record[v]} if @parsed_csv
+      @raw_csv << line if @raw_csv
+    end
+
+    def accumulate_entry_for(record)
+      field_names.inject(self) do |mem, field_name|
+        key = record[field_name]
+        mem[key] ||= CommaPile::PivotNode.new
+        mem[key].counter += 1
+        sum_on_field_names.each do |sum_on_field_name|
+          mem[key].add_to(sum_on_field_name, record[sum_on_field_name])
+        end
+        mem[key]
+      end
+    end
+
+    def parse_line(line)
+      line_parser.with(line) do |record|
+        yield(record) if block_given?
+      end
+    end
+
+    def with_conditions_met_for(record)
+      yield and return unless conditions
+      yield and return if conditions.respond_to?(:call) && conditions.call(record)
+      yield and return if conditions.respond_to?(:all?) && conditions.all?{|(k,v)| record.send(k) =~ (v.is_a?(Regexp) ? v : /^#{Regexp.escape(v)}$/) }
+    end
+
+  end
+end

data/lib/comma_pile.rb

@@ -0,0 +1,12 @@
+require 'comma_pile/line_parser'
+require 'comma_pile/pivot_node'
+require 'comma_pile/report'
+require 'comma_pile/config'
+
+module CommaPile
+  def self.new
+    config = CommaPile::Config.new
+    yield(config)
+    CommaPile::Report.new(config)
+  end
+end

data/test/comma_pile_test.rb

@@ -0,0 +1,169 @@
+require 'test_helper'
+require 'example_line_parser'
+
+class CommaPileTest < Test::Unit::TestCase
+  CSV_FILE_PATH = File.join(File.dirname(__FILE__), "../test/fixtures/report.csv")
+
+  def output_filename
+    @output_filename ||= File.expand_path(File.join(File.dirname(__FILE__), 'tmp/output.csv'))
+  end
+
+  should 'not require line_parser' do
+    report = CommaPile.new do |config|
+      config.source = CSV_FILE_PATH
+      config.on = [0]
+    end
+    report.generate!
+    
+    assert_equal 3, report['129.74.105.126'].counter
+  end
+
+  # CSV_FILE_PATH = '/Users/jeremyf/Downloads/FullMonthlyRpt_Undame_7_2009.csv'
+  should 'have results takes a conditions option that is a hash with string value' do
+    report = CommaPile.new do |config|
+      config.line_parser = ExampleLineParser
+      config.source = CSV_FILE_PATH
+      config.on = [:project, :viewer_geolocation]
+      config.conditions = {:viewer_event => 'play'}
+    end
+    report.generate!
+
+
+    assert_equal 6, report['vocation'].counter
+    assert_equal 5, report['vocation']['off-campus'].counter
+    assert_equal 1, report['vocation']['on-campus'].counter
+    assert_equal 85, report['commencement'].counter
+    assert_equal 85, report['commencement']['off-campus'].counter
+    assert_equal 3, report['innovationpark'].counter
+    assert_equal 3, report['innovationpark']['off-campus'].counter
+  end
+
+  should 'have results takes a conditions option that is a hash with regular express' do
+    report = CommaPile.new do |config|
+      config.line_parser = ExampleLineParser
+      config.source = CSV_FILE_PATH
+      config.on << :project
+      config.on << :viewer_event
+      config.on << :viewer_geolocation
+      config.conditions = {:viewer_event => /(play|stop)/}
+    end
+    report.generate!
+    
+
+    assert_equal 12, report['vocation'].counter
+    assert_equal 6, report['vocation']['play'].counter
+    assert_equal 5, report['vocation']['play']['off-campus'].counter
+    assert_equal 1, report['vocation']['play']['on-campus'].counter
+    assert_equal 6, report['vocation']['stop'].counter
+    assert_equal 5, report['vocation']['stop']['off-campus'].counter
+    assert_equal 1, report['vocation']['stop']['on-campus'].counter
+  end
+
+  should 'have results takes a conditions option that is a lambda' do
+    report = CommaPile.new do |config|
+      config.line_parser = ExampleLineParser
+      config.source = CSV_FILE_PATH
+      config.field_names << :project
+      config.on << :viewer_event
+      config.conditions = lambda {|r| r.viewer_event == 'play' || r.project == 'vocation'}
+    end
+    report.generate!
+    
+    assert_equal 94, report['vocation'].counter
+    assert_equal 6, report['vocation']['play'].counter
+    assert_equal 6, report['vocation']['stop'].counter
+    assert_nil report['commencement']['stop']
+    assert_equal 85, report['commencement']['play'].counter
+  end
+
+  should 'have results that no options' do
+    report = CommaPile.new do |config|
+      config.line_parser = ExampleLineParser
+      config.source = CSV_FILE_PATH
+      config.on = [:project, :viewer_event, :viewer_geolocation]
+      config.conditions = lambda {|r| r.viewer_event == 'play' || r.project == 'vocation'}
+    end
+    report.generate!
+    
+
+    assert_equal 94, report['vocation'].counter
+    assert_equal 6,  report['vocation']['play'].counter
+    assert_equal 29, report['vocation']['pause'].counter
+    assert_equal 28, report['vocation']['unpause'].counter
+    assert_equal 25, report['vocation']['seek'].counter
+    assert_equal 6, report['vocation']['stop'].counter
+  end
+
+  should 'have results that output conditional matches to a file' do
+    report = CommaPile.new do |config|
+      config.line_parser = ExampleLineParser
+      config.input = CSV_FILE_PATH
+      config.on = :viewer_event
+      config.output = output_filename
+      config.conditions = {:viewer_event => 'play'}
+    end
+    report.generate!
+    
+    File.readlines(output_filename).each do |line|
+      assert_match(/^(viewer_event|play)$/i, line)
+      @yielded = true
+    end
+    assert @yielded
+  end
+
+  should 'have sub-results that add up to parent results' do
+    report = CommaPile.new do |config|
+      config.line_parser = ExampleLineParser
+      config.input = CSV_FILE_PATH
+      config.on = [:project, :viewer_event]
+    end
+    
+    report.generate!
+    
+
+    report.each do |name, collector|
+      @yielded = true
+      assert_equal collector.counter, collector.inject(0){|m,(k,v)| m += v.counter}
+    end
+
+    assert @yielded, "Making sure the above method is called"
+  end
+
+  should 'allow column numbers to be used instead of field names' do
+    report = CommaPile.new do |config|
+      config.line_parser = ExampleLineParser
+      config.input = CSV_FILE_PATH
+      config.on << :viewer_geolocation
+      config.on << 0
+    end
+    report.generate!
+    assert_equal 3, report['on-campus'].counter
+    assert_equal 94, report['off-campus']['68.45.25.118'].counter
+  end
+  
+  should "have a summary" do
+    report = CommaPile.new do |config|
+      config.line_parser = ExampleLineParser
+      config.input = CSV_FILE_PATH
+      config.on << :viewer_geolocation
+      config.on << 0
+      config.sum_on << :filesize
+    end
+    report.generate!
+    assert_match /^#{Regexp.escape('1,off-campus,71.103.212.224,92094')}$/, report.summary
+  end
+
+  should "allow one or more accumulators" do
+    report = CommaPile.new do |config|
+      config.line_parser = ExampleLineParser
+      config.input = CSV_FILE_PATH
+      config.on << :viewer_geolocation
+      config.on << 0
+      config.sum_on << :filesize
+    end
+    report.generate!
+        
+    assert_equal 17155, report['on-campus'].sum[:filesize]
+    assert_equal 371622045, report['off-campus'].sum[:filesize]
+  end
+end

data/test/example_line_parser.rb

@@ -0,0 +1,43 @@
+class ExampleLineParser < CommaPile::LineParser
+  INDEX_FOR_IP_ADDRESS = 0
+  HEADER_VALUE_FOR_IP_ADDRESS = 'c_ip'
+  INDEX_FOR_FILENAME = 65
+  INDEX_FOR_CLIENT_SIDE_REFERRER = 14
+  INDEX_FOR_VIEWER_EVENT = 54
+  INDEX_FOR_FILESIZE = 69
+  def self.with(line)
+    super if line[INDEX_FOR_IP_ADDRESS] != HEADER_VALUE_FOR_IP_ADDRESS
+  end
+
+  def filesize
+    line[69].to_i
+  end
+
+  def viewer_geolocation
+    line[INDEX_FOR_IP_ADDRESS].strip =~ /^129\.74\./ ? "on-campus" : "off-campus"
+  end
+
+  def viewer_event
+    line[INDEX_FOR_VIEWER_EVENT]
+  end
+
+  def project
+    line[INDEX_FOR_FILENAME].match(/[\\|\/]?undame[\\|\/]([^\\|\/]*)/i)[1] rescue nil
+  end
+
+  def system_name
+    File.basename(line[65].gsub(/\\/,'/')) if line[65]
+  end
+
+  def referrer
+    line[INDEX_FOR_CLIENT_SIDE_REFERRER]
+  end
+  
+  def [](value)
+    if value.is_a?(Integer)
+      line[value]
+    else
+      send(value)
+    end
+  end
+end