log_weaver 0.0.1

data/features/step_definitions/log_weaver_steps.rb

@@ -0,0 +1,16 @@
+require 'fileutils'
+
+if ENV['FAILFAST']
+  After do |s|
+    Cucumber.wants_to_quit = true if s.failed?
+  end
+end
+
+Given /^no file named "([^"]*)"$/ do |file_name|
+  check_file_presence( [file_name], false )
+end
+
+Before do
+  `rm -rf tmp`
+end
+

data/features/support/env.rb

@@ -0,0 +1,17 @@
+require 'aruba/cucumber'
+require 'methadone/cucumber'
+
+ENV['PATH'] = "#{File.expand_path(File.dirname(__FILE__) + '/../../bin')}#{File::PATH_SEPARATOR}#{ENV['PATH']}"
+LIB_DIR = File.join(File.expand_path(File.dirname(__FILE__)),'..','..','lib')
+
+Before do
+  @aruba_timeout_seconds = 3600
+  # Using "announce" causes massive warnings on 1.9.2
+  @puts = true
+  @original_rubylib = ENV['RUBYLIB']
+  ENV['RUBYLIB'] = LIB_DIR + File::PATH_SEPARATOR + ENV['RUBYLIB'].to_s
+end
+
+After do
+  ENV['RUBYLIB'] = @original_rubylib
+end

data/lib/log_weaver.rb

@@ -0,0 +1,4 @@
+require 'require_all'
+
+require_all "#{File.dirname(File.expand_path(__FILE__))}/log_weaver"
+

data/lib/log_weaver/combined_log.rb

@@ -0,0 +1,37 @@
+module LogWeaver
+
+  class CombinedLog
+    attr_accessor :logs
+
+    def initialize(logs)
+      @logs = logs
+      @index = CombinedLog::build_index(@logs)
+    end
+
+    def self.build_index(logs)
+      # need to sort by timestamp, then prefix
+      index = {}
+      logs.each do |log|
+        log.lines.each do |t,l|
+          key = CombinedLogIndexKey.new(log.prefix, t)
+          index[key] = l
+        end
+      end
+      #TODO: sorting at this point may have seriously bad performance for large logs; consider a
+      # data structure that stays sorted as you insert
+      Hash[index.sort]
+    end
+
+    def to_s
+      res = ""
+      @index.each do |key, lines|
+        lines.each do |l|
+          res << "#{key.prefix}#{l}\n"
+        end
+      end
+      res
+    end
+  end
+end
+
+

data/lib/log_weaver/combined_log_index_key.rb

@@ -0,0 +1,14 @@
+module LogWeaver
+  CombinedLogIndexKey = Struct.new(:prefix, :timestamp) do
+    include Comparable
+
+    def <=>(other)
+      return timestamp <=> other.timestamp unless timestamp == other.timestamp
+      return prefix <=> other.prefix
+    end
+
+    def to_s
+      "#{prefix}#{timestamp}"
+    end
+  end
+end

data/lib/log_weaver/monkey_patch.rb

@@ -0,0 +1,14 @@
+
+class Array
+  def uniq?
+    self.length == self.uniq.length
+  end
+end
+
+class Time
+  def to_s
+    self.strftime("%Y-%m-%d %H:%M:%S.%L")
+  end
+end
+
+

data/lib/log_weaver/parsed_log.rb

@@ -0,0 +1,41 @@
+require 'time'
+
+module LogWeaver
+  class ParsedLog
+    attr_accessor :lines #TODO: rename; attr_reader should suffice
+    attr_reader :prefix
+
+    def initialize(prefix, log)
+      @prefix = prefix
+      @lines = ParsedLog.parse_log log
+    end
+
+    #private TODO: see http://stackoverflow.com/questions/4952980/creating-private-class-method; test per
+    # http://kailuowang.blogspot.ca/2010/08/testing-private-methods-in-rspec.html
+    def self.parse_log(log)
+      res = {}
+      previous_key = nil
+      log.string.split("\n").each do |line|
+        (timestamp, message) = extract_time_stamp(line)
+        if timestamp
+          key = timestamp
+          res[key] = [] unless key == previous_key
+          previous_key = key
+        else
+          raise ArgumentError, "Log does not begin with a timestamp." if previous_key.nil?
+        end
+
+        res[previous_key] << line #message
+      end
+      res
+    end
+
+    def self.extract_time_stamp(line)
+      timestamp = line[/^[0-9]{4}-[01][0-9]-[0-3][0-9] [0-2][0-9](:[0-5][0-9]){2}\.[0-9]{3}/,0]
+      message = line.sub(/^#{timestamp}/,'')
+      timestamp = Time.parse(timestamp) unless timestamp.nil?
+      [timestamp,message]
+    end
+
+  end
+end

data/lib/log_weaver/parsed_log_key.rb

@@ -0,0 +1,11 @@
+module LogWeaver
+  ParsedLogKey = Struct.new( :prefix, :timestamp, :count ) do
+    include Comparable
+
+    def <=>(other)
+      return timestamp <=> other.timestamp unless timestamp == other.timestamp
+      return count <=> other.count unless count == other.count
+      prefix <=> other.prefix
+    end
+  end
+end

data/lib/log_weaver/prefix_generator.rb

@@ -0,0 +1,98 @@
+require 'methadone'
+
+include Methadone::Main
+
+module LogWeaver
+  module PrefixGenerator
+
+    # given an array of file paths, generate file name prefixes given the following rules:
+    # 1. prefixes have to differ
+    # 2. prefixes have to be at least as long as min_length, unless file name is shorter
+    # 3. if file names match, and are shorter than min_length, grab whole directories from directory path until they don't match
+    # results are returned as a hash keyed on passed-in file names
+    def get_file_prefixes(file_paths, min_length = 4)
+      # pseudocode:
+      # sort by base_name length
+      # get common prefix of base_names
+      # append letters to prefix from file name at least until min_length and all unique
+      # prepend directories until all unique
+
+      base_names = []
+      expanded_paths = []
+      processed_file_paths = {}
+      max_base_name_length = 0
+      max_path_component_length = 0
+
+      file_paths.each do |fp|
+        max_base_name_length = fp.length if fp.length > max_base_name_length
+        base_name = File.basename fp
+        base_names << base_name
+        processed_file_paths[fp] = {}
+        processed_file_paths[fp][:base_name] = base_name
+        processed_file_paths[fp][:expanded_path] = File.expand_path(fp)
+        expanded_paths << processed_file_paths[fp][:expanded_path]
+        path_dirs = processed_file_paths[fp][:expanded_path].split('/')
+        path_dirs.pop
+        processed_file_paths[fp][:path_dirs] = path_dirs
+        max_path_component_length = processed_file_paths[fp][:path_dirs].length if processed_file_paths[fp][:path_dirs].length > max_path_component_length
+      end
+
+      raise ArgumentError, "File list is not unique." unless expanded_paths.uniq?
+
+      # initialize accumulator data structures with the common prefix
+      prefix = get_longest_common_prefix base_names
+      prefixes = []
+      file_paths.each do |fp|
+        processed_file_paths[fp][:prefix] = prefix.dup
+        prefixes << processed_file_paths[fp][:prefix]
+      end
+
+      # append as many remaining characters from file basename as it will take to take us
+      # over min_length and make each prefix unique
+      (prefix.length .. max_base_name_length - 1).each do |i|
+        file_paths.each do |fp|
+          # append an additional letter; note, if nil, to_s will convert it to ""
+          processed_file_paths[fp][:prefix] << processed_file_paths[fp][:base_name][i].to_s
+        end
+        if i+1 >= min_length
+          break if prefixes.uniq?
+        end
+      end
+
+      # prepend dir path components if still not unique
+      (max_path_component_length - 1).downto(0) do |i|
+        break if prefixes.uniq?
+        file_paths.each do |fp|
+          processed_file_paths[fp][:prefix].insert(0, processed_file_paths[fp][:path_dirs][i].to_s + "/")
+        end
+      end
+
+      # pick out the results
+      res = {}
+      longest_prefix_length = 0
+      file_paths.each do |fp|
+        res[fp] = processed_file_paths[fp][:prefix]
+        longest_prefix_length = res[fp].length if res[fp].length > longest_prefix_length
+      end
+
+      file_paths.each do |fp|
+        orig_prefix_length = res[fp].length
+        res[fp] << ": " << " " * (longest_prefix_length - orig_prefix_length)
+      end
+
+      res
+    end
+
+    def get_longest_common_prefix(words)
+      words = words.dup
+      return nil if words.include? nil
+      prefix =  words.shift.dup
+      until prefix == ""
+        break if words.all?{ |w| w =~ /^#{prefix}/ }
+        prefix.chop!
+      end
+      prefix
+    end
+
+  end
+end

data/lib/log_weaver/version.rb

@@ -0,0 +1,3 @@
+module LogWeaver
+  VERSION = "0.0.1"
+end

data/log_weaver.gemspec

@@ -0,0 +1,40 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/log_weaver/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.add_development_dependency('readme', '~> 0.1')
+  require 'readme'
+  r = Readme.file
+  gem.authors       = r.authors
+  gem.email         = ["raphael.borowiecki@gmail.com"]
+  gem.description   = r.description
+  gem.summary       = "Weaves log files by timestamp."
+  gem.homepage      = r.homepage
+
+  if File.directory?('.hg')
+    gem.files         = `hg manifest`.split($\)
+  elsif File.directory?('.git')
+    gem.files         = `git ls-files`.split($\)
+  else
+    raise "Need to build from a git or hg repo to generate manifest."
+  end
+
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "log_weaver"
+  gem.require_paths = ["lib"]
+  gem.version       = LogWeaver::VERSION
+
+  gem.add_development_dependency('yard', '~> 0.8')
+  gem.add_development_dependency('unindent', '~> 1.0')
+  gem.add_development_dependency('aruba', '~> 0.5')
+  gem.add_development_dependency('cucumber', '~> 1.2')
+  gem.add_development_dependency('rspec', '~> 2.13')
+  gem.add_development_dependency('factory_girl', '~> 4.2')
+  gem.add_development_dependency('rake','~> 0.9')
+  gem.add_dependency('methadone', '~> 1.2')
+  gem.add_dependency('require_all', '~> 1.2')
+end
+
+
+

data/spec/combined_log_index_key_spec.rb

@@ -0,0 +1,19 @@
+require 'spec_helper'
+
+module LogWeaver
+  class CombinedLogIndexKey
+    describe "#<=>" do
+      it "should compare timestamp first" do
+        ($k_p1_t2 <=> $k_p2_t1).should == ($k_p1_t2.timestamp <=> $k_p2_t1.timestamp)
+      end
+      it "should compare prefix second" do
+        ($k_p1_t1 <=> $k_p2_t1).should == ($k_p1_t1.prefix <=> $k_p2_t1.prefix)
+      end
+    end
+    describe"#to_s" do
+      it "prints 'prefix: timestamp'" do
+        $k_p1_t1.to_s.should == "#{$p1}#{$t1}"
+      end
+    end
+  end
+end

data/spec/combined_log_spec.rb

@@ -0,0 +1,95 @@
+require 'spec_helper'
+require 'unindent'
+
+module LogWeaver
+
+  def index(logs)
+    CombinedLog.new(logs).instance_variable_get(:index)
+  end
+
+
+  describe CombinedLog do
+=begin
+    before(:all) do
+    end
+
+
+    describe ".initialize" do
+      it "stores the logs" do
+        @cl.logs.should == @logs
+      end
+    end
+=end
+
+    describe ".build_index" do
+      # note: it's much briefer/easier/more readable to give symbolic descriptions
+      # for the examples; pn is the prefix, tn is the timestamp, ln are the line contents
+=begin
+      it "handles [p1_t1_l1, p2_t2_l1] (2 logs, ordered timestamps, 1 line each)" do
+        CombinedLog.build_index([$pl_p1_t1_l1, $pl_p2_t2_l1]).to_a.should == $hash_p1_t1_l1_and_p2_t2_l1.to_a
+      end
+      it "handles [p1_t1_l1, p2_t1_l1] (2 logs, same timestamp, 1 line each)" do
+        CombinedLog.build_index([$pl_p1_t1_l1, $pl_p2_t1_l1]).to_a.should == $hash_p1_t1_l1_and_p2_t1_l1.to_a
+      end
+      it "handles [p1_t2_l1, p2_t1_l1] (2 logs, p2 timestamp comes before p1)" do
+        CombinedLog.build_index([$pl_p1_t2_l1, $pl_p2_t1_l1]).to_a.should == $hash_p1_t2_l1_and_p2_t1_l1.to_a
+      end
+
+
+      sum = @parsed_log_p1_t1l1_t2l1 + @parsed_log_p1_t1l2_t2l2
+      sum = @parsed_log_p1_t2l1 + @parsed_log_p2_t1l1
+      sum = @parsed_log_p1_t1l1_t2l1 + @parsed_log_p2_t1l1_t2l1
+      it "handles [p1_t2_l1, p2_t1_l1] (2 logs, p2 timestamp comes before p1)" do
+        cl = CombinedLog.build_index([$pl_p1_t2_l1, $pl_p2_t1_l1])
+        cl.should == $hash_p1_t2_l1_and_p2_t1_l1
+        CombinedLog.build_index([$pl_p1_t2_l1, $pl_p2_t1_l1]).to_a.should == $hash_p1_t2_l1_and_p2_t1_l1.to_a
+      end
+=end
+    end
+
+    describe "#to_s" do
+      it "prints p1_t1_l1, p2_t2_l1" do
+        output = <<-eos
+          #{$out_p1_t1_l1}
+          #{$out_p2_t2_l1}
+        eos
+        CombinedLog.new([$pl_p1_t1_l1, $pl_p2_t2_l1]).to_s.should == output.unindent
+      end
+      it "prints p1_t1_l1, p2_t1_l1 (same timestamp across files)" do
+        output = <<-eos
+          #{$out_p1_t1_l1}
+          #{$out_p2_t1_l1}
+        eos
+        CombinedLog.new([$pl_p1_t1_l1, $pl_p2_t1_l1]).to_s.should == output.unindent
+      end
+      it "prints p1_t1_l1_t1_l2, p2_t1_l3_t1_l4 (same timestamp across files, more lines)" do
+        output = <<-eos
+          #{$out_p1_t1_l1}
+          #{$out_p1_t1_l2}
+          #{$out_p2_t1_l3}
+          #{$out_p2_t1_l4}
+        eos
+        CombinedLog.new([$pl_p1_t1_l1_t1_l2, $pl_p2_t1_l3_t1_l4]).to_s.should == output.unindent
+      end
+      it "prints p1_t1_l1_t2_l1, p2_t3_l1" do
+        output = <<-eos
+          #{$out_p1_t1_l1}
+          #{$out_p1_t2_l1}
+          #{$out_p2_t3_l1}
+        eos
+        CombinedLog.new([$pl_p1_t1_l1_t2_l1, $pl_p2_t3_l1]).to_s.should == output.unindent
+      end
+=begin
+      # from parsed_log_spec
+      it "prepends the prefix to every line with a timestamp" do
+        ParsedLog.parse_log(@t1l1_t2l1_log, @p1).should == @hash_with_one_line_per_timestamp
+      end
+      it "does not prepend the prefix to lines with no time stamp" do
+        ParsedLog.parse_log(@log_with_missing_timestamps, @p1).should == @hash_with_more_than_one_line_per_timestamp
+      end
+=end
+    end
+  end
+end
+
+

data/spec/factories_and_globals.rb

@@ -0,0 +1,177 @@
+require 'time'
+require 'log_weaver'
+
+
+# Vars common to specs and factories; I tried wrapping them in a module, i.e.:
+# module CommonVariables
+#   def p1() 'p1' end
+# end
+# but having modules deep in Rspec get access to it by extending was way more voodoo
+# than I can stomach for now. So we'll go with globals.
+
+$p1 = 'p1:'
+$p2 = 'p2:'
+
+$t1 = Time.parse('2000-01-01 00:00:01.000') # NOTE: init time this way to discard values below msec
+$t2 = $t1 + 1
+$t3 = $t2 + 1
+
+$l1 = ' l1'
+$l2 = ' l2'
+$l3 = ' l3'
+$l4 = ' l4'
+
+$t1_l1 = "#{$t1}#{$l1}"
+$t1_l2 = "#{$t1}#{$l2}"
+$t1_l3 = "#{$t1}#{$l3}"
+$t1_l4 = "#{$t1}#{$l4}"
+$t2_l1 = "#{$t2}#{$l1}"
+$t2_l2 = "#{$t2}#{$l2}"
+$t3_l1 = "#{$t3}#{$l1}"
+
+$no_t_l1 = $l1
+$no_t_l2 = $l2
+
+$out_p1_t1_l1 = "#{$p1}#{$t1_l1}"
+$out_p1_t1_l2 = "#{$p1}#{$t1_l2}"
+$out_p1_t2_l1 = "#{$p1}#{$t2_l1}"
+$out_p2_t1_l1 = "#{$p2}#{$t1_l1}"
+$out_p2_t1_l3 = "#{$p2}#{$t1_l3}"
+$out_p2_t1_l4 = "#{$p2}#{$t1_l4}"
+$out_p2_t2_l1 = "#{$p2}#{$t2_l1}"
+$out_p2_t3_l1 = "#{$p2}#{$t3_l1}"
+
+$io_empty                      = StringIO.new
+$io_t1_l1                      = StringIO.new($t1_l1)
+$io_t2_l1                      = StringIO.new($t2_l1)
+$io_t1_l1_t2_l1                = StringIO.new([$t1_l1, $t2_l1].join("\n"))
+$io_t1_l1_t1_l2                = StringIO.new([$t1_l1, $t1_l2].join("\n"))
+$io_no_t_l1_no_t_l2            = StringIO.new([$no_t_l1, $no_t_l2].join("\n"))
+$io_no_t_l1_t1_l2              = StringIO.new([$no_t_l1, $t1_l2].join("\n"))
+$io_t1_l1_no_t_l2              = StringIO.new([$t1_l1, $no_t_l2].join("\n"))
+
+$io_t1l2_t2l2                  = StringIO.new([$t1_l2, $t2_l2].join("\n"))
+$io_with_missing_timestamps    = StringIO.new([$t1_l1, $no_t_line, $t2_l1].join("\n"))
+$io_with_duplicate_timestamp   = StringIO.new([$t1_l1, $t1_l1].join("\n"))
+$io_starting_with_no_timestamp = StringIO.new([$no_t_line, $t2_l1].join("\n"))
+
+$hash_t1_l1 = { $t1 => [$t1_l1] }
+$hash_t1_l1_t2_l1 = {
+    $t1 => [$t1_l1],
+    $t2 => [$t2_l1]
+}
+$hash_t1_l3_t1_l4 = {
+    $t1 => [$t1_l3, $t1_l4]
+}
+$hash_t2_l1 = { $t2 => [$t2_l1] }
+$hash_t3_l1 = { $t3 => [$t3_l1] }
+$hash_t1_l1_t1_l2 = {
+    $t1 => [$t1_l1, $t1_l2]
+}
+$hash_t1_l1_no_t_l2 = {
+    $t1 => [$t1_l1, $no_t_l2]
+}
+
+# need to monkey-patch in argumentless constructors for FactoryGirl to be happy;
+# see http://stackoverflow.com/a/6838145/26819
+module LogWeaver
+  class ParsedLog
+    attr_accessor :prefix
+    def initialize
+    end
+  end
+end
+
+FactoryGirl.define do
+  #TODO: can this be LogWeaver::ParsedLog instead of a string?
+  factory 'log_weaver/parsed_log' do
+    factory :pl_p1 do
+      prefix {$p1} #TODO: are the curlies needed here?
+      factory :pl_p1_t1_l1 do
+        lines $hash_t1_l1
+      end
+      factory :pl_p1_t1_l1_t1_l2 do
+        lines $hash_t1_l1_t1_l2
+      end
+      factory :pl_p1_t1_l1_t2_l1 do
+        lines $hash_t1_l1_t2_l1
+      end
+      factory :pl_p1_t2_l1 do
+        lines $hash_t2_l1
+      end
+    end
+
+    factory :pl_p2 do
+      prefix $p2
+      factory :pl_p2_t1_l1 do
+        lines $hash_t1_l1
+      end
+      factory :pl_p2_t1_l3_t1_l4 do
+        lines $hash_t1_l3_t1_l4
+      end
+      factory :pl_p2_t2_l1 do
+        lines $hash_t2_l1
+      end
+      factory :pl_p2_t3_l1 do
+        lines $hash_t3_l1
+      end
+    end
+  end
+
+  $pl_p1_t1_l1       = FactoryGirl.build :pl_p1_t1_l1
+  $pl_p1_t1_l1_t1_l2 = FactoryGirl.build :pl_p1_t1_l1_t1_l2
+  $pl_p1_t1_l1_t2_l1 = FactoryGirl.build :pl_p1_t1_l1_t2_l1
+  $pl_p1_t2_l1       = FactoryGirl.build :pl_p1_t2_l1
+  $pl_p2_t1_l1       = FactoryGirl.build :pl_p2_t1_l1
+  $pl_p2_t1_l3_t1_l4 = FactoryGirl.build :pl_p2_t1_l3_t1_l4
+  $pl_p2_t2_l1       = FactoryGirl.build :pl_p2_t2_l1
+  $pl_p2_t3_l1       = FactoryGirl.build :pl_p2_t3_l1
+
+
+  factory 'log_weaver/combined_log_index_key' do
+    factory :k_p1 do
+      prefix $p1
+      factory :k_p1_t1 do
+        timestamp $t1
+      end
+      factory :k_p1_t2 do
+        timestamp $t2
+      end
+    end
+    factory :k_p2 do
+      prefix $p2
+      factory :k_p2_t1 do
+        timestamp $t1
+      end
+      factory :k_p2_t2 do
+        timestamp $t2
+      end
+    end
+  end
+end
+
+$k_p1_t1 = FactoryGirl.build :k_p1_t1
+$k_p1_t2 = FactoryGirl.build :k_p1_t2
+$k_p2_t1 = FactoryGirl.build :k_p2_t1
+$k_p2_t2 = FactoryGirl.build :k_p2_t2
+
+$hash_p1_t1_l1_and_p2_t2_l1 = {
+    $k_p1_t1 => [$l1],
+    $k_p2_t2 => [$l1]
+}
+
+$hash_p1_t1_l1_and_p2_t1_l1 = {
+    $k_p1_t1 => [$l1],
+    $k_p2_t1 => [$l1]
+}
+
+$hash_p1_t2_l1_and_p2_t1_l1 = {
+    $k_p2_t1 => [$l1],
+    $k_p1_t2 => [$l1]
+}
+
+
+
+
+
+