appstats 0.12.5 → 0.13.0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    appstats (0.12.5)
+    appstats (0.13.0)
       daemons
       net-scp
       rails (>= 2.3.0)

data/db/migrations/20110225192624_add_appstats_log_collector_local_filename.rb

@@ -0,0 +1,9 @@
+class AddAppstatsLogCollectorLocalFilename < ActiveRecord::Migration
+  def self.up
+    add_column :appstats_log_collectors, :local_filename, :string
+  end
+
+  def self.down
+    remove_column :appstats_log_collectors, :local_filename
+  end
+end

data/db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended to check this file into your version control system.
 
-ActiveRecord::Schema.define(:version => 20110223212232) do
+ActiveRecord::Schema.define(:version => 20110225192624) do
 
   create_table "appstats_actions", :force => true do |t|
     t.string   "name"
@@ -89,6 +89,7 @@ ActiveRecord::Schema.define(:version => 20110223212232) do
     t.string   "status"
     t.datetime "created_at"
     t.datetime "updated_at"
+    t.string   "local_filename"
   end
 
   add_index "appstats_log_collectors", ["host"], :name => "index_appstats_log_collectors_on_host"

data/lib/appstats/log_collector.rb

@@ -5,11 +5,15 @@ module Appstats
   class LogCollector < ActiveRecord::Base
     set_table_name "appstats_log_collectors"
   
-    attr_accessible :host, :filename, :status
+    attr_accessible :host, :filename, :status, :local_filename
     has_many :entries, :table_name => 'appstats_entries', :foreign_key => 'appstats_log_collector_id', :order => 'action'
 
-    def local_filename
-      File.expand_path("#{File.dirname(__FILE__)}/../../log/appstats_remote_log_#{id}.log")
+    def calculated_local_filename
+      if Appstats::LogCollector.downloaded_log_directory.nil?
+        File.expand_path("#{File.dirname(__FILE__)}/../../log/appstats_remote_log_#{id}.log")
+      else
+        File.expand_path("#{Appstats::LogCollector.downloaded_log_directory}/appstats_remote_log_#{id}.log")
+      end
     end
 
     def processed_filename
@@ -30,6 +34,14 @@ module Appstats
       true
     end
 
+    def self.downloaded_log_directory=(value)
+      @@downloaded_log_directory = value
+    end
+
+    def self.downloaded_log_directory
+      @@downloaded_log_directory
+    end
+
     def self.should_process(last_time)
       return true if last_time.nil?
       Time.now.day > last_time.day
@@ -91,18 +103,19 @@ module Appstats
         password = normalized_logins[host][:password]
         begin
           Net::SCP.start( host, user, :password => password ) do |scp|
-            scp.download!( log_collector.filename, log_collector.local_filename )
+            scp.download!( log_collector.filename, log_collector.calculated_local_filename )
           end
         rescue Exception => e
           Appstats.log(:error,"Something bad occurred during Appstats::LogCollector#download_remote_files")
           Appstats.log(:error,e.message)
         end
-        if File.exists?(log_collector.local_filename)
-          Appstats.log(:info,"  - #{user}@#{host}:#{log_collector.filename} > #{log_collector.local_filename}")
+        if File.exists?(log_collector.calculated_local_filename)
+          log_collector.local_filename = log_collector.calculated_local_filename
           log_collector.status = 'downloaded'
+          Appstats.log(:info,"  - #{user}@#{host}:#{log_collector.filename} > #{log_collector.local_filename}")
           count += 1  
         else
-          Appstats.log(:error, "File #{log_collector.local_filename} did not download.")
+          Appstats.log(:error, "File #{log_collector.calculated_local_filename} did not download.")
           log_collector.status = 'failed_download'
         end
         log_collector.save

data/lib/appstats/logger.rb

@@ -20,7 +20,7 @@ module Appstats
       @@filename_template ||= 'appstats'
       @@filename_template
     end
-    
+
     def self.filename
       "#{filename_template}_#{today}.log"
     end

data/lib/appstats/version.rb

@@ -1,3 +1,3 @@
 module Appstats
-  VERSION = "0.12.5"
+  VERSION = "0.13.0"
 end

data/lib/templates/appstats_config.rb

@@ -1,3 +1,7 @@
 
-Appstats::Logger.filename_template = "log/appstats"
-Appstats::Logger.default_contexts[:app_name] = "YOUR_APP_NAME_HERE"
+# LOGGER CONFIGURATIONS (i.e. what is integrated within all of your apps)
+Appstats::Logger.filename_template = "log/appstats" # usually left as-is
+Appstats::Logger.default_contexts[:app_name] = "YOUR_APP_NAME_HERE" # replace me with your app name
+
+# LOG COLLECTOR (i.e. the process / app that  downloads all remote logs and processed them)
+Appstats::LogCollector.downloaded_log_directory = "/tmp" # only required for consolidated app

data/spec/entry_spec.rb

@@ -193,18 +193,18 @@ module Appstats
       end
       
       it "should understand an entry without contexts" do
-        entry = Entry.create_from_logger_string("0.12.5 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search")
+        entry = Entry.create_from_logger_string("0.13.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search")
         Entry.count.should == @before_count + 1
         entry.action.should == "address_search"
-        entry.raw_entry.should == "0.12.5 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search"
+        entry.raw_entry.should == "0.13.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search"
         entry.occurred_at.should == Time.parse("2010-09-21 23:15:20")
       end
       
       it "should understand contexts" do
-        entry = Entry.create_from_logger_string("0.12.5 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live")
+        entry = Entry.create_from_logger_string("0.13.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live")
         Entry.count.should == @before_count + 1
         entry.action.should == "address_filter"
-        entry.raw_entry.should == "0.12.5 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live"
+        entry.raw_entry.should == "0.13.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live"
         entry.occurred_at.should == Time.parse("2010-09-21 23:15:20")
         entry.contexts.size.should == 2
         entry.contexts[0].context_key = "app_name"
@@ -214,10 +214,10 @@ module Appstats
       end
       
       it "should handle 'action' as a context" do
-        entry = Entry.create_from_logger_string('0.12.5 setup[:,=,-n] 2011-02-24 12:59:57 action=page-view : action=save_ovcen : app_name=cdb')
+        entry = Entry.create_from_logger_string('0.13.0 setup[:,=,-n] 2011-02-24 12:59:57 action=page-view : action=save_ovcen : app_name=cdb')
         Entry.count.should == @before_count + 1
         entry.action.should == "page-view"
-        entry.raw_entry.should == "0.12.5 setup[:,=,-n] 2011-02-24 12:59:57 action=page-view : action=save_ovcen : app_name=cdb"
+        entry.raw_entry.should == "0.13.0 setup[:,=,-n] 2011-02-24 12:59:57 action=page-view : action=save_ovcen : app_name=cdb"
         entry.occurred_at.should == Time.parse("2011-02-24 12:59:57")
         entry.contexts.size.should == 2
         entry.contexts[0].context_key = "action"
@@ -228,10 +228,10 @@ module Appstats
       end
       
       it "should handle multiple of the same 'context'" do
-        entry = Entry.create_from_logger_string('0.12.5 setup[:,=,-n] 2011-02-24 12:59:57 action=page-view : app_name=market : app_name=cdb')
+        entry = Entry.create_from_logger_string('0.13.0 setup[:,=,-n] 2011-02-24 12:59:57 action=page-view : app_name=market : app_name=cdb')
         Entry.count.should == @before_count + 1
         entry.action.should == "page-view"
-        entry.raw_entry.should == "0.12.5 setup[:,=,-n] 2011-02-24 12:59:57 action=page-view : app_name=market : app_name=cdb"
+        entry.raw_entry.should == "0.13.0 setup[:,=,-n] 2011-02-24 12:59:57 action=page-view : app_name=market : app_name=cdb"
         entry.occurred_at.should == Time.parse("2011-02-24 12:59:57")
         entry.contexts.size.should == 2
         entry.contexts[0].context_key = "app_name"

data/spec/log_collector_spec.rb

@@ -12,12 +12,14 @@ module Appstats
       @login = { :host => "myhost.localnet", :user => "deployer", :password => "pass" }
       @login2 = { :host => "yourhost.localnet", :user => "deployer", :password => "ssap" }
       @logins = [@login2, @login]
+      Appstats::LogCollector.downloaded_log_directory = nil
     end
     
     after(:each) do
       LogCollector.all.each do |log_collector|
-        File.delete(log_collector.local_filename) if File.exists?(log_collector.local_filename)
+        File.delete(log_collector.local_filename) if !log_collector.local_filename.nil? && File.exists?(log_collector.local_filename)
       end
+      Appstats::LogCollector.downloaded_log_directory = nil
     end
     
     def simple_path(local_path_to_filename)
@@ -37,14 +39,19 @@ module Appstats
       it "should set status to unprocessed" do
         @log_collector.status.should == nil
       end
+      
+      it "should set local_filename to nil" do
+        @log_collector.local_filename.should == nil
+      end
     
       it "should set on constructor" do
-        log_collector = Appstats::LogCollector.new(:host => 'a', :filename => 'b', :status => 'c')
+        log_collector = Appstats::LogCollector.new(:host => 'a', :filename => 'b', :status => 'c', :local_filename => 'd')
         log_collector.host.should == 'a'
         log_collector.filename.should == 'b'
         log_collector.status.should == 'c'
+        log_collector.local_filename.should == 'd'
       end
-    
+      
     end
     
     
@@ -182,11 +189,21 @@ module Appstats
       
     end
     
-    describe "#local_filename" do
+    describe "#calculated_local_filename" do
+      
+      before(:each) do
+        Appstats::LogCollector.downloaded_log_directory = nil
+      end
       
       it "should return a standardized name with the log collector id" do
         log = LogCollector.create
-        log.local_filename.should == simple_path("../log/appstats_remote_log_#{log.id}.log")
+        log.calculated_local_filename.should == simple_path("../log/appstats_remote_log_#{log.id}.log")
+      end
+      
+      it "should use the downloaded_log_directory if set" do
+        Appstats::LogCollector.downloaded_log_directory = "/a/b/c"
+        log = LogCollector.create
+        log.calculated_local_filename.should == "/a/b/c/appstats_remote_log_#{log.id}.log"
       end
       
     end
@@ -241,8 +258,8 @@ module Appstats
         log1 = LogCollector.find_by_filename("/my/path/log/app1")
         log2 = LogCollector.find_by_filename("/my/path/log/app2")
         
-        File.open(log1.local_filename, 'w') {|f| f.write("testfile - delete") }
-        File.open(log2.local_filename, 'w') {|f| f.write("testfile - delete") }
+        File.open(log1.calculated_local_filename, 'w') {|f| f.write("testfile - delete") }
+        File.open(log2.calculated_local_filename, 'w') {|f| f.write("testfile - delete") }
 
 
         scp = mock(Net::SCP)
@@ -255,12 +272,16 @@ module Appstats
         LogCollector.download_remote_files(@logins).should == 2
         
         log1.reload and log2.reload
+        
+        log1.local_filename.should == log1.calculated_local_filename
+        log2.local_filename.should == log2.calculated_local_filename
+        
         log1.status.should == "downloaded"
         log2.status.should == "downloaded"
         
         LogCollector.load_remote_files(@login,"/my/path/log",["app3"]).should == 1
         log3 = LogCollector.find_by_filename("/my/path/log/app3")
-        File.open(log3.local_filename, 'w') {|f| f.write("testfile - delete") }
+        File.open(log3.calculated_local_filename, 'w') {|f| f.write("testfile - delete") }
         
         localfile = simple_path("../log/appstats_remote_log_#{log3.id}.log")
         scp.should_receive(:download!).with("/my/path/log/app3",localfile)
@@ -291,6 +312,8 @@ module Appstats
         LogCollector.load_remote_files(@login,"/my/path/log",["appstats1"]).should == 1
         log3 = LogCollector.find_by_filename("/my/path/log/appstats1")
         log3.status = "downloaded" and log3.save.should == true
+        log3.local_filename = log3.calculated_local_filename
+        log3.save
         File.open(log3.local_filename, 'w') {|f| f.write(Appstats::Logger.entry_to_s("test_action1") + "\n" + Appstats::Logger.entry_to_s("test_action2")) }
 
         Appstats.should_receive(:log).with(:info,"About to process 1 file(s).")
@@ -310,6 +333,7 @@ module Appstats
         LogCollector.load_remote_files(@login,"/my/path/log",["appstats1"]).should == 1
         log3 = LogCollector.find_by_filename("/my/path/log/appstats1")
         log3.status = "downloaded" and log3.save.should == true
+        log3.local_filename = log3.calculated_local_filename
         File.open(log3.local_filename, 'w') {|f| f.write(Appstats::Logger.entry_to_s("test_action1") + "\n" + Appstats::Logger.entry_to_s("test_action2")) }
 
         File.stub!(:open).and_raise("bad error")

data/spec/logger_spec.rb

@@ -115,7 +115,7 @@ module Appstats
       
       it "should accept numbers" do
         Appstats::Logger.entry(5, :blah => 6)   
-        Appstats::Logger.raw_read.should == ["0.12.5 setup[:,=,-n] 2010-09-21 23:15:20 action=5 : blah=6"]
+        Appstats::Logger.raw_read.should == ["0.13.0 setup[:,=,-n] 2010-09-21 23:15:20 action=5 : blah=6"]
       end
 
     end
@@ -124,7 +124,7 @@ module Appstats
 
       it "should look similar to regular entry" do
         Appstats::Logger.exception_entry(RuntimeError.new("blah"),:on => "login")   
-        Appstats::Logger.raw_read.should == ["0.12.5 setup[:,=,-n] 2010-09-21 23:15:20 action=appstats-exception : error=blah : on=login"]
+        Appstats::Logger.raw_read.should == ["0.13.0 setup[:,=,-n] 2010-09-21 23:15:20 action=appstats-exception : error=blah : on=login"]
       end
       
     end
@@ -141,47 +141,47 @@ module Appstats
     
        it "should handle a statistics entry" do
          expected = { :action => "address_search", :timestamp => "2010-09-21 23:15:20" }
-         actual = Appstats::Logger.entry_to_hash("0.12.5 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search")
+         actual = Appstats::Logger.entry_to_hash("0.13.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search")
          actual.should == expected
        end
        
        it "should handle contexts" do
          expected = { :action => "address_filter", :timestamp => "2010-09-21 23:15:20", :server => "Live", :app_name => 'Market' }
-         actual = Appstats::Logger.entry_to_hash("0.12.5 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live")
+         actual = Appstats::Logger.entry_to_hash("0.13.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live")
          actual.should == expected
        end
 
        it "should handle multiple actions" do
          expected = { :action => ["address_filter", "blah"], :timestamp => "2010-09-21 23:15:20", :server => "Live", :app_name => 'Market' }
-         actual = Appstats::Logger.entry_to_hash("0.12.5 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : action=blah : app_name=Market : server=Live")
+         actual = Appstats::Logger.entry_to_hash("0.13.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : action=blah : app_name=Market : server=Live")
          actual.should == expected
        end
 
        it "should handle multiple of same context" do
          expected = { :action => "address_filter", :timestamp => "2010-09-21 23:15:20", :server => "Live", :app_name => ['Sin','Market'] }
-         actual = Appstats::Logger.entry_to_hash("0.12.5 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Sin : app_name=Market : server=Live")
+         actual = Appstats::Logger.entry_to_hash("0.13.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Sin : app_name=Market : server=Live")
          actual.should == expected
        end
 
        it "should handle no actions" do
          expected = { :action => "UNKNOWN_ACTION", :timestamp => "2010-09-21 23:15:20", :server => "Live", :app_name => 'Market' }
-         actual = Appstats::Logger.entry_to_hash("0.12.5 setup[:,=,-n] 2010-09-21 23:15:20 app_name=Market : server=Live")
+         actual = Appstats::Logger.entry_to_hash("0.13.0 setup[:,=,-n] 2010-09-21 23:15:20 app_name=Market : server=Live")
          actual.should == expected
        end
     
        it "should handle actions with the delimiter (and change the delimiter)" do
          expected = { :action => "address:=search-n", :timestamp => "2010-09-21 23:15:20" }
-         actual = Appstats::Logger.entry_to_hash("0.12.5 setup[::,==,--n] 2010-09-21 23:15:20 action==address:=search-n")
+         actual = Appstats::Logger.entry_to_hash("0.13.0 setup[::,==,--n] 2010-09-21 23:15:20 action==address:=search-n")
          actual.should == expected
     
          expected = { :action => "address::search==--n", :timestamp => "2010-09-21 23:15:20" }
-         actual = Appstats::Logger.entry_to_hash("0.12.5 setup[:::,===,---n] 2010-09-21 23:15:20 action===address::search==--n")
+         actual = Appstats::Logger.entry_to_hash("0.13.0 setup[:::,===,---n] 2010-09-21 23:15:20 action===address::search==--n")
          actual.should == expected
        end
     
        it "should handle contexts with the delimiter (and change the delimiter)" do
          expected = { :action => "address", :timestamp => "2010-09-21 23:15:20", :server => "market:eval=-n" }
-         actual = Appstats::Logger.entry_to_hash("0.12.5 setup[::,==,--n] 2010-09-21 23:15:20 action==address :: server==market:eval=-n")
+         actual = Appstats::Logger.entry_to_hash("0.13.0 setup[::,==,--n] 2010-09-21 23:15:20 action==address :: server==market:eval=-n")
          actual.should == expected
        end
        
@@ -190,66 +190,66 @@ module Appstats
     describe "#entry_to_s" do
       
       it "should handle a statistics entry" do
-        expected = "0.12.5 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search"
+        expected = "0.13.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search"
         actual = Appstats::Logger.entry_to_s("address_search")
         actual.should == expected
       end
       
       it "should handle numbers" do
-        expected = "0.12.5 setup[:,=,-n] 2010-09-21 23:15:20 action=1 : note=2.2"
+        expected = "0.13.0 setup[:,=,-n] 2010-09-21 23:15:20 action=1 : note=2.2"
         actual = Appstats::Logger.entry_to_s(1,:note => 2.2)
         actual.should == expected
       end
       
       it "should handle default contexts" do
         Appstats::Logger.default_contexts[:app_name] = "market"
-        expected = "0.12.5 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search : app_name=market"
+        expected = "0.13.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search : app_name=market"
         actual = Appstats::Logger.entry_to_s("address_search")
         actual.should == expected
       end
       
       it "should handle contexts (and sort them by symbol)" do
-        expected = "0.12.5 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live"
+        expected = "0.13.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live"
         actual = Appstats::Logger.entry_to_s("address_filter", { :server => "Live", :app_name => 'Market' })
         actual.should == expected
       end
 
       it "should handle actions with the delimiter (and change the delimiter)" do
-        expected = "0.12.5 setup[::,==,--n] 2010-09-21 23:15:20 action==address:=search-n"
+        expected = "0.13.0 setup[::,==,--n] 2010-09-21 23:15:20 action==address:=search-n"
         actual = Appstats::Logger.entry_to_s("address:=search-n")
         actual.should == expected
 
-        expected = "0.12.5 setup[:::,===,---n] 2010-09-21 23:15:20 action===address::search==--n"
+        expected = "0.13.0 setup[:::,===,---n] 2010-09-21 23:15:20 action===address::search==--n"
         actual = Appstats::Logger.entry_to_s("address::search==--n")
         actual.should == expected
       end
 
       it "should handle contexts with the delimiter (and change the delimiter)" do
-        expected = "0.12.5 setup[::,==,--n] 2010-09-21 23:15:20 action==address :: server==market:eval=-n"
+        expected = "0.13.0 setup[::,==,--n] 2010-09-21 23:15:20 action==address :: server==market:eval=-n"
         actual = Appstats::Logger.entry_to_s("address", :server => 'market:eval=-n')
         actual.should == expected
       end
 
       it "should ignore spaces" do
-        expected = "0.12.5 setup[:,=,-n] 2010-09-21 23:15:20 action=address search"
+        expected = "0.13.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address search"
         actual = Appstats::Logger.entry_to_s("address search")
         actual.should == expected
       end
       
       it "should convert newlines in action" do
-        expected = "0.12.5 setup[:,=,-n] 2010-09-21 23:15:20 action=address_-nsearch"
+        expected = "0.13.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_-nsearch"
         actual = Appstats::Logger.entry_to_s("address_\nsearch")
         actual.should == expected
       end
 
       it "should convert newlines in context" do
-        expected = "0.12.5 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search : blah=some-nlong-nstatement"
+        expected = "0.13.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search : blah=some-nlong-nstatement"
         actual = Appstats::Logger.entry_to_s("address_search",:blah => "some\nlong\nstatement")
         actual.should == expected
       end
       
       it "should convert newlines based on the delimiter" do
-        expected = "0.12.5 setup[::,==,--n] 2010-09-21 23:15:20 action==address:=--nsearch-n"
+        expected = "0.13.0 setup[::,==,--n] 2010-09-21 23:15:20 action==address:=--nsearch-n"
         actual = Appstats::Logger.entry_to_s("address:=\nsearch-n")
         actual.should == expected
       end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: appstats
 version: !ruby/object:Gem::Version 
-  hash: 37
+  hash: 43
   prerelease: false
   segments: 
   - 0
-  - 12
-  - 5
-  version: 0.12.5
+  - 13
+  - 0
+  version: 0.13.0
 platform: ruby
 authors: 
 - Andrew Forward
@@ -158,6 +158,7 @@ files:
 - db/migrations/20110217234136_add_appstats_results_contexts.rb
 - db/migrations/20110222215437_create_appstats_jobs.rb
 - db/migrations/20110223212232_add_appstats_entries_week_and_quarter.rb
+- db/migrations/20110225192624_add_appstats_log_collector_local_filename.rb
 - db/schema.rb
 - lib/appstats.rb
 - lib/appstats/action.rb