oas-log-collector 0.1.4 → 0.1.5

data/Gemfile

@@ -1,4 +1,4 @@
 source "http://rubygems.org"
 
-# Specify your gem's dependencies in salvatore.gemspec
+# Specify your gem's dependencies in oas-log-collector.gemspec
 gemspec

data/LICENSE

@@ -1,4 +1,4 @@
-opyright (c) 2012 Realmedia Latin America LLC
+Copyright (c) 2012 Realmedia Latin America LLC
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.md

@@ -1,10 +1,35 @@
-OAS Log Collector [![Build Status](https://secure.travis-ci.org/realmedia/oas-log-collector.png)](http://travis-ci.org/realmedia/oas-log-collector)
-=================
+OAS Log Collector [![Build Status](https://secure.travis-ci.org/realmedia/oas-log-collector.png)][travis]
+=====================
 
+Simple Open Adstream log collector tool.
 
-Installation
-------------
+[travis]: http://travis-ci.org/realmedia/oas-log-collector
+
+Requirements
+-----------------
+
+* Ruby 1.8.7, 1.9.2 or 1.9.3.
+* Redis 2.0 or greater.
 
-OAS Log Collector is available through [Rubygems](http://rubygems.org/gems/oas-log-collector) and can be installed via:
+Installation
+-----------------
 
     gem install oas-log-collector
+
+Usage
+-----------------
+
+Starting. There is an sample file under the `examples/` directory.
+
+    oas-log-collector -C config.yml -P /var/run/oas-log-collector.pid -l /var/log/oas-log-collector.log -d
+
+Stopping
+
+    oas-log-collector -k -P /var/run/oas-log-collector.pid
+
+**Important:** Make sure you have passwordless authentication to the servers were the logs are located.
+
+Copyright
+----------------
+
+Copyright © 2012 Realmedia Latin America LLC. See [LICENSE](https://github.com/realmedia/oas-log-collector/blob/master/LICENSE) for details.

data/bin/oas-log-collector

@@ -6,7 +6,7 @@ $:.unshift(path) if File.directory?(path) && !$:.include?(path)
 
 require 'dante'
 require 'yaml'
-require 'oas/log_collector/cli'
+require 'oas/log_collector/manager'
 
 runner = Dante::Runner.new('oas-log-collector')
 runner.description = "OAS Log Collector"
@@ -18,8 +18,8 @@ end
 
 runner.execute do |options|
   begin
-    cli = OAS::LogCollector::CLI.new(YAML.load_file(options[:config_file]))
-    cli.run
+    manager = OAS::LogCollector::Manager.new(YAML.load_file(options[:config_file]))
+    manager.start!
   rescue Interrupt
     # shutdown 
   end

data/examples/config.yml

@@ -1,11 +1,12 @@
 # Sample configuration file for OAS Log Collector.
 #  oas-log-collector -C config.yml
 ---
-:days_to_search: 10
-:files_to_copy:  90
-:sleep_seconds:  56400
-:logs_path: '~/oas-log-collector'
-:source_logs_path: '/home/central/RealMedia/ads/Logs/uv/out'
-:sources:
-  - ['de-01', 'central']
-  - ['de-02', 'central']
+:redis_url: 'redis://localhost:6379/0' # Redis url.
+:days_to_search: 10    # Search files which were modified n*24 hs ago.
+:files_to_copy:  90    # Collect at max n files per run.
+:sleep_seconds:  900   # Time in seconds to wait betweetn runs.
+:logs_path: '~/oas-log-collector'  # Directory were the collected logs are going to be saved.
+:source_logs_path: '/path/to/logs/directory' # Directory in remote servers were the logs are located.
+:sources:              # Array of log servers.
+  - ['server1', 'appuser']   # [host, user, port] - Port is optional, defaults to 22.
+  - ['server2', 'appuser']

data/lib/oas/log_collector.rb

@@ -10,7 +10,7 @@ module OAS
       :logs_path        => "~/oas-log-collector",
       :days_to_search   => 10,
       :files_to_copy    => 90,
-      :sleep_seconds    => 56400,
+      :sleep_seconds    => 900,
       :redis_url        => 'redis://localhost:6379/0'
     }
 

data/lib/oas/log_collector/log_file.rb

@@ -0,0 +1,30 @@
+module OAS
+  module LogCollector
+    class LogFile
+
+      attr_reader :name, :time
+
+      def initialize(file_path)
+        @name = File.basename(file_path)
+        @time = Time.at(name[/.*\.log\.([0-9]+)\.?(.+)?$/, 1].to_i)
+      end
+
+      def timestamp
+        @time.to_i
+      end
+
+      def year
+        @time.strftime("%Y")
+      end
+
+      def month
+        @time.strftime("%m")
+      end
+
+      def day
+        @time.strftime("%d")
+      end
+
+    end
+  end
+end

data/lib/oas/log_collector/manager.rb

@@ -1,30 +1,59 @@
+require 'oas/log_collector'
+require 'oas/log_collector/log_file'
+require 'oas/log_collector/source'
+
 module OAS
   module LogCollector
     class Manager
 
+      def initialize(opts = {})
+        sources = opts.delete(:sources) || []
+        sources.each { |host, user, port| (opts[:sources] ||= []) << OAS::LogCollector::Source.new(host, user, port || 22) }
+
+        options.merge!(opts)
+      end
+
+      def start!
+        loop do
+          threads = []
+          options[:sources].each do |source|
+            threads << Thread.new do
+              collect(source)
+            end
+          end
+          threads.each(&:join)
+
+          sleep options[:sleep_seconds]
+        end
+      end
+
+      def options
+        OAS::LogCollector.options
+      end
+
       def collect(source)
         hostname = source.host
 
-        result = source.find(OAS::LogCollector.options[:source_logs_path], {
-          :mtime => "-#{OAS::LogCollector.options[:days_to_search]}",
+        result = source.find(options[:source_logs_path], {
+          :mtime => "-#{options[:days_to_search]}",
           :type  => 'f',
           :regex => '.*\.log\.[0-9]+\.bz2'
         })
 
-        files = []
+        files = Hash.new
         total = 0
         last_timestamp = last_run = get_last_run(hostname)
         result.sort!.each do |path|
-          timestamp = Integer(File.basename(path, ".bz2")[/.*\.log\.([0-9]+)/,1])
-          if timestamp > Integer(last_run) && total < OAS::LogCollector.options[:files_to_copy]
-            files << path
+          log_file  = OAS::LogCollector::LogFile.new(path)
+          if log_file.timestamp > Integer(last_run) && total < options[:files_to_copy]
+            files[path] = File.join(options[:logs_path], hostname, log_file.year, log_file.month, log_file.day, log_file.name)
+            last_timestamp = log_file.timestamp
             total += 1
-            last_timestamp = timestamp
           end
         end
 
         if files.any?
-          source.copy(files, OAS::LogCollector.options[:logs_path])
+          source.download(files)
           set_last_run(hostname, last_timestamp)
         end
       end

data/lib/oas/log_collector/source.rb

@@ -24,13 +24,11 @@ module OAS
         out.split("\n")
       end
 
-      def copy(files, logs_path)
-        dest_path = File.join(logs_path, host)
-        FileUtils.mkdir_p(dest_path)
-
+      def download(files)
         Net::SCP.start(host, user, :port => port) do |scp|
-          files.each do |path|
-            scp.download!(path, File.join(dest_path, File.basename(path)))
+          files.each do |remote_path, dest_path|
+            FileUtils.mkdir_p(File.dirname(dest_path))
+            scp.download!(remote_path, dest_path)
           end
         end
       end

data/lib/oas/log_collector/version.rb

@@ -1,5 +1,5 @@
 module OAS
   module LogCollector
-    VERSION = "0.1.4"
+    VERSION = "0.1.5"
   end
 end

data/test/test_log_file.rb

@@ -0,0 +1,17 @@
+require 'helper'
+require 'oas/log_collector/log_file'
+require 'date'
+
+class TestLogFile < MiniTest::Unit::TestCase
+  def setup
+    @timestamp = DateTime.new(2012,8,1,15,30).to_time.to_i
+  end
+
+  def test_parse_on_initialize
+    logfile = OAS::LogCollector::LogFile.new("adstream.log.#{@timestamp}.bz2")
+    assert_equal @timestamp, logfile.timestamp
+    assert_equal "2012", logfile.year
+    assert_equal "08", logfile.month
+    assert_equal "01", logfile.day
+  end
+end

data/test/test_manager.rb

@@ -3,16 +3,17 @@ require 'oas/log_collector/manager'
 
 class TestManager < MiniTest::Unit::TestCase
   def setup
-    OAS::LogCollector.options.merge!({
+    @timestamp = DateTime.new(2012,8,1,15,30).to_time.to_i
+    @hostname = 'remote1'
+    @result   = ["adstream.log.#{@timestamp}.bz2", "adstream.log.#{@timestamp + 60}.bz2", "adstream.log.#{@timestamp + 120}.bz2"]
+    @manager  = OAS::LogCollector::Manager.new({
       :source_logs_path => "/tmp",
       :files_to_copy  => 90,
       :days_to_search => 10,
       :logs_path => "/tmp/logs"
     })
-    @result  = ["adstream.log.223456789.bz2", "adstream.log.123456789.bz2", "adstream.log.323456789.bz2"]
-    @manager = OAS::LogCollector::Manager.new
-    @source  = Minitest::Mock.new
-    @source.expect :host, 'localhost'
+    @source   = Minitest::Mock.new
+    @source.expect :host, @hostname
     @source.expect :find, @result, ["/tmp", { :mtime => '-10', :type  => 'f', :regex => '.*\.log\.[0-9]+\.bz2' }]
 
     OAS::LogCollector.redis = REDIS
@@ -20,29 +21,36 @@ class TestManager < MiniTest::Unit::TestCase
   end
 
   def test_set_last_run_to_the_last_downloaded_file
-    @source.expect :copy, nil, [@result, "/tmp/logs"]
+    files = Hash.new
+    @result.each { |p| files[p] = build_destination_path(p) }
+    @source.expect :download, nil, [files]
     @manager.collect(@source)
     @source.verify
-    assert_equal "323456789", @manager.get_last_run('localhost')
+    assert_equal @timestamp + 120, @manager.get_last_run(@hostname).to_i
   end
 
   def test_do_not_select_more_files_than_allowed
-    @manager.set_last_run('localhost', '123456789')
+    @manager.set_last_run(@hostname, @timestamp)
 
     OAS::LogCollector.options.merge!({
       :files_to_copy => 1,
     })
-
-    @source.expect :copy, nil, [["adstream.log.223456789.bz2"], "/tmp/logs"]
+    path = "adstream.log.#{@timestamp + 60}.bz2"
+    @source.expect :download, nil, [Hash[path, build_destination_path(path)]]
     @manager.collect(@source)
     @source.verify
   end
 
   def test_select_files_with_timestamp_greater_than_last_run
-    @manager.set_last_run('localhost', '223456789')
+    @manager.set_last_run(@hostname, @timestamp + 60)
 
-    @source.expect :copy, nil, [["adstream.log.323456789.bz2"], "/tmp/logs"]
+    path = "adstream.log.#{@timestamp + 120}.bz2"
+    @source.expect :download, nil, [Hash[path, build_destination_path(path)]]
     @manager.collect(@source)
     @source.verify
   end
+
+  def build_destination_path(path)
+    File.join("/tmp/logs", @hostname, "2012", "08", "01", File.basename(path))
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: oas-log-collector
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-06 00:00:00.000000000 Z
+date: 2012-08-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dante
@@ -158,12 +158,13 @@ files:
 - examples/config.yml
 - lib/oas-log-collector.rb
 - lib/oas/log_collector.rb
-- lib/oas/log_collector/cli.rb
+- lib/oas/log_collector/log_file.rb
 - lib/oas/log_collector/manager.rb
 - lib/oas/log_collector/source.rb
 - lib/oas/log_collector/version.rb
 - oas-log-collector.gemspec
 - test/helper.rb
+- test/test_log_file.rb
 - test/test_manager.rb
 - test/test_source.rb
 homepage: http://github.com/realmedia/oas-log-collector
@@ -180,7 +181,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1520991714995813635
+      hash: -1912528220465064973
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -189,7 +190,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1520991714995813635
+      hash: -1912528220465064973
 requirements: []
 rubyforge_project: oas-ivc
 rubygems_version: 1.8.24

data/lib/oas/log_collector/cli.rb

@@ -1,41 +0,0 @@
-require 'oas/log_collector'
-require 'oas/log_collector/manager'
-require 'oas/log_collector/source'
-
-module OAS
-  module LogCollector
-    class CLI
-      attr_writer :manager
-
-      def initialize(opts = {})
-        sources = opts.delete(:sources) || []
-        sources.each { |host, user, port| (opts[:sources] ||= []) << OAS::LogCollector::Source.new(host, user, port || 22) }
-
-        options.merge!(opts)
-      end
-
-      def manager
-        @manager ||= OAS::LogCollector::Manager.new
-      end
-
-      def options
-        OAS::LogCollector.options
-      end
-
-      def run
-        loop do
-          threads = []
-          options[:sources].each do |source|
-            threads << Thread.new do
-              manager.collect(source)
-            end
-          end
-          threads.each(&:join)
-
-          sleep options[:sleep_seconds]
-        end
-      end
-
-    end
-  end
-end