oas-log-collector 0.1.4

data/.gitignore

@@ -0,0 +1,6 @@
+*.gem
+.bundle
+Gemfile.lock
+coverage
+doc/*
+pkg/

data/.rvmrc

@@ -0,0 +1 @@
+rvm use ruby-1.9.3-p194@oas-log-collector --create

data/.travis.yml

@@ -0,0 +1,4 @@
+rvm:
+  - 1.8.7
+  - 1.9.2
+  - 1.9.3

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in salvatore.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,20 @@
+opyright (c) 2012 Realmedia Latin America LLC
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,10 @@
+OAS Log Collector [![Build Status](https://secure.travis-ci.org/realmedia/oas-log-collector.png)](http://travis-ci.org/realmedia/oas-log-collector)
+=================
+
+
+Installation
+------------
+
+OAS Log Collector is available through [Rubygems](http://rubygems.org/gems/oas-log-collector) and can be installed via:
+
+    gem install oas-log-collector

data/Rakefile

@@ -0,0 +1,11 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+Rake::TestTask.new do |t|
+  t.libs = ["test"]
+  t.verbose = true
+  t.test_files = FileList['test/**/test_*.rb']
+end
+
+task :default => :test

data/bin/oas-log-collector

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+require 'rubygems' unless defined?(Gem)
+
+path = File.expand_path('../../lib', __FILE__)
+$:.unshift(path) if File.directory?(path) && !$:.include?(path)
+
+require 'dante'
+require 'yaml'
+require 'oas/log_collector/cli'
+
+runner = Dante::Runner.new('oas-log-collector')
+runner.description = "OAS Log Collector"
+runner.with_options do |opts|
+  opts.on("-C", "--config PATH", String, "Path to YAML config file") do |arg|
+    options[:config_file] = File.expand_path(arg)
+  end
+end
+
+runner.execute do |options|
+  begin
+    cli = OAS::LogCollector::CLI.new(YAML.load_file(options[:config_file]))
+    cli.run
+  rescue Interrupt
+    # shutdown 
+  end
+end

data/examples/config.yml

@@ -0,0 +1,11 @@
+# Sample configuration file for OAS Log Collector.
+#  oas-log-collector -C config.yml
+---
+:days_to_search: 10
+:files_to_copy:  90
+:sleep_seconds:  56400
+:logs_path: '~/oas-log-collector'
+:source_logs_path: '/home/central/RealMedia/ads/Logs/uv/out'
+:sources:
+  - ['de-01', 'central']
+  - ['de-02', 'central']

data/lib/oas/log_collector/cli.rb

@@ -0,0 +1,41 @@
+require 'oas/log_collector'
+require 'oas/log_collector/manager'
+require 'oas/log_collector/source'
+
+module OAS
+  module LogCollector
+    class CLI
+      attr_writer :manager
+
+      def initialize(opts = {})
+        sources = opts.delete(:sources) || []
+        sources.each { |host, user, port| (opts[:sources] ||= []) << OAS::LogCollector::Source.new(host, user, port || 22) }
+
+        options.merge!(opts)
+      end
+
+      def manager
+        @manager ||= OAS::LogCollector::Manager.new
+      end
+
+      def options
+        OAS::LogCollector.options
+      end
+
+      def run
+        loop do
+          threads = []
+          options[:sources].each do |source|
+            threads << Thread.new do
+              manager.collect(source)
+            end
+          end
+          threads.each(&:join)
+
+          sleep options[:sleep_seconds]
+        end
+      end
+
+    end
+  end
+end

data/lib/oas/log_collector/manager.rb

@@ -0,0 +1,45 @@
+module OAS
+  module LogCollector
+    class Manager
+
+      def collect(source)
+        hostname = source.host
+
+        result = source.find(OAS::LogCollector.options[:source_logs_path], {
+          :mtime => "-#{OAS::LogCollector.options[:days_to_search]}",
+          :type  => 'f',
+          :regex => '.*\.log\.[0-9]+\.bz2'
+        })
+
+        files = []
+        total = 0
+        last_timestamp = last_run = get_last_run(hostname)
+        result.sort!.each do |path|
+          timestamp = Integer(File.basename(path, ".bz2")[/.*\.log\.([0-9]+)/,1])
+          if timestamp > Integer(last_run) && total < OAS::LogCollector.options[:files_to_copy]
+            files << path
+            total += 1
+            last_timestamp = timestamp
+          end
+        end
+
+        if files.any?
+          source.copy(files, OAS::LogCollector.options[:logs_path])
+          set_last_run(hostname, last_timestamp)
+        end
+      end
+
+      def get_last_run(hostname)
+        OAS::LogCollector.redis do |conn|
+          conn.hget("timestamps", hostname) || 0
+        end
+      end
+
+      def set_last_run(hostname, timestamp)
+        OAS::LogCollector.redis do |conn|
+          conn.hset("timestamps", hostname, timestamp)
+        end
+      end
+    end
+  end
+end

data/lib/oas/log_collector/source.rb

@@ -0,0 +1,49 @@
+require 'net/ssh'
+require 'net/scp'
+require 'fileutils'
+
+module OAS
+  module LogCollector
+    class Source
+      class Error < StandardError; end
+
+      attr_reader :host, :user, :port
+
+      def initialize(host, user, port = 22)
+        @host, @user, @port = host, user, port
+      end
+
+      def find(path, options = {})
+        out = ''
+        ssh = Net::SSH.start(host, user, :port => port)
+        ssh.exec!(build_find(path, options)) do |channel, stream, data|
+          raise Error.new(data) if stream == :stderr
+          out << data
+        end
+        ssh.close
+        out.split("\n")
+      end
+
+      def copy(files, logs_path)
+        dest_path = File.join(logs_path, host)
+        FileUtils.mkdir_p(dest_path)
+
+        Net::SCP.start(host, user, :port => port) do |scp|
+          files.each do |path|
+            scp.download!(path, File.join(dest_path, File.basename(path)))
+          end
+        end
+      end
+
+      private
+      def build_find(path, options)
+        str = "find #{path}"
+        options.each do |k, v|
+          str += " -#{k} '#{v}'"
+        end
+        str
+      end
+
+    end
+  end
+end

data/lib/oas/log_collector/version.rb

@@ -0,0 +1,5 @@
+module OAS
+  module LogCollector
+    VERSION = "0.1.4"
+  end
+end

data/lib/oas/log_collector.rb

@@ -0,0 +1,37 @@
+require 'oas/log_collector/version'
+require 'connection_pool'
+require 'redis'
+
+module OAS
+  module LogCollector
+    DEFAULTS = {
+      :sources  => [],
+      :source_logs_path => "/tmp",
+      :logs_path        => "~/oas-log-collector",
+      :days_to_search   => 10,
+      :files_to_copy    => 90,
+      :sleep_seconds    => 56400,
+      :redis_url        => 'redis://localhost:6379/0'
+    }
+
+    extend self
+
+    def options
+      @options ||= DEFAULTS.dup
+    end
+
+    def redis(&block)
+      @redis ||= ConnectionPool.new(:timeout => 1, :size => options[:sources].size) { Redis.connect(:url => options[:redis_url]) }
+      raise ArgumentError, "requires a block" if !block
+      @redis.with(&block)
+    end
+
+    def redis=(conn)
+      if conn.is_a?(ConnectionPool)
+        @redis = conn
+      else
+        raise ArgumentError, "redis= requires a ConnectionPool"
+      end
+    end
+  end
+end

data/lib/oas-log-collector.rb

@@ -0,0 +1 @@
+require 'oas/log_collector'

data/oas-log-collector.gemspec

@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "oas/log_collector/version"
+
+Gem::Specification.new do |s|
+  s.name        = "oas-log-collector"
+  s.version     = OAS::LogCollector::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Damian Caruso"]
+  s.email       = ["damian.caruso@gmail.com"]
+  s.homepage    = "http://github.com/realmedia/oas-log-collector"
+  s.description = s.summary = "Description goes here"
+
+  s.rubyforge_project = "oas-ivc"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency "dante",   "~> 0.1"
+  s.add_dependency "net-ssh", "~> 2.5"
+  s.add_dependency "net-scp", "~> 1.0.4"
+  s.add_dependency "redis",   "~> 3"
+  s.add_dependency "connection_pool", "~> 0.9.2"
+  
+  s.add_development_dependency "minitest", "~> 3"
+  s.add_development_dependency "rake"
+  s.add_development_dependency "simplecov"
+end

data/test/helper.rb

@@ -0,0 +1,8 @@
+require 'simplecov'
+SimpleCov.start
+
+require 'minitest/autorun'
+require 'minitest/pride'
+
+require 'oas/log_collector'
+REDIS = ConnectionPool.new(:timeout => 1, :size => 1) { Redis.connect(:url => "redis://localhost/15") }

data/test/test_manager.rb

@@ -0,0 +1,48 @@
+require 'helper'
+require 'oas/log_collector/manager'
+
+class TestManager < MiniTest::Unit::TestCase
+  def setup
+    OAS::LogCollector.options.merge!({
+      :source_logs_path => "/tmp",
+      :files_to_copy  => 90,
+      :days_to_search => 10,
+      :logs_path => "/tmp/logs"
+    })
+    @result  = ["adstream.log.223456789.bz2", "adstream.log.123456789.bz2", "adstream.log.323456789.bz2"]
+    @manager = OAS::LogCollector::Manager.new
+    @source  = Minitest::Mock.new
+    @source.expect :host, 'localhost'
+    @source.expect :find, @result, ["/tmp", { :mtime => '-10', :type  => 'f', :regex => '.*\.log\.[0-9]+\.bz2' }]
+
+    OAS::LogCollector.redis = REDIS
+    OAS::LogCollector.redis {|c| c.flushdb }
+  end
+
+  def test_set_last_run_to_the_last_downloaded_file
+    @source.expect :copy, nil, [@result, "/tmp/logs"]
+    @manager.collect(@source)
+    @source.verify
+    assert_equal "323456789", @manager.get_last_run('localhost')
+  end
+
+  def test_do_not_select_more_files_than_allowed
+    @manager.set_last_run('localhost', '123456789')
+
+    OAS::LogCollector.options.merge!({
+      :files_to_copy => 1,
+    })
+
+    @source.expect :copy, nil, [["adstream.log.223456789.bz2"], "/tmp/logs"]
+    @manager.collect(@source)
+    @source.verify
+  end
+
+  def test_select_files_with_timestamp_greater_than_last_run
+    @manager.set_last_run('localhost', '223456789')
+
+    @source.expect :copy, nil, [["adstream.log.323456789.bz2"], "/tmp/logs"]
+    @manager.collect(@source)
+    @source.verify
+  end
+end

data/test/test_source.rb

@@ -0,0 +1,61 @@
+require 'helper'
+require 'oas/log_collector/source'
+
+module MySSHSession
+  def exec!(command, &block)
+    _exec("/tmp/fileone.txt\n/tmp/filetwo.txt", :stdout, &block)
+  end
+
+  def close; end
+
+  private
+  def _exec(data, stream = :stdout, &block)
+    if block_given?
+      block.call true, stream, data
+    else
+      return data
+    end
+  end
+end
+
+module MySSHSessionWithError
+  include MySSHSession
+
+  def exec!(command, &block)
+    _exec("something was wrong", :stderr, &block)
+  end
+end
+
+class TestSource < MiniTest::Unit::TestCase
+  def setup
+    @source = OAS::LogCollector::Source.new('remote-host.com', 'user')
+  end
+
+  def test_find_with_error
+    session = Class.new { include MySSHSessionWithError }
+    Net::SSH.stub :start, session.new do
+      assert_raises OAS::LogCollector::Source::Error do 
+        @source.find("/tmp")
+      end
+    end
+  end
+
+  def test_find_successful
+    session = Class.new { include MySSHSession }
+    Net::SSH.stub :start, session.new do
+      result = @source.find("/tmp")
+      assert_instance_of Array, result
+      assert_equal 2, result.size
+    end
+  end
+
+  def test_find_command_builder
+    session = Minitest::Mock.new
+    session.expect :exec!, "", ["find /tmp -type 'f' -name '*.txt'"]
+    session.expect :close, nil
+    Net::SSH.stub :start, session do
+      result = @source.find("/tmp", { :type => 'f', :name => '*.txt' })
+    end
+    session.verify
+  end
+end

metadata

@@ -0,0 +1,199 @@
+--- !ruby/object:Gem::Specification
+name: oas-log-collector
+version: !ruby/object:Gem::Version
+  version: 0.1.4
+  prerelease: 
+platform: ruby
+authors:
+- Damian Caruso
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-08-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: dante
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: net-ssh
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.5'
+- !ruby/object:Gem::Dependency
+  name: net-scp
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.4
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: connection_pool
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Description goes here
+email:
+- damian.caruso@gmail.com
+executables:
+- oas-log-collector
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rvmrc
+- .travis.yml
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/oas-log-collector
+- examples/config.yml
+- lib/oas-log-collector.rb
+- lib/oas/log_collector.rb
+- lib/oas/log_collector/cli.rb
+- lib/oas/log_collector/manager.rb
+- lib/oas/log_collector/source.rb
+- lib/oas/log_collector/version.rb
+- oas-log-collector.gemspec
+- test/helper.rb
+- test/test_manager.rb
+- test/test_source.rb
+homepage: http://github.com/realmedia/oas-log-collector
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 1520991714995813635
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 1520991714995813635
+requirements: []
+rubyforge_project: oas-ivc
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Description goes here
+test_files: []