s3log 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 21b8adc04ba1abb0fa4de1cd1ba9d29b53a1a3c0
+  data.tar.gz: c2944d29a19ec7acd679bac7afb758db7eeaae52
+SHA512:
+  metadata.gz: b4ea2df8459df2321f9ce7c3a449f8bd6ab2d8e33d3d2cba2176d364fefc9e7fc5b7908dd58cb44391a428856a9f11a68cad1def236aee4c48fb795aefa3715e
+  data.tar.gz: 7399046d15b3265d0e7b6a08b1368887a243b045071898b0c5ef0061a8a3b6bdf12084b8926e327446d1551776a56211bf013d207c43128edda9637969871fd0

data/.gitignore

@@ -0,0 +1,20 @@
+*.gem
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/
+
+.ruby-version

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+cache: bundler
+rvm:
+  - 2.0.0

data/CHANGELOG.md

@@ -0,0 +1,6 @@
+S3log Changelog
+-------------------
+
+### v0.0.1 - wip
+
+* setup base gem skeleton

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in s3log.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,57 @@
+PATH
+  remote: .
+  specs:
+    s3log (0.0.1)
+      aws-sdk
+      thor
+      whenever
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.0.2)
+      i18n (~> 0.6, >= 0.6.4)
+      minitest (~> 4.2)
+      multi_json (~> 1.3)
+      thread_safe (~> 0.1)
+      tzinfo (~> 0.3.37)
+    atomic (1.1.14)
+    aws-sdk (1.32.0)
+      json (~> 1.4)
+      nokogiri (>= 1.4.4)
+      uuidtools (~> 2.1)
+    chronic (0.10.2)
+    diff-lcs (1.2.5)
+    i18n (0.6.9)
+    json (1.8.1)
+    mini_portile (0.5.2)
+    minitest (4.7.5)
+    multi_json (1.8.4)
+    nokogiri (1.6.1)
+      mini_portile (~> 0.5.0)
+    rake (10.1.1)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.7)
+    rspec-expectations (2.14.4)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.4)
+    thor (0.18.1)
+    thread_safe (0.1.3)
+      atomic
+    tzinfo (0.3.38)
+    uuidtools (2.1.4)
+    whenever (0.9.0)
+      activesupport (>= 2.3.4)
+      chronic (>= 0.6.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.3)
+  rake
+  rspec
+  s3log!

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Faria Systems Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,67 @@
+# S3log
+
+[![Gem Version](https://badge.fury.io/rb/s3log.png)](http://rubygems.org/gems/s3log)
+[![Dependency Status](https://gemnasium.com/eduvo/s3log.png)](https://gemnasium.com/eduvo/s3log)
+
+Downloader for aws S3 logs and agglomerator.
+
+On S3 buckets you can activate logging, but the options are quite limited. You can point the logging to another bucket in the same region only and logging will generate a lot of small logging files, and will not delete them. It's not very convenient for debugging.
+
+S3log script is intended to transform this logging in a unified logfile wherever you need it, removing the logfiles from the logging bucket when they are downloaded.
+
+It's designed to run on debian/ubuntu servers but should work on any linux server, and it also work on mac osx.
+
+(work in progress, not ready for use yet).
+
+## Installation
+
+    gem install s3log
+    s3log init      # this will create a s3log/ dir at your current location
+
+    cd s3log/
+    vi config.yml   # customize according to your need
+    s3log buckets   # this will verify the list of buckets that your credential give access to
+    s3log list      # this will list the logfiles that will be candidate for downloading
+
+## Usage
+
+First you need to edit the config.yml file. The purpose is to have one config file per bucket you want to retrieve, so you can create as many config files as you need.
+
+* jobname: the unique identifier for that job
+* awspublic: the AWS access key
+* awsprivate: the AWS secret key id
+* bucket: the name of the bucket, with no s3:// prefix
+* prefix: the path and prefix the same way you specified it in the logging setup
+* logdir: the local dir where s3log logs are kept
+* loglevel: the level of logging. use warn for only error display, info to record info on operations, debug to have much more details
+* outputfile: the faile where the downloaded logs are going to be appended
+* schedule: the cron formatted frequency of downloading the logs from bucket ie. `0/5 * * * *`
+
+When your configuration is ready, veridy that it works with
+
+    s3log list
+    s3log -c another_config.yml schecule
+
+Then you can update the user crontab for each config file.
+
+    s3log schedule
+    s3log -c another_config.yml schedule
+
+The `schedule` command will update the user crontab, that you can verify with `crontab -l`.
+In all s3log commands if you don't specify the config file with `-c`, it will use `config.yml`.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## author
+
+* mose
+
+## License
+
+Copyright (c) 2014 Faria Systems Inc. distributed under MIT license

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/s3log

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require "bundler"
+require "s3log"
+
+S3log::Cli.start

data/lib/s3log/cli.rb

@@ -0,0 +1,52 @@
+require "thor"
+
+module S3log
+
+  class Cli < Thor
+    include Thor::Actions
+
+    def self.source_root
+      File.expand_path("../../../templates", __FILE__)
+    end
+
+    default_task :help
+    class_option :configfile,
+      aliases: "-c",
+      banner: "PATH",
+      default: File.expand_path("config.yml", Dir.pwd),
+      desc: "Path to the configuration file to use"
+
+    desc "init", "Creates a s3log dir with default config files."
+    def init(name="s3log")
+      directory "s3log", name
+    end
+
+    desc "list", "Lists log files that are waiting on the S3 bucket."
+    def list
+      @s3log = S3log::Runner.new(options[:configfile])
+      @s3log.items.each do |o|
+        puts o
+      end
+    end
+
+    desc "buckets", "List available buckets."
+    def buckets
+      @s3log = S3log::Runner.new(options[:configfile])
+      @s3log.buckets
+    end
+
+    desc "download", "Downloads and delete logs from the logging S3 bucket."
+    def download
+      @s3log = S3log::Runner.new(options[:configfile])
+      @s3log.download
+    end
+
+    desc "schedule", "Install cronjob for S3 logs download and agglomeration."
+    def schedule
+      @cron = S3log::Cron.new(options[:configfile])
+      @cron.update
+    end
+
+  end
+
+end

data/lib/s3log/cron.rb

@@ -0,0 +1,48 @@
+module S3log
+  class Cron
+
+    def initialize(configfile)
+      @configfile = configfile
+      @config = YAML::load_file(configfile)
+      @path = File.dirname(configfile)
+      @jobname = @config['jobname']
+      @schedule = @config['schedule']
+      @logdir = @config['logdir']
+      FileUtils.mkdir(@logdir) unless Dir.exists? @logdir
+      S3log::Log.set_logger(File.join(@logdir, 's3log.log'), @config['loglevel'])
+    end
+
+    def update
+      line = "#{@schedule} cd #{@path} && bash -l -c 'bundle exec s3log -c #{@configfile} download >> /dev/null 2>&1' # s3log_#{@jobname}\n"
+      tmp_cron_file = Tempfile.open('tmp_cron')
+      included = false
+      existing.each_line do |l|
+        if l =~ Regexp.new("# s3log_#{@jobname}")
+          tmp_cron_file << line
+          included = true
+        else
+          tmp_cron_file << l
+        end
+      end
+      tmp_cron_file << line unless included
+      tmp_cron_file.fsync
+      if system("crontab #{tmp_cron_file.path}")
+        S3log::Log.info "[update] crontab updated."
+      else
+        S3log::Log.warn "[fail] Couldn't write crontab."
+        tmp_cron_file.close!
+        exit(1)
+      end
+    end
+
+    def existing
+      existing = %x(crontab -l 2> /dev/null)
+      if $?.exitstatus.zero?
+        existing
+      else
+        ""
+      end
+    end
+
+  end
+end

data/lib/s3log/logger.rb

@@ -0,0 +1,55 @@
+require "logger"
+
+module S3log
+  module Log
+    extend self
+
+    def set_logger(logfile, level="info")
+      if logfile.is_a?(String) && !Dir.exists?(File.dirname(logfile))
+        FileUtils.mkdir_p(File.dirname(logfile))
+      end
+      @logger = ::Logger.new(logfile)
+      @logger.level = get_level_constant(level)
+      @logger.formatter = proc do |severity, datetime, progname, msg|
+        "[#{datetime.utc}] #{severity}: #{msg}\n"
+      end
+    end
+
+    def logger
+      @logger ||= ::Logger.new(STDOUT)
+    end
+
+    def debug(msg)
+      logger.debug(msg)
+    end
+
+    def info(msg)
+      logger.info(msg)
+    end
+
+    def warn(msg)
+      logger.warn(msg)
+    end
+
+    def error(msg)
+      logger.error(msg)
+    end
+
+    def fatal(msg)
+      logger.fatal(msg)
+    end
+
+    def get_level_constant(level)
+      if level
+        begin
+          ::Logger.const_get(level.to_s.upcase)
+        rescue NameError
+          return ::Logger::INFO
+        end
+      else
+        ::Logger::INFO
+      end
+    end
+
+  end
+end

data/lib/s3log/runner.rb

@@ -0,0 +1,49 @@
+require 'aws-sdk'
+
+module S3log
+
+  class Runner
+
+    def initialize(configfile)
+      @config = YAML::load_file(configfile)
+      @s3 = AWS::S3.new(
+        access_key_id: @config['awspublic'],
+        secret_access_key: @config['awsprivate']
+      )
+      @bucket = @s3.buckets[@config['bucket']]
+      @prefix = @config['prefix']
+      @logdir = @config['logdir']
+      FileUtils.mkdir(@logdir) unless Dir.exists? @logdir
+      S3log::Log.set_logger(File.join(@logdir, 's3log.log'), @config['loglevel'])
+    end
+
+    def items
+      @_items ||= @bucket.objects.with_prefix(@prefix).collect(&:key)
+    end
+
+    def buckets
+      @s3.buckets.each do |bucket|
+        puts bucket.name
+      end
+    end
+
+    def download
+      if items.size > 0
+        time = Time.now
+        S3log::Log.info "Downloading #{items.size} file."
+        File.open(@config['outputfile'], 'a+') do |f|
+          items.each do |i|
+            f.puts @bucket.objects[i].read
+            S3log::Log.debug "    #{i} added."
+            @bucket.objects[i].delete
+          end
+        end
+        S3log::Log.info "... done in #{Time.now - time}s."
+      else
+        S3log::Log.debug "No file to download."
+      end
+    end
+
+  end
+
+end

data/lib/s3log/version.rb

@@ -0,0 +1,3 @@
+module S3log
+  VERSION = "0.0.1"
+end

data/lib/s3log.rb

@@ -0,0 +1,8 @@
+require "s3log/version"
+require "s3log/cli"
+require "s3log/runner"
+require "s3log/logger"
+require "s3log/cron"
+
+module S3log
+end

data/s3log.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 's3log/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "s3log"
+  spec.version       = S3log::VERSION
+  spec.authors       = ["mose"]
+  spec.email         = ["mose@mose.com"]
+  spec.description   = %q{Downloader for aws S3 logs.}
+  spec.summary       = %q{Downloader for aws S3 logs, scheduler agglomerator.}
+  spec.homepage      = "https://github.com/eduvo/s3log"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^spec/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "thor"
+  spec.add_dependency "aws-sdk"
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "rake"
+end

data/spec/lib/s3log/cli_spec.rb

@@ -0,0 +1,27 @@
+# encoding: utf-8
+
+require 'spec_helper'
+require 's3log/cli'
+
+describe S3log::Cli do
+
+  before :each do
+    @cli = S3log::Cli.new([], { 'configfile' => 'config.yml' })
+    @testdir = File.join('spec','files','init')
+    @cli.shell.mute do
+      @cli.init(@testdir)
+    end
+    @oldpwd = Dir.pwd
+    Dir.chdir @testdir
+  end
+
+  after :each do
+    Dir.chdir @oldpwd
+    FileUtils.rm_rf @testdir if Dir.exists? @testdir
+  end
+
+  it "init creates a working directory" do
+    expect(File.file? 'config.yml').to be_true
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,10 @@
+$LOAD_PATH << File.expand_path('../../lib', __FILE__)
+require 'rubygems'
+require 'bundler'
+
+RSpec.configure do |config|
+  config.mock_with :rspec
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+  end
+end

data/templates/s3log/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gem "s3log"

data/templates/s3log/config.yml

@@ -0,0 +1,10 @@
+---
+jobname: job
+awspublic: xxx
+awsprivate: xxx
+bucket: bucketname # no s3:// prefix, just the name
+prefix: "somedir/prefix_" # the same as prefix stated in logging interface for the logged bucket
+logdir: logs
+loglevel: info
+outputfile: s3.log
+schedule: "0/5 * * * *"

metadata

@@ -0,0 +1,137 @@
+--- !ruby/object:Gem::Specification
+name: s3log
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- mose
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-01-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Downloader for aws S3 logs.
+email:
+- mose@mose.com
+executables:
+- s3log
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- CHANGELOG.md
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- bin/s3log
+- lib/s3log.rb
+- lib/s3log/cli.rb
+- lib/s3log/cron.rb
+- lib/s3log/logger.rb
+- lib/s3log/runner.rb
+- lib/s3log/version.rb
+- s3log.gemspec
+- spec/lib/s3log/cli_spec.rb
+- spec/spec_helper.rb
+- templates/s3log/Gemfile
+- templates/s3log/config.yml
+homepage: https://github.com/eduvo/s3log
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: Downloader for aws S3 logs, scheduler agglomerator.
+test_files:
+- spec/lib/s3log/cli_spec.rb
+- spec/spec_helper.rb