travis-artifacts 0.0.1

data/.gitignore

@@ -0,0 +1,2 @@
+vendor/bundle
+/log/

data/.rspec

@@ -0,0 +1 @@
+--color

data/.travis.yml

@@ -0,0 +1,6 @@
+language: 'ruby'
+script: 'bundle exec rspec spec'
+rvm:
+  - 1.8.7
+  - 1.9.3
+  - jruby

data/Gemfile

@@ -0,0 +1,3 @@
+source :rubygems
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,52 @@
+PATH
+  remote: .
+  specs:
+    travis-artifacts (0.0.1)
+      faraday
+      faraday_middleware
+      fog
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    builder (3.1.4)
+    diff-lcs (1.1.3)
+    excon (0.16.8)
+    faraday (0.8.4)
+      multipart-post (~> 1.1)
+    faraday_middleware (0.9.0)
+      faraday (>= 0.7.4, < 0.9)
+    fog (1.7.0)
+      builder
+      excon (~> 0.14)
+      formatador (~> 0.2.0)
+      mime-types
+      multi_json (~> 1.0)
+      net-scp (~> 1.0.4)
+      net-ssh (>= 2.1.3)
+      nokogiri (~> 1.5.0)
+      ruby-hmac
+    formatador (0.2.4)
+    mime-types (1.19)
+    multi_json (1.3.7)
+    multipart-post (1.1.5)
+    net-scp (1.0.4)
+      net-ssh (>= 1.99.1)
+    net-ssh (2.6.1)
+    nokogiri (1.5.5)
+    rspec (2.12.0)
+      rspec-core (~> 2.12.0)
+      rspec-expectations (~> 2.12.0)
+      rspec-mocks (~> 2.12.0)
+    rspec-core (2.12.0)
+    rspec-expectations (2.12.0)
+      diff-lcs (~> 1.1.3)
+    rspec-mocks (2.12.0)
+    ruby-hmac (0.4.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  rspec
+  travis-artifacts!

data/bin/travis-artifacts

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+$: << File.expand_path('../../lib', __FILE__)
+
+require 'bundler/setup'
+require 'travis/artifacts'
+
+Travis::Artifacts::Cli.new(ARGV).start

data/lib/travis/artifacts.rb

@@ -0,0 +1,25 @@
+module Travis
+  autoload :Client, 'travis/client'
+
+  module Artifacts
+    autoload :Cli,          'travis/artifacts/cli'
+    autoload :Uploader,     'travis/artifacts/uploader'
+    autoload :Artifact,     'travis/artifacts/artifact'
+    autoload :Path,         'travis/artifacts/path'
+    autoload :Logger,       'travis/artifacts/logger'
+    autoload :Test,         'travis/artifacts/test'
+    autoload :ConfigParser, 'travis/artifacts/config_parser'
+    autoload :Prefix,       'travis/artifacts/prefix'
+
+    def self.aws_config
+      { :aws_access_key_id => ENV['ARTIFACTS_AWS_ACCESS_KEY_ID'],
+        :aws_secret_access_key => ENV['ARTIFACTS_AWS_SECRET_ACCESS_KEY'],
+        :region => ENV['ARTIFACTS_AWS_REGION'] || 'us-east-1'
+      }
+    end
+
+    def self.bucket_name
+      ENV['ARTIFACTS_S3_BUCKET']
+    end
+  end
+end

data/lib/travis/artifacts/artifact.rb

@@ -0,0 +1,17 @@
+require 'mime/types'
+
+module Travis::Artifacts
+  class Artifact < Struct.new(:source, :destination)
+    def content_type
+      if MIME::Types.type_for(source).any?
+        MIME::Types.type_for(source).first.content_type
+      else
+        'application/octet-stream'
+      end
+    end
+
+    def read
+      File.read(source)
+    end
+  end
+end

data/lib/travis/artifacts/cli.rb

@@ -0,0 +1,93 @@
+require 'optparse'
+
+module Travis::Artifacts
+  class Cli
+    attr_reader :options, :argv, :paths
+    attr_accessor :command, :client
+
+    VALID_COMMANDS = ['upload']
+
+    def initialize(argv = nil)
+      @argv    = argv || ARGV
+      @options = { :paths => [] }
+      @paths   = []
+      @client  = Travis::Client.new
+    end
+
+    # I would like to use option parser for now, to keep
+    # the code super simple and without too much external
+    # dependencies, if it grows too much, I may change it
+    def start
+      parse!
+      create_paths
+
+      execute_command
+    end
+
+    def root
+      options[:root] || Dir.pwd
+    end
+
+    def upload
+      Uploader.new(paths, options[:target_path]).upload
+    end
+
+    private
+
+    def execute_command
+      if VALID_COMMANDS.include? command
+        send(command)
+      else
+        STDERR.puts 'Could not find command'
+        exit 1
+      end
+    end
+
+    def fetch_paths
+      options[:paths]
+    end
+
+    def create_paths
+      fetch_paths.each do |path|
+        from, to = path.split(':')
+        paths << Path.new(from, to, root)
+      end
+    end
+
+    def parse!
+      self.command = argv[0]
+      parser.parse! argv
+    end
+
+    def parser
+      @opt_parser ||= begin
+        options = self.options
+
+        OptionParser.new do |opt|
+          opt.banner = 'Usage: travis-uploader COMMAND [OPTIONS]'
+          opt.separator  ''
+          opt.separator  'Commands'
+          opt.separator  '     upload: upload files to server'
+          opt.separator  ''
+          opt.separator  'Options'
+
+          opt.on('--path PATH','path(s) to upload to a server') do |path|
+            options[:paths] << path
+          end
+
+          opt.on('--target-path TARGET_PATH', 'path to upload files to') do |target_path|
+            options[:target_path] = target_path
+          end
+
+          opt.on('--root ROOT', 'root directory for relative paths') do |root|
+            options[:root] = root
+          end
+
+          opt.on('-h','--help','help') do
+            puts @opt_parser
+          end
+        end
+      end
+    end
+  end
+end

data/lib/travis/artifacts/logger.rb

@@ -0,0 +1,17 @@
+require 'logger'
+
+module Travis::Artifacts
+  module Logger
+    class << self
+      attr_accessor :output
+    end
+    self.output = STDOUT
+
+
+    attr_reader :logger
+
+    def logger
+      @logger ||= ::Logger.new(Travis::Artifacts::Logger.output)
+    end
+  end
+end

data/lib/travis/artifacts/path.rb

@@ -0,0 +1,15 @@
+module Travis::Artifacts
+  class Path < Struct.new(:from, :to, :root)
+    def fullpath
+      if from =~ /^\//
+        from
+      else
+        File.join(root, from)
+      end
+    end
+
+    def directory?
+      File.directory?(fullpath)
+    end
+  end
+end

data/lib/travis/artifacts/test.rb

@@ -0,0 +1,27 @@
+module Travis::Artifacts
+  class Test
+    def success?
+      ENV['TRAVIS_TEST_RESULT'].to_i == 0
+    end
+
+    def failure?
+      !success?
+    end
+
+    def build_number
+      ENV['TRAVIS_BUILD_NUMBER']
+    end
+
+    def build_id
+      ENV['TRAVIS_BUILD_ID']
+    end
+
+    def job_number
+      ENV['TRAVIS_JOB_NUMBER']
+    end
+
+    def job_id
+      ENV['TRAVIS_JOB_ID']
+    end
+  end
+end

data/lib/travis/artifacts/uploader.rb

@@ -0,0 +1,109 @@
+require 'find'
+require 'fog'
+
+module Travis::Artifacts
+  class Uploader
+    include Travis::Artifacts::Logger
+
+    attr_reader :paths, :target_path
+
+    def initialize(paths, target_path = nil)
+      @paths  = paths
+      @test   = Test.new
+      @target_path = target_path || "artifacts/#{@test.build_number}/#{@test.job_number}"
+    end
+
+    def upload
+      files.each do |file|
+        upload_file(file)
+      end
+    end
+
+    def files
+      files = []
+
+      paths.each do |artifact_path|
+        to   = artifact_path.to
+        from = artifact_path.from
+        root = artifact_path.root
+
+        if artifact_path.directory?
+          root = File.join(root, from)
+          root << '/' unless root =~ /\/$/
+        end
+
+        Find.find(artifact_path.fullpath) do |path|
+          next unless File.file? path
+
+          relative = path.sub(/#{root}\/?/, '')
+
+          destination = if to
+            artifact_path.directory? ? File.join(to, relative) : to
+          else
+            relative
+          end
+
+          files << Artifact.new(path, destination)
+        end
+      end
+
+      files
+    end
+
+
+    def upload_file(file)
+      retries = 0
+
+      begin
+        _upload(file)
+      rescue StandardError => e
+        if retries < 2
+          logger.info "Attempt to upload failed, retrying"
+          retries += 1
+          retry
+        else
+          if e.respond_to?(:request)
+            # we don't want to display sensitive data, make the error message simpler
+            request  = e.request
+            response = e.response
+            raise e
+            raise e.class.new("Expected(#{request[:expects].inspect}) <=> Actual(#{response.status})")
+          else
+            raise
+          end
+        end
+      end
+    end
+
+    def _upload(file)
+      destination = File.join(target_path, file.destination)
+
+      logger.info "Uploading file #{file.source} to #{destination}"
+
+      bucket.files.create({
+        :key => destination,
+        :public => true,
+        :body => file.read,
+        :content_type => file.content_type,
+        :metadata => { "Cache-Control" => 'public, max-age=315360000'}
+      })
+    end
+
+    private
+
+    def job_id
+      test.job_id
+    end
+
+    def bucket
+      @bucket ||= s3.directories.get(Travis::Artifacts.bucket_name)
+    end
+
+    def s3
+      @s3 ||= begin
+        config = { :provider => 'AWS' }.merge Travis::Artifacts.aws_config
+        Fog::Storage.new(config)
+      end
+    end
+  end
+end

data/lib/travis/client.rb

@@ -0,0 +1,47 @@
+require 'faraday'
+require 'faraday_middleware'
+
+module Travis
+  class Client
+    attr_reader :connection
+
+    def initialize
+      @connection = create_connection
+    end
+
+    # TODO: this needs to be fixed before going too far, returning body without
+    #       ability to inspect response or returning nil on any failure are not
+    #       the best ways to deal with API client
+    def fetch_config(job_id)
+      response = get("/jobs/#{job_id}")
+      if (200..299).include?(response.status)
+        response.body['job']['config']
+      else
+        # TODO: I should probably raise here
+        nil
+      end
+    end
+
+    private
+
+    http_methods = [:get, :post, :put, :patch, :delete, :head]
+    http_methods.each do |method|
+
+      define_method(method) do |*args, &block|
+        path, params, headers = *args
+        connection.send(method, path, params, headers, &block)
+      end
+    end
+
+    def create_connection
+      # TODO: support for pro
+      Faraday.new(:url => 'https://api.travis-ci.org') do |faraday|
+        faraday.response :json, :content_type => /\bjson$/
+
+        faraday.adapter :net_http
+      end.tap do |connection|
+        connection.headers['Accept'] = 'application/vnd.travis-ci.2+json'
+      end
+    end
+  end
+end

data/lib/travis/client_spec.rb

@@ -0,0 +1,4 @@
+require 'spec_helper'
+
+describe Travis::Client do
+end

data/script/console

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+$:.unshift(File.expand_path('../../lib', __FILE__))
+
+require 'bundler/setup'
+require 'irb'
+require 'irb/completion'
+require 'travis/artifacts'
+ARGV.clear
+IRB.start
+

data/spec/spec_helper.rb

@@ -0,0 +1,9 @@
+require 'travis/artifacts'
+require 'fileutils'
+
+FileUtils.mkdir_p "log"
+Travis::Artifacts::Logger.output = "log/test.log"
+
+RSpec.configure do |config|
+  config.expect_with :rspec, :stdlib
+end

data/spec/travis/artifacts/artifact_spec.rb

@@ -0,0 +1,6 @@
+require 'spec_helper'
+
+module Travis::Artifacts
+  describe Artifact do
+  end
+end

data/spec/travis/artifacts/cli_spec.rb

@@ -0,0 +1,65 @@
+require 'spec_helper'
+
+module Travis::Artifacts
+  describe Cli do
+    let(:cli) { described_class.new(argv) }
+
+
+    context 'with multiple paths' do
+      before { cli.stub(:upload) }
+      let(:argv) { ['upload', '--path', 'path/to/foo', '--path', 'path/to/bar:bar'] }
+
+      before { cli.start }
+
+      it 'parses them' do
+        cli.options[:paths].should == ['path/to/foo', 'path/to/bar:bar']
+      end
+
+      it 'adds paths to paths array' do
+        cli.paths.map { |p| [p.from, p.to] }.should == [['path/to/foo', nil], ['path/to/bar', 'bar']]
+      end
+    end
+
+    describe 'upload' do
+      let(:argv) { ['upload', '--path', 'foo', '--target-path', 'bar'] }
+      it 'calls Uploader with given paths and target_path' do
+        uploader = mock('uploader')
+        Uploader.should_receive(:new).with([Path.new('foo', nil, Dir.pwd)], 'bar').and_return(uploader)
+        uploader.should_receive(:upload)
+
+        cli.start
+      end
+    end
+
+    describe '#root' do
+      before { cli.stub(:upload) }
+      before { cli.start }
+
+      context 'with root passed as an argument' do
+        let(:argv) { ['upload', '--root', 'foo'] }
+        it 'returns passed value' do
+          cli.root.should == 'foo'
+        end
+      end
+
+      context 'without root passed as an argument' do
+        before { cli.stub(:upload) }
+        let(:argv) { ['upload'] }
+        it 'returns cwd' do
+          cli.root.should == Dir.pwd
+        end
+      end
+    end
+
+    context 'with a command' do
+      before { cli.stub(:upload) }
+      let(:argv) { ['upload'] }
+
+      before { cli.start }
+
+      it 'saves it' do
+        cli.command.should == 'upload'
+      end
+    end
+  end
+end

data/spec/travis/artifacts/uploader_spec.rb

@@ -0,0 +1,137 @@
+require 'spec_helper'
+
+module Travis::Artifacts
+  describe Uploader do
+    let(:uploader) { Uploader.new(paths, 'artifacts/1') }
+    let(:paths)    { [] }
+
+    context 'without given target_path' do
+      let(:uploader) { Uploader.new(paths, nil) }
+
+      it 'sets a default' do
+        test = mock('test', :job_number => "10.1", :build_number => "10")
+        Test.stub(:new => test)
+
+        uploader.target_path.should == 'artifacts/10/10.1'
+      end
+    end
+
+    describe '#upload_file' do
+      it 'retries 3 times before giving up' do
+        file = Artifact.new('source/file.png', 'destination/file.png')
+
+        uploader.should_receive(:_upload).exactly(3).times.and_raise(StandardError)
+
+        expect {
+          uploader.upload_file(file)
+        }.to raise_error(StandardError)
+      end
+
+      it 'simplifies excon error to not show request data' do
+        file = Artifact.new('source/file.png', 'destination/file.png')
+
+        request  = { :expects => 200 }
+        response = mock('response', :status => 500)
+        error_class = Class.new(Excon::Errors::HTTPStatusError)
+        my_error = error_class.new('message', request, response)
+        uploader.should_receive(:_upload).exactly(4).times.and_raise(my_error)
+
+        expect {
+          uploader.upload_file(file)
+        }.to raise_error { |e|
+          e.message.should == 'Expected(200) <=> Actual(500)'
+        }
+      end
+    end
+
+    describe '#upload' do
+      it 'uploads file to S3' do
+        files = [
+          Artifact.new('source/path.png', 'destination/path.png')
+        ]
+        files[0].stub(:read => 'contents')
+        uploader.stub(:files => files)
+
+        bucket       = mock('bucker')
+        bucket_files = mock('bucket_files')
+        uploader.stub(:bucket => bucket)
+        bucket.stub(:files => bucket_files)
+
+        bucket_files.should_receive(:create).with({
+          :key => 'artifacts/1/destination/path.png',
+          :public => true,
+          :body => 'contents',
+          :content_type => 'image/png',
+          :metadata => {'Cache-Control' => 'public, max-age=315360000'}
+
+        })
+
+        uploader.upload
+      end
+    end
+
+    describe '#files' do
+      let(:root)  { File.expand_path('../../../fixtures', __FILE__) }
+
+      context 'with nested files from root' do
+        let(:paths) { [Path.new('.', nil, root)] }
+
+        it 'resolves paths into files to upload' do
+          files = [
+            'files/foo/bar/baz.txt',
+            'files/logs/bar.log',
+            'files/logs/foo.log',
+            'files/output.txt'
+          ]
+          files.map! { |file| Artifact.new(File.join(root, '.', file), file) }
+
+          uploader.files.sort_by { |o| o.source }.should == files.sort_by { |o| o.source }
+        end
+      end
+
+      context 'with destination path and directory' do
+        let(:paths) { [Path.new('files/logs', 'logs', root)] }
+
+        it 'resolves paths into files to upload' do
+          files = [
+            'logs/bar.log',
+            'logs/foo.log'
+          ]
+          files.map! { |file| Artifact.new(File.join(root, 'files', file), "#{file}") }
+
+          uploader.files.sort_by { |o| o.source }.should == files.sort_by { |o| o.source }
+        end
+      end
+
+      context 'with individual files' do
+        let(:paths) {
+          [Path.new('files/logs/bar.log', nil, root),
+           Path.new('files/logs/foo.log', 'new/location/foo.log', root)]
+
+        }
+
+        it 'resolves paths into files to upload' do
+          file1 = 'files/logs/bar.log'
+          file2 = 'files/logs/foo.log'
+
+          files = [
+            Artifact.new(File.join(root, file1), 'files/logs/bar.log'),
+            Artifact.new(File.join(root, file2), 'new/location/foo.log')
+          ]
+
+          uploader.files.should == files
+        end
+      end
+
+      context 'with multilevel directory and to' do
+        let(:paths) {
+          [Path.new('files/foo', 'foo1', root)]
+        }
+
+        it 'resolves paths into files to upload' do
+          uploader.files.should == [Artifact.new(File.join(root, 'files/foo/bar/baz.txt'), 'foo1/bar/baz.txt')]
+        end
+      end
+    end
+  end
+end

data/travis-artifacts.gemspec

@@ -0,0 +1,22 @@
+Gem::Specification.new do |s|
+  s.name        = 'travis-artifacts'
+  s.version     = '0.0.1'
+
+  s.description = 'Travis build artifacts tools'
+  s.summary     = s.description
+
+  s.homepage    = 'https://github.com/travis-ci/travis-artifacts'
+  s.authors     = ['admin@travis-ci.org']
+  s.email       = 'admin@travis-ci.org'
+
+  s.add_dependency 'fog'
+  s.add_dependency 'faraday'
+  s.add_dependency 'faraday_middleware'
+
+  s.add_development_dependency 'rspec'
+
+  s.files         = `git ls-files`.split($/)
+  s.executables   = s.files.grep(/^bin/).map{|f| File.basename(f) }
+  s.test_files    = s.files.grep(/^spec/)
+  s.require_paths = ['lib']
+end

metadata

@@ -0,0 +1,142 @@
+--- !ruby/object:Gem::Specification
+name: travis-artifacts
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- admin@travis-ci.org
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-11-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fog
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: faraday_middleware
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Travis build artifacts tools
+email: admin@travis-ci.org
+executables:
+- travis-artifacts
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .travis.yml
+- Gemfile
+- Gemfile.lock
+- bin/travis-artifacts
+- lib/travis/artifacts.rb
+- lib/travis/artifacts/artifact.rb
+- lib/travis/artifacts/cli.rb
+- lib/travis/artifacts/logger.rb
+- lib/travis/artifacts/path.rb
+- lib/travis/artifacts/test.rb
+- lib/travis/artifacts/uploader.rb
+- lib/travis/client.rb
+- lib/travis/client_spec.rb
+- script/console
+- spec/fixtures/files/foo/bar/baz.txt
+- spec/fixtures/files/logs/bar.log
+- spec/fixtures/files/logs/foo.log
+- spec/fixtures/files/output.txt
+- spec/spec_helper.rb
+- spec/travis/artifacts/artifact_spec.rb
+- spec/travis/artifacts/cli_spec.rb
+- spec/travis/artifacts/uploader_spec.rb
+- travis-artifacts.gemspec
+homepage: https://github.com/travis-ci/travis-artifacts
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Travis build artifacts tools
+test_files:
+- spec/fixtures/files/foo/bar/baz.txt
+- spec/fixtures/files/logs/bar.log
+- spec/fixtures/files/logs/foo.log
+- spec/fixtures/files/output.txt
+- spec/spec_helper.rb
+- spec/travis/artifacts/artifact_spec.rb
+- spec/travis/artifacts/cli_spec.rb
+- spec/travis/artifacts/uploader_spec.rb