buildbox 0.6.beta2 → 0.6.beta3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d09a24715e1c3a79b74647a1b775b7166cdcf952
-  data.tar.gz: 124c2dad15dbe1e38273803411e2eb73095cee42
+  metadata.gz: 6d0de360c32f22ab29259e472e4ecba54dc326aa
+  data.tar.gz: 12631903e7780f9d7fea9ea16964fb09402e30de
 SHA512:
-  metadata.gz: 039183e2b691d6f18c4e009ec9468bed4225f3d90d15c200ea72773e86f53646a49c6ab8791ad12dd124f185f7f5b23481e005b1b978a4760b599f9cdf6d43b6
-  data.tar.gz: e87d1f3ce2c272bee640828402baf01e28eca316b4319bed05598f35de6c89cb7d5f926f1e038329782080ab515c42ed741a5983e3cef81cb7672466ac284daa
+  metadata.gz: 211edbd383cb6b699ceef9187c18d12374ada689dbf13210b87117efdccad1bf261a8f4b7e3dcfe3bb2ea8d83c9b936dfc75a4a19a548e996cbceb49cef05f85
+  data.tar.gz: 77e43c81f73e764bdcf432b815df5dde9d5acb92ac3c7edea20bb195cd90cb07d3f7900790ae5c03231d0456417cb8e320e06fc8a2b7ad3ddfe93b2ce3e24e38

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    buildbox (0.6.beta2)
+    buildbox (0.6.beta3)
       celluloid (~> 0.14)
       childprocess (~> 0.3)
       faraday (~> 0.8)

data/lib/buildbox.rb

@@ -16,7 +16,6 @@ module Buildbox
   autoload :Server,        "buildbox/server"
   autoload :UTF8,          "buildbox/utf8"
   autoload :Agent,         "buildbox/agent"
-  autoload :Uploader,      "buildbox/uploader"
   autoload :VERSION,       "buildbox/version"
 
   def self.config

data/lib/buildbox/agent.rb

@@ -42,13 +42,12 @@ module Buildbox
     end
 
     def upload_artifacts_from(build_directory, artifact_path)
-      files = Artifact.files_to_upload(build_directory, artifact_path)
+      artifacts = Artifact::Collector.collect_and_copy(build_directory, artifact_path)
 
-      files.each_pair do |relative_path, absolute_path|
-        Celluloid::Actor[:uploader_pool].async.upload(@api, @access_token, @current_build, relative_path, absolute_path)
-      end
+      Artifact::Uploader.new(@api, @access_token, @current_build, artifacts).async.prepare_and_upload
     rescue => e
       error "There was an error uploading artifacts for path: #{artifact_path} (#{e.class.name}: #{e.message})"
+
       e.backtrace[0..3].each { |line| error(line) }
     end
   end

data/lib/buildbox/api.rb

@@ -54,8 +54,8 @@ module Buildbox
       put("#{access_token}/builds/#{build.id}", options)
     end
 
-    def create_artifact(access_token, build, options)
-      post("#{access_token}/builds/#{build.id}/artifacts", options)
+    def create_artifacts(access_token, build, artifacts)
+      post("#{access_token}/builds/#{build.id}/artifacts", :artifacts => artifacts.map(&:as_json))
     end
 
     def update_artifact(access_token, build, artifact_id, options)

data/lib/buildbox/artifact.rb

@@ -1,62 +1,30 @@
-require 'fileutils'
-require 'tempfile'
+require 'securerandom'
 
 module Buildbox
   class Artifact
-    include Celluloid::Logger
+    autoload :Poster,    "buildbox/artifact/poster"
+    autoload :Collector, "buildbox/artifact/collector"
+    autoload :Uploader,  "buildbox/artifact/uploader"
 
-    def self.files_to_upload(build_directory, glob)
-      new(build_directory, glob).files_to_upload
-    end
+    attr_reader :id, :name, :path
+    attr_accessor :remote_id, :upload_instructions
 
-    def initialize(build_directory, glob)
-      @build_directory = build_directory
-      @glob            = glob
+    def self.create(name, path)
+      new(SecureRandom.uuid, name, path)
     end
 
-    def files_to_upload
-      tmpdir           = Dir.mktmpdir
-      path_to_absolute = {}
-
-      copy_files_to_upload(tmpdir).each do |file|
-        path_to_absolute[relativize_to_dir(file, tmpdir)] = file
-      end
-
-      path_to_absolute
+    def initialize(id, name, path)
+      @id   = id
+      @name = name
+      @path = path
     end
 
-    private
-
-    def copy_files_to_upload(dir)
-      expanded_directory = File.expand_path(@build_directory)
-      absolute_glob      = File.expand_path(@glob, expanded_directory)
-
-      target_files = Dir.glob(absolute_glob)
-
-      target_files.each do |file|
-        relative_path = relativize_to_dir(file, expanded_directory)
-        copy_to       = File.join(dir, relative_path)
-
-        if File.file?(file)
-          FileUtils.mkdir_p(File.dirname(copy_to))
-          FileUtils.cp(file, copy_to)
-        end
-      end
-
-      # Grab all the files we're going to upload.
-      Dir.glob(File.join(dir, "**", "*")).reject { |file| File.directory?(file) }
+    def basename
+      File.basename(@path)
     end
 
-    # /foo/build-directory/something.txt => /something.txt
-    # /var/random/something.txt => /var/random/something.txt
-    def relativize_to_dir(path, directory)
-      if path.to_s.index(directory.to_s) == 0
-        parts = path.to_s.split(directory.to_s)
-        parts.shift
-        parts.join(directory.to_s)
-      else
-        path
-      end
+    def as_json
+      { :id => @id, :name => @name, :path => @path, :file_size => File.size(@path) }
     end
   end
 end

data/lib/buildbox/artifact/collector.rb

@@ -0,0 +1,63 @@
+require 'fileutils'
+require 'tempfile'
+
+module Buildbox
+  class Artifact::Collector
+    include Celluloid::Logger
+
+    MAX_ARTIFACT_LIMIT = 500
+
+    class TooManyArtifactsError < RuntimeError; end
+
+    def self.collect_and_copy(build_directory, glob)
+      new(build_directory, glob).collect_and_copy
+    end
+
+    def initialize(build_directory, glob)
+      @build_directory = build_directory
+      @glob            = glob
+    end
+
+    def collect_and_copy
+      index     = 0
+      artifacts = []
+      tmpdir    = Dir.mktmpdir
+
+      globbed_files.each do |file|
+        raise TooManyArtifactsError if index > MAX_ARTIFACT_LIMIT
+
+        absolute_path = File.expand_path(file, @build_directory)
+        copy_to_path  = File.join(tmpdir, file)
+
+        if File.file?(absolute_path)
+          artifacts << Artifact.create(file, copy_to_path)
+
+          FileUtils.mkdir_p(File.dirname(copy_to_path))
+          FileUtils.cp(absolute_path, copy_to_path)
+        end
+
+        index += 1
+      end
+
+      artifacts
+    end
+
+    private
+
+    def glob_path
+      Pathname.new(@glob)
+    end
+
+    def build_directory_path
+      Pathname.new(@build_directory)
+    end
+
+    def globbed_files
+      if glob_path.relative?
+        Dir.chdir(build_directory_path.expand_path) { Dir.glob(glob_path) }
+      else
+        Dir.glob(glob_path)
+      end
+    end
+  end
+end

data/lib/buildbox/artifact/poster.rb

@@ -0,0 +1,51 @@
+require 'celluloid'
+require 'mime/types'
+
+module Buildbox
+  class Artifact::Poster
+    include Celluloid
+    include Celluloid::Logger
+
+    def post(api, access_token, build, artifact)
+      upload_action = artifact.upload_instructions['action']
+      form_data     = artifact.upload_instructions['data'].to_hash.dup
+
+      connection = Faraday.new(:url => upload_action['url']) do |faraday|
+        faraday.request :multipart
+
+        faraday.response :raise_error
+
+        faraday.options[:timeout] = 60
+        faraday.options[:open_timeout] = 60
+
+        faraday.adapter Faraday.default_adapter
+      end
+
+      mime_type = MIME::Types.type_for(artifact.path)[0].to_s
+
+      file_input_key = upload_action['file_input']
+      form_data[file_input_key] = Faraday::UploadIO.new(artifact.path, mime_type)
+
+      api.update_artifact(access_token, build, artifact.remote_id, :state => 'uploading')
+
+      upload_exception = nil
+      response         = nil
+
+      begin
+        response = connection.post(upload_action['path'], form_data)
+      rescue => e
+        upload_exception = e
+      end
+
+      if upload_exception
+        error "Error uploading #{artifact.basename} with a status of (#{upload_exception.class.name}: #{upload_exception.message})"
+        finished_state = 'error'
+      else
+        info "Finished uploading #{artifact.basename} with a status of #{response.status}"
+        finished_state = 'finished'
+      end
+
+      api.update_artifact(access_token, build, artifact.remote_id, :state => finished_state)
+    end
+  end
+end

data/lib/buildbox/artifact/uploader.rb

@@ -0,0 +1,32 @@
+require 'celluloid'
+require 'mime/types'
+
+module Buildbox
+  class Artifact::Uploader
+    include Celluloid
+    include Celluloid::Logger
+
+    def initialize(api, access_token, build, artifacts)
+      @api          = api
+      @access_token = access_token
+      @build        = build
+      @artifacts    = artifacts
+    end
+
+    def prepare_and_upload
+      info "Preparing #{@artifacts.count} artifacts for upload"
+
+      responses = @api.create_artifacts(@access_token, @build, @artifacts)
+      responses.each do |response|
+        artifact = @artifacts.find { |artifact| artifact.id == response['id'] }
+
+        artifact.remote_id           = response['artifact']['id']
+        artifact.upload_instructions = response['artifact']['uploader']
+      end
+
+      @artifacts.each do |artifact|
+        Celluloid::Actor[:artifact_poster_pool].async.post(@api, @access_token, @build, artifact)
+      end
+    end
+  end
+end

data/lib/buildbox/server.rb

@@ -12,7 +12,7 @@ module Buildbox
 
     def start
       Celluloid.logger = @logger
-      Celluloid::Actor[:uploader_pool] = Uploader.pool
+      Celluloid::Actor[:artifact_poster_pool] = Artifact::Poster.pool
 
       agent_access_tokens.each do |access_token|
         @supervisors << Buildbox::Agent.supervise(access_token)

data/lib/buildbox/version.rb

@@ -1,3 +1,3 @@
 module Buildbox
-  VERSION = "0.6.beta2"
+  VERSION = "0.6.beta3"
 end

data/spec/buildbox/buildbox/artifact/collector_spec.rb

@@ -0,0 +1,45 @@
+require 'spec_helper'
+
+describe Buildbox::Artifact::Collector do
+  let(:directory) { File.join(FIXTURES_PATH, "artifact-globber") }
+
+  def test_for_files(files, expected)
+    expected.each do |path|
+      expect(files.map(&:name)).to include(path)
+      expect(File.exist?(files.find{ |f| f.name == path }.path)).to be_true
+    end
+  end
+
+  describe "#collect_and_copy" do
+    it "handles specific files" do
+      files = Buildbox::Artifact::Collector.collect_and_copy(directory, "foo.txt")
+
+      expect(files.length).to eql(1)
+      test_for_files(files, %w(foo.txt))
+    end
+
+    it "handles globs" do
+      files = Buildbox::Artifact::Collector.collect_and_copy(directory, "bar/**/*.txt")
+
+      expect(files.length).to eql(4)
+      test_for_files(files, %w(bar/bang.txt bar/bang1.txt bar/bang2.txt bar/inside-bar/bang3.txt))
+    end
+
+    it "handles absolute globs" do
+      files = Buildbox::Artifact::Collector.collect_and_copy(directory, File.join(File.expand_path(directory), "**/*.txt"))
+      expected_files = %w(foo.txt bar/bang.txt bar/bang1.txt bar/bang2.txt bar/inside-bar/bang3.txt).map do |file|
+        File.join(directory, file)
+      end
+
+      expect(files.length).to eql(5)
+      test_for_files(files, expected_files)
+    end
+
+    it "handles specifying everything under a folder" do
+      files = Buildbox::Artifact::Collector.collect_and_copy(directory, "coverage/**/*")
+
+      expect(files.length).to eql(25)
+      test_for_files(files, %w(coverage/index.html coverage/assets/0.8.0/application.js))
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: buildbox
 version: !ruby/object:Gem::Version
-  version: 0.6.beta2
+  version: 0.6.beta3
 platform: ruby
 authors:
 - Keith Pitt
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-12 00:00:00.000000000 Z
+date: 2013-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday
@@ -185,6 +185,9 @@ files:
 - lib/buildbox/agent.rb
 - lib/buildbox/api.rb
 - lib/buildbox/artifact.rb
+- lib/buildbox/artifact/collector.rb
+- lib/buildbox/artifact/poster.rb
+- lib/buildbox/artifact/uploader.rb
 - lib/buildbox/build.rb
 - lib/buildbox/canceler.rb
 - lib/buildbox/cli.rb
@@ -194,13 +197,12 @@ files:
 - lib/buildbox/platform.rb
 - lib/buildbox/runner.rb
 - lib/buildbox/server.rb
-- lib/buildbox/uploader.rb
 - lib/buildbox/utf8.rb
 - lib/buildbox/version.rb
 - lib/certs/cacert.pem
 - spec/buildbox/buildbox/agent_spec.rb
 - spec/buildbox/buildbox/api_spec.rb
-- spec/buildbox/buildbox/artifact_spec.rb
+- spec/buildbox/buildbox/artifact/collector_spec.rb
 - spec/buildbox/buildbox/build_spec.rb
 - spec/buildbox/buildbox/cli_spec.rb
 - spec/buildbox/buildbox/command_spec.rb
@@ -297,7 +299,7 @@ summary: Ruby agent for buildbox
 test_files:
 - spec/buildbox/buildbox/agent_spec.rb
 - spec/buildbox/buildbox/api_spec.rb
-- spec/buildbox/buildbox/artifact_spec.rb
+- spec/buildbox/buildbox/artifact/collector_spec.rb
 - spec/buildbox/buildbox/build_spec.rb
 - spec/buildbox/buildbox/cli_spec.rb
 - spec/buildbox/buildbox/command_spec.rb

data/lib/buildbox/uploader.rb

@@ -1,58 +0,0 @@
-require 'celluloid'
-require 'mime/types'
-
-module Buildbox
-  class Uploader
-    include Celluloid
-    include Celluloid::Logger
-
-    def upload(api, access_token, current_build, relative_path, absolute_path)
-      info "Uploading #{absolute_path}"
-
-      artifact = api.create_artifact(access_token, current_build,
-                                     :path => relative_path,
-                                     :file_size => File.size(absolute_path))
-
-
-      upload_action = artifact[:uploader][:action]
-      form_data     = artifact[:uploader][:data].to_hash.dup
-
-      connection = Faraday.new(:url => upload_action[:url]) do |faraday|
-        faraday.request :multipart
-
-        faraday.response :raise_error
-
-        faraday.options[:timeout] = 60
-        faraday.options[:open_timeout] = 60
-
-        faraday.adapter Faraday.default_adapter
-      end
-
-      mime_type = MIME::Types.type_for(absolute_path)[0].to_s
-
-      file_input_key = upload_action[:file_input]
-      form_data[file_input_key] = Faraday::UploadIO.new(absolute_path, mime_type)
-
-      api.update_artifact(access_token, current_build, artifact[:id], :state => 'uploading')
-
-      upload_exception = nil
-      response         = nil
-
-      begin
-        response = connection.post(upload_action[:path], form_data)
-      rescue => e
-        upload_exception = e
-      end
-
-      if upload_exception
-        error "Error uploading #{File.basename(absolute_path)} with a status of (#{upload_exception.class.name}: #{upload_exception.message})"
-        finished_state = 'error'
-      else
-        info "Finished uploading #{File.basename(absolute_path)} with a status of #{response.status}"
-        finished_state = 'finished'
-      end
-
-      api.update_artifact(access_token, current_build, artifact[:id], :state => finished_state)
-    end
-  end
-end

data/spec/buildbox/buildbox/artifact_spec.rb

@@ -1,42 +0,0 @@
-require 'spec_helper'
-
-describe Buildbox::Artifact do
-  let(:directory) { File.join(FIXTURES_PATH, "artifact-globber") }
-
-  def test_for_files(files, expected)
-    expected.each do |path|
-      expect(files.keys).to include(path)
-      expect(File.exist?(files[path])).to be_true
-    end
-  end
-
-  describe "#files_to_upload" do
-    it "handles specific files" do
-      files = Buildbox::Artifact.files_to_upload(directory, "foo.txt")
-
-      expect(files.length).to eql(1)
-      test_for_files(files, %w(/foo.txt))
-    end
-
-    it "handles globs" do
-      files = Buildbox::Artifact.files_to_upload(directory, "bar/**/*.txt")
-
-      expect(files.length).to eql(4)
-      test_for_files(files, %w(/bar/bang.txt /bar/bang1.txt /bar/bang2.txt /bar/inside-bar/bang3.txt))
-    end
-
-    it "handles absolute globs" do
-      files = Buildbox::Artifact.files_to_upload(directory, File.join(File.expand_path(directory), "**/*.txt"))
-
-      expect(files.length).to eql(5)
-      test_for_files(files, %w(/foo.txt /bar/bang.txt /bar/bang1.txt /bar/bang2.txt /bar/inside-bar/bang3.txt))
-    end
-
-    it "handles specifying everything under a folder" do
-      files = Buildbox::Artifact.files_to_upload(directory, "coverage/**/*")
-
-      expect(files.length).to eql(25)
-      test_for_files(files, %w(/coverage/index.html /coverage/assets/0.8.0/application.js))
-    end
-  end
-end