drunker 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1e4f32c3b7e312a8fed15ab1d6181193fb13744b
-  data.tar.gz: b5535975e7b881f411db29139d5ba72afb39d1b3
+  metadata.gz: 52bccd6c28771917af0c742d9bb2f1d590338bb5
+  data.tar.gz: d8a9836770d2ce3a68628e747f3ae839363cd044
 SHA512:
-  metadata.gz: 0055dbb90fd76fe7c75b38aa11f77a1ff96d2db1ce17652f9376e7187ae469f04ab056f5a75aef296a33bf1993cbe3be1099a68f22101609976d7e1336149350
-  data.tar.gz: d5d3607e90d875220781900b1bac374cbbbf789034b570715d35a5de699820d1747f0177ffacc83717a65502390a29f4d6c90257da6e35649737a48ac3c50135
+  metadata.gz: 9bcfd3438e029f4a985147d7c131f819f3e9215c511361bb38bc6b2f12c6724a610d33d6f60ac7b1a1f7537c244f809a966049b259695fcc41c1deeaf828b7c3
+  data.tar.gz: 81329e16744b8025c1a49d4d613c53ca3ccd0f757437b5085d7181eab94e3e4ab3180ea57766279547fb340147fe54982fb6ee328dfa6e5ae7ca96bae388917a

data/.rubocop.yml

@@ -0,0 +1,142 @@
+# To use the MeowCop gem.
+inherit_gem:
+  meowcop:
+    - config/rubocop.yml
+
+# Modify the version if you don't use MRI 2.1.
+AllCops:
+  TargetRubyVersion: 2.4
+
+Rails:
+  # If you use RuboCop with Ruby on Rails, turn on this option.
+  Enabled: false
+
+# You can customize rubocop settings.
+Lint/ShadowingOuterLocalVariable:
+  Enabled: false
+
+Metrics/LineLength:
+  Exclude:
+    - "spec/**"
+
+# EnforcedStyle: with_first_parameter => 0 offense
+# EnforcedStyle: with_fixed_indentation => 15 offenses
+Style/AlignParameters:
+  EnforcedStyle: with_first_parameter
+  Enabled: true
+
+# EnforcedStyle: nested => 0 offense
+# EnforcedStyle: compact => 14 offenses
+Style/ClassAndModuleChildren:
+  EnforcedStyle: nested
+  Enabled: true
+
+# EnforcedStyle: leading => 0 offense
+# EnforcedStyle: trailing => 28 offenses
+Style/DotPosition:
+  EnforcedStyle: leading
+  Enabled: true
+
+# EnforcedStyle: empty_lines => 617 offenses
+# EnforcedStyle: no_empty_lines => 1 offense
+Style/EmptyLinesAroundBlockBody:
+  EnforcedStyle: no_empty_lines
+  Enabled: true
+
+# EnforcedStyle: empty_lines => 34 offenses
+# EnforcedStyle: empty_lines_except_namespace => 26 offenses
+# EnforcedStyle: empty_lines_special => 19 offenses
+# EnforcedStyle: no_empty_lines => 0 offense
+Style/EmptyLinesAroundClassBody:
+  EnforcedStyle: no_empty_lines
+  Enabled: true
+
+# EnforcedStyle: when_needed => 30 offenses
+# EnforcedStyle: always => 30 offenses
+# EnforcedStyle: never => 0 offense
+Style/FrozenStringLiteralComment:
+  EnforcedStyle: never
+  Enabled: true
+
+# EnforcedStyle: ruby19 => 40 offenses
+# EnforcedStyle: hash_rockets => 451 offenses
+# EnforcedStyle: no_mixed_keys => 0 offense
+# EnforcedStyle: ruby19_no_mixed_keys => 40 offenses
+Style/HashSyntax:
+  EnforcedStyle: no_mixed_keys
+  Enabled: true
+
+# EnforcedStyle: require_parentheses => 0 offense
+# EnforcedStyle: require_no_parentheses => 28 offenses
+# EnforcedStyle: require_no_parentheses_except_multiline => 27 offenses
+Style/MethodDefParentheses:
+  EnforcedStyle: require_parentheses
+  Enabled: true
+
+# EnforcedStyle: snake_case => 0 offense
+# EnforcedStyle: camelCase => 23 offenses
+Style/MethodName:
+  EnforcedStyle: snake_case
+  Enabled: true
+
+# EnforcedStyle: symmetrical => 0 offense
+# EnforcedStyle: new_line => 23 offenses
+# EnforcedStyle: same_line => 12 offenses
+Style/MultilineMethodCallBraceLayout:
+  EnforcedStyle: symmetrical
+  Enabled: true
+
+# EnforcedStyle: aligned => 28 offenses
+# EnforcedStyle: indented => 28 offenses
+# EnforcedStyle: indented_relative_to_receiver => 6 offenses
+Style/MultilineMethodCallIndentation:
+  EnforcedStyle: indented_relative_to_receiver
+  Enabled: true
+
+# EnforcedStyle: compact => 0 offense
+# EnforcedStyle: exploded => 10 offenses
+Style/RaiseArgs:
+  EnforcedStyle: compact
+  Enabled: true
+
+# EnforcedStyleInsidePipes: space => 33 offenses
+# EnforcedStyleInsidePipes: no_space => 0 offense
+Style/SpaceAroundBlockParameters:
+  EnforcedStyleInsidePipes: no_space
+  Enabled: true
+
+# EnforcedStyle: space => 1 offense
+# EnforcedStyle: no_space => 152 offenses
+Style/SpaceBeforeBlockBraces:
+  EnforcedStyle: space
+  Enabled: true
+
+# EnforcedStyle: space => 134 offenses
+# EnforcedStyle: no_space => 0 offense
+Style/SpaceInsideStringInterpolation:
+  EnforcedStyle: no_space
+  Enabled: true
+
+# EnforcedStyle: single_quotes => 687 offenses
+# EnforcedStyle: double_quotes => 3 offenses
+Style/StringLiterals:
+  EnforcedStyle: double_quotes
+  Enabled: true
+
+# EnforcedStyle: final_newline => 1 offense
+# EnforcedStyle: final_blank_line => 30 offenses
+Style/TrailingBlankLines:
+  EnforcedStyle: final_newline
+  Enabled: true
+
+# EnforcedStyle: snake_case => 0 offense
+# EnforcedStyle: camelCase => 46 offenses
+Style/VariableName:
+  EnforcedStyle: snake_case
+  Enabled: true
+
+# EnforcedStyle: percent => 0 offense
+# EnforcedStyle: brackets => 47 offenses
+Style/WordArray:
+  EnforcedStyle: percent
+  Enabled: true

data/Gemfile

@@ -1,4 +1,4 @@
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
 # Specify your gem's dependencies in drunker.gemspec
 gemspec

data/README.md

@@ -1,4 +1,7 @@
 # Drunker
+[![Build Status](https://travis-ci.org/wata727/drunker.svg?branch=master)](https://travis-ci.org/wata727/drunker)
+[![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat)](LICENSE.txt)
+[![Gem Version](https://badge.fury.io/rb/drunker.svg)](https://badge.fury.io/rb/drunker)
 
 Distributed CLI runner on [AWS CodeBuild](https://aws.amazon.com/codebuild/). This is a wrapper for handling CodeBuild container more easily.
 
@@ -93,6 +96,17 @@ EXIT_STATUS: 0
 
 What is surprising is that do not require your local machine specs at all even if you increase the number of parallels. You can increase the number of parallels within the bounds of common sense.
 
+### Limitation
+The number of parallelism of CodeBuild is determined by default. If the limit is reached, Drunker queues remaining builders.
+
+```
+INFO: Maximum number of concurrent running builds has been reached. it will retry later...
+INFO: Waiting builder: 1/25, queues: 4
+...
+```
+
+For details about limitation, please see [here](http://docs.aws.amazon.com/codebuild/latest/userguide/limits.html).
+
 ### EC2 Container Registry
 
 Do you want to use private images? You can also specify the image of [EC2 Container Registry](https://aws.amazon.com/ecr/). Following example:
@@ -126,7 +140,6 @@ Options:
   [--file-pattern=FILE_PATTERN]  # FILES target file pattern, can use glob to specify, but files beginning with a dot are ignored.
                                  # Default: **/*
   [--aggregator=AGGREGATOR]      # Aggregator name. If you want to use custom aggregator, please install that beforehand.
-                                 # Default: pretty
   [--loglevel=LOGLEVEL]          # Output log level
                                  # Default: info
                                  # Possible values: debug, info, warn, error, fatal
@@ -254,16 +267,18 @@ buildspec:
 
 ## Customize Output
 
-Do you want to customize the output format? You can customize output format, exit code by creating Gem called aggregator. For example, the default aggregator is implemented as a separate [Gem](https://github.com/wata727/drunker-aggregator-pretty).
-
-The specifications that the aggregator must satisfy are the following three.
+Do you want to customize the output format? You can customize output format, exit code by creating Gem called aggregator. The specifications that the aggregator must satisfy are the following five.
 
-- Inherit `Drunker::Aggregator::Base`
+- The name of Gem must be `drunker-aggregator-#{name}`.
+- Create `lib/drunker-aggregator-#{name}.rb`, and require `Drunker::Aggregator::#{name}` class
+- `Drunker::Aggregator::#{name}` must inherit `Drunker::Aggregator::Base`
 - Implement `run` and `exit_status` methods to receive array of `Drunker::Artifact::Layer`
 - Return a number in `exit_status` method
 
 `Drunker::Artifact::Layer` has outputs and build ID for each build. Please see the [implementation](https://github.com/wata727/drunker/blob/master/lib/drunker/artifact/layer.rb) for details.
 
+For example, [this aggregator](https://github.com/wata727/drunker-aggregator-phpmd) can aggregate for PHPMD XML format report.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/drunker.gemspec

@@ -1,7 +1,7 @@
 # coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
+lib = File.expand_path("../lib", __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'drunker/version'
+require "drunker/version"
 
 Gem::Specification.new do |spec|
   spec.name          = "drunker"
@@ -29,5 +29,4 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "thor", "~> 0.19"
   spec.add_runtime_dependency "rubyzip", "~> 1.2"
   spec.add_runtime_dependency "aws-sdk", "~> 2"
-  spec.add_runtime_dependency "drunker-aggregator-pretty", "~> 0.1"
 end

data/lib/drunker.rb

@@ -18,3 +18,4 @@ require "drunker/executor/builder"
 require "drunker/config"
 require "drunker/aggregator"
 require "drunker/aggregator/base"
+require "drunker/aggregator/pretty"

data/lib/drunker/aggregator.rb

@@ -1,9 +1,13 @@
 module Drunker
   class Aggregator
     def self.create(config)
-      require config.aggregator.name
-      klass = Object.const_get(config.aggregator.name.split("-").map(&:capitalize).join("::"))
-      klass.new
+      if config.aggregator
+        require config.aggregator.name
+        klass = Object.const_get(config.aggregator.name.split("-").map(&:capitalize).join("::"))
+        klass.new
+      else
+        Pretty.new
+      end
     end
   end
 end

data/lib/drunker/aggregator/base.rb

@@ -10,4 +10,4 @@ module Drunker
       end
     end
   end
-end
+end

data/lib/drunker/aggregator/pretty.rb

@@ -0,0 +1,27 @@
+module Drunker
+  class Aggregator
+    class Pretty < Base
+      def run(layers)
+        layers.each do |layer|
+          puts
+          puts "-------------------------------------------------------------------------------------------"
+          puts "BUILD_ID: #{layer.build_id}"
+          if layer.invalid?
+            puts "RESULT: FAILED"
+          else
+            puts "RESULT: SUCCESS"
+            puts "STDOUT: #{layer.stdout}"
+            puts "STDERR: #{layer.stderr}"
+            puts "EXIT_STATUS: #{layer.exit_status}"
+          end
+          puts "-------------------------------------------------------------------------------------------"
+          puts
+        end
+      end
+
+      def exit_status(layers)
+        layers.map { |layer| layer.invalid? ? 1 : layer.exit_status }.max
+      end
+    end
+  end
+end

data/lib/drunker/artifact.rb

@@ -6,7 +6,7 @@ module Drunker
     attr_reader :exit_status
 
     def initialize(config:, logger:)
-      timestamp = Time.now.to_i.to_s
+      timestamp = Time.now.to_i
       s3 = Aws::S3::Resource.new(client: Aws::S3::Client.new(config.aws_client_options))
 
       @bucket = s3.create_bucket(bucket: "drunker-artifact-store-#{timestamp}")

data/lib/drunker/cli.rb

@@ -8,7 +8,7 @@ module Drunker
     method_option :env, :type => :hash, :default => {}, :desc => "Environment variables in containers"
     method_option :buildspec, :type => :string, :desc => "Location of custom buildspec"
     method_option :file_pattern, :type => :string, :default => "**/*", :desc => "FILES target file pattern, can use glob to specify, but files beginning with a dot are ignored."
-    method_option :aggregator, :type => :string, :default => "pretty", :desc => "Aggregator name. If you want to use custom aggregator, please install that beforehand."
+    method_option :aggregator, :type => :string, :desc => "Aggregator name. If you want to use custom aggregator, please install that beforehand."
     method_option :loglevel, :type => :string, :default => "info", :enum => %w(debug info warn error fatal), :desc => "Output log level"
     method_option :debug, :type => :boolean, :default => false, :desc => "Enable debug mode. This mode does not delete the AWS resources created by Drunker"
     method_option :access_key, :type => :string, :desc => "AWS access key token used by Drunker"

data/lib/drunker/config.rb

@@ -95,7 +95,8 @@ module Drunker
     end
 
     def aggregator_gem!(name)
-      gem = Gem::Specification.select { |gem| gem.name == "drunker-aggregator-#{name}" }.max { |a, b| a.version <=> b.version }
+      return unless name
+      gem = Gem::Specification.all.select { |gem| gem.name == "drunker-aggregator-#{name}" }.max_by(&:version)
       raise InvalidConfigException.new("Invalid aggregator. `drunker-aggregator-#{name}` is already installed?") unless gem
       gem
     end
@@ -135,7 +136,7 @@ module Drunker
                   "Invalid timeout. It should be number (Not string). got: #{yaml["timeout"]}"
                 when yaml["buildspec"] && !(yaml["buildspec"].is_a?(String) || yaml["buildspec"].is_a?(Hash))
                   "Invalid buildspec. It should be string or hash. got: #{yaml["buildspec"]}"
-                when yaml["environment_variables"] && !yaml["environment_variables"]&.values.all? { |v| v.is_a?(String) || v.is_a?(Numeric) }
+                when yaml["environment_variables"] && !yaml["environment_variables"]&.values&.all? { |v| v.is_a?(String) || v.is_a?(Numeric) }
                   "Invalid environment variables. It should be flatten hash. got: #{yaml["environment_variables"]}"
                 when yaml["file_pattern"] && !yaml["file_pattern"].is_a?(String)
                   "Invalid file pattern. It should be string. got: #{yaml["file_pattern"]}"

data/lib/drunker/executor.rb

@@ -3,32 +3,28 @@ module Drunker
     RETRY_LIMIT = 10
 
     def initialize(source:, config:, logger:)
-      @project_name = "drunker-executor-#{Time.now.to_i.to_s}"
+      @project_name = "drunker-executor-#{Time.now.to_i}"
       @source = source
       logger.info("Creating artifact...")
       @artifact = Drunker::Artifact.new(config: config, logger: logger)
       @config = config
       @client = Aws::CodeBuild::Client.new(config.aws_client_options)
-      @builders = []
       @logger = logger
     end
 
     def run
       setup_project do
-        @builders = parallel_build
+        builders = create_builders
 
         loop do
-          builders.select(&:access_denied?).each do |builder|
-            failed_id = builder.build_id
-            if builder.retriable?
-              build_id = builder.retry
-              artifact.replace_build(before: failed_id ,after: build_id)
-            end
-          end
+          start_builders builders
+          restart_failed_builders builders
 
-          running, finished = builders.partition(&:running?)
-          finished.select(&:failed?).each do |failed|
-            failed.errors.each do |error|
+          ran, waiting = builders.partition(&:ran?)
+          running, finished = ran.partition(&:running?)
+
+          finished.select(&:failed?).each do |builder|
+            builder.errors.each do |error|
               logger.warn("Build failed: #{failed.build_id}")
               logger.warn("\tphase_type: #{error[:phase_type]}")
               logger.warn("\tphase_status: #{error[:phase_status]}")
@@ -37,12 +33,14 @@ module Drunker
             end
           end
 
-          break if running.count.zero?
-          logger.info("Waiting builder: #{finished.count}/#{builders.count}")
+          if waiting.count.zero? && running.count.zero?
+            logger.info("Build is completed!")
+            break
+          end
+          logger.info("Waiting builders: #{finished.count}/#{builders.count}, queues: #{waiting.count}")
           sleep 5
           builders.each(&:refresh)
         end
-        logger.info("Build is completed!")
         artifact.layers # load artifact layers from S3
       end
 
@@ -56,7 +54,6 @@ module Drunker
     attr_reader :artifact
     attr_reader :config
     attr_reader :client
-    attr_reader :builders
     attr_reader :logger
 
     def setup_project
@@ -104,19 +101,33 @@ module Drunker
       end
     end
 
-    def parallel_build
-      builders = []
-
+    def create_builders
       files_list = source.target_files.each_slice(source.target_files.count.quo(config.concurrency).ceil).to_a
       logger.info("Start parallel build: { files: #{source.target_files.count}, concurrency: #{config.concurrency}, real_concurrency: #{files_list.count} }")
-      files_list.to_a.each do |files|
+      files_list.to_a.each_with_object([]) do |files, builders|
         builder = Builder.new(project_name: project_name, targets: files, artifact: artifact, config: config, logger: logger)
+        builders << builder
+      end
+    end
+
+    def start_builders(builders)
+      builders.reject(&:ran?).each do |builder|
         build_id = builder.run
         artifact.set_build(build_id)
-        builders << builder
       end
+    rescue Aws::CodeBuild::Errors::AccountLimitExceededException => exn
+      logger.info("Maximum number of concurrent running builds has been reached. it will retry later...")
+      logger.debug("Exception: #{exn.inspect}")
+    end
 
-      builders
+    def restart_failed_builders(builders)
+      builders.select(&:access_denied?).each do |builder|
+        failed_id = builder.build_id
+        if builder.retriable?
+          build_id = builder.retry
+          artifact.replace_build(before: failed_id ,after: build_id)
+        end
+      end
     end
   end
 end

data/lib/drunker/executor/builder.rb

@@ -51,16 +51,20 @@ module Drunker
         end
       end
 
+      def ran?
+        !!build_id
+      end
+
       def running?
-        status == IN_PROGRESS
+        ran? && status == IN_PROGRESS
       end
 
       def failed?
-        status == FAILED
+        ran? && status == FAILED
       end
 
       def success?
-        status == SUCCEEDED
+        ran? && status == SUCCEEDED
       end
 
       def refresh

data/lib/drunker/executor/iam.rb

@@ -4,7 +4,7 @@ module Drunker
       attr_reader :role
 
       def initialize(source:, artifact:, config:, logger:)
-        timestamp = Time.now.to_i.to_s
+        timestamp = Time.now.to_i
         client = Aws::IAM::Client.new(config.aws_client_options)
         iam = Aws::IAM::Resource.new(client: client)
 

data/lib/drunker/source.rb

@@ -3,7 +3,7 @@ module Drunker
     attr_reader :target_files
 
     def initialize(target_dir, config:, logger:)
-      timestamp = Time.now.to_i.to_s
+      timestamp = Time.now.to_i
       s3 = Aws::S3::Resource.new(client: Aws::S3::Client.new(config.aws_client_options))
 
       @bucket = s3.create_bucket(bucket: "drunker-source-store-#{timestamp}")
@@ -45,16 +45,16 @@ module Drunker
     attr_reader :logger
 
     def archive(target_dir)
-      archive_path = Pathname.new("#{target_dir.to_s}/#{name}")
+      archive_path = Pathname.new("#{target_dir}/#{name}")
 
       Zip::File.open(archive_path.to_s, Zip::File::CREATE) do |zip|
-        Pathname.glob(target_dir.to_s + "/**/*", File::Constants::FNM_DOTMATCH).select(&:file?).each do |real_path|
+        Pathname.glob(target_dir + "**/*", File::Constants::FNM_DOTMATCH).select(&:file?).each do |real_path|
           archive_file = real_path.relative_path_from(target_dir)
           zip.add(archive_file, real_path.to_s)
-          logger.debug("Archived: #{archive_file.to_s}")
+          logger.debug("Archived: #{archive_file}")
         end
       end
-      logger.debug("Archived source: #{archive_path.to_s}")
+      logger.debug("Archived source: #{archive_path}")
       yield archive_path
       archive_path.unlink
       logger.debug("Deleted archive")
@@ -62,7 +62,7 @@ module Drunker
 
     def set_target_files(target_dir)
       logger.debug("Use file pattern: #{config.file_pattern}")
-      Pathname.glob(target_dir.to_s + "/" + config.file_pattern).select(&:file?).each do |real_path|
+      Pathname.glob(target_dir + config.file_pattern).select(&:file?).each do |real_path|
         file = real_path.relative_path_from(target_dir).to_s
         @target_files << file
         logger.debug("Set target: #{file}")

data/lib/drunker/version.rb

@@ -1,3 +1,3 @@
 module Drunker
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: drunker
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Kazuma Watanabe
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-05-07 00:00:00.000000000 Z
+date: 2017-05-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -108,20 +108,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2'
-- !ruby/object:Gem::Dependency
-  name: drunker-aggregator-pretty
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.1'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.1'
 description: Distributed CLI runner on AWS CodeBuild
 email:
 - watassbass@gmail.com
@@ -132,6 +118,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
+- ".rubocop.yml"
 - ".travis.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -145,6 +132,7 @@ files:
 - lib/drunker.rb
 - lib/drunker/aggregator.rb
 - lib/drunker/aggregator/base.rb
+- lib/drunker/aggregator/pretty.rb
 - lib/drunker/artifact.rb
 - lib/drunker/artifact/layer.rb
 - lib/drunker/cli.rb