dmatrix 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 5b884e0670f6ecedebb47a88ecb56dd6f790ae672580493bb6f3a853cda71938
+  data.tar.gz: ae3960ae3408be5ddaf75bf670be80e4eb3f1daea63c2f7512f2baf249335b4e
+SHA512:
+  metadata.gz: 3b8b586ff6af822e4a320838c4be798aa3ff5932571ef6c26b56e6f6e2b1e264e020c7bdb01ad95aec52b8af9f3c15aa055baaaf5d2c752fc5973a60cae24b0f
+  data.tar.gz: f58d296ea65ffad4163683975b3a82b3021f046f5e1a9999b40a287f624dddab229d111f74147f3a07ebdae8d0476e864a7ce3928ebb50955160dcc000472bcb

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+spec/examples.txt

data/.matrix.yaml

@@ -0,0 +1,5 @@
+matrix:
+  build_arg:
+    FROM_IMAGE:
+      - ruby:2.5-alpine
+      - ruby:2.6-alpine

data/.rspec

@@ -0,0 +1,2 @@
+--require spec_helper
+--format documentation

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+# Changelog
+
+## v0.1.0
+
+- Initial release

data/Dockerfile

@@ -0,0 +1,2 @@
+ARG FROM_IMAGE=ruby:2.6-alpine
+FROM $FROM_IMAGE

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in dmatrix.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,47 @@
+PATH
+  remote: .
+  specs:
+    dmatrix (0.1.0)
+      parallel (~> 1.13)
+      slop (~> 4.5)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    coderay (1.1.2)
+    diff-lcs (1.3)
+    jet_black (0.5.1)
+    method_source (0.9.2)
+    parallel (1.17.0)
+    pry (0.12.2)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    rake (12.3.2)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.3)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+    slop (4.6.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler
+  dmatrix!
+  jet_black (~> 0.5)
+  pry
+  rake
+  rspec (~> 3.8)
+
+BUNDLED WITH
+   2.0.1

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2019 Oliver Peate
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,93 @@
+# Dmatrix
+
+Docker matrix. Parallel execution of each possible combination defined in your
+matrix file.
+
+The matrix file can contain `build_arg` and `env` values which will be passed
+to Docker during the image build and execution of your command.
+
+Inspired by the [Travis Build Matrix](https://docs.travis-ci.com/user/build-matrix/).
+
+## Example
+
+### `.matrix.yaml`
+
+```
+matrix:
+  build_arg:
+    FROM_IMAGE:
+      - ruby:2.4-alpine
+      - ruby:2.5-alpine
+      - ruby:2.6-alpine
+    BUNDLE_GEMFILE:
+      - gemfiles/factory_bot_4_8.gemfile
+      - gemfiles/factory_bot_5.gemfile
+```
+
+This would produce six combinations. In this example the `gemfiles` are created
+with the [Appraisal gem](https://github.com/thoughtbot/appraisal).
+
+### `Dockerfile`
+
+```
+ARG FROM_IMAGE=ruby:2.6-alpine
+FROM $FROM_IMAGE
+
+RUN apk update && \
+    apk add git && \
+    mkdir -p /app/lib/my_gem
+
+WORKDIR /app
+
+COPY Gemfile* my_gem.gemspec Appraisals /app/
+COPY gemfiles/*.gemfile /app/gemfiles/
+COPY lib/my_gem/version.rb /app/lib/my_gem/version.rb
+
+ARG BUNDLE_GEMFILE=Gemfile
+ENV BUNDLE_GEMFILE=$BUNDLE_GEMFILE
+
+RUN bundle install --jobs=4 --retry=3
+
+COPY . /app
+```
+
+### Running a command
+
+```
+bundle exec dmatrix -- bundle exec rspec
+```
+
+Would use `dmatrix` to build & run `bundle exec rspec` in each combination your
+matrix defines.
+
+## Setup
+
+```
+# Gemfile
+
+gem "dmatrix"
+```
+
+Create a `.matrix.yaml` file:
+
+```
+matrix:
+  build_arg:
+    ARG1:
+      - abc
+      - def
+  env:
+    ENV1:
+      - 123
+      - 456
+```
+
+N.B. the `build_arg` and `env` keys can both contain multiple variants.
+
+Run:
+
+```
+bundle exec dmatrix -- <your command>
+```
+
+If not command is specified the default Docker command should run.

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "dmatrix"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/dmatrix.gemspec

@@ -0,0 +1,30 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "dmatrix/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "dmatrix"
+  spec.version = Dmatrix::VERSION
+  spec.authors = ["Oliver Peate"]
+
+  spec.summary       = "Docker matrix runner with parallel execution"
+  spec.homepage      = "https://github.com/odlp/dmatrix"
+  spec.license       = "MIT"
+
+  spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "parallel", "~> 1.13"
+  spec.add_dependency "slop", "~> 4.5"
+
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "jet_black", "~> 0.5"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec", "~> 3.8"
+end

data/exe/dmatrix

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+
+require "slop"
+require "dmatrix"
+
+opts = Slop.parse do |o|
+  o.string "-m", "--matrix", "Matrix file", default: ".matrix.yaml"
+  o.string "--log-dir", "Log directory", default: "tmp/dmatrix"
+  o.on "--version", "Display the version" do
+    puts Dmatrix::VERSION
+    exit 0
+  end
+end
+
+Dmatrix::Runner.new(options: opts.to_hash, run_command: opts.arguments).call

data/lib/dmatrix.rb

@@ -0,0 +1,5 @@
+require "dmatrix/version"
+require "dmatrix/runner"
+
+module Dmatrix
+end

data/lib/dmatrix/executor.rb

@@ -0,0 +1,103 @@
+require "forwardable"
+require "shellwords"
+require_relative "image_tag"
+
+module Dmatrix
+  class Executor
+    extend Forwardable
+
+    def initialize(combination:, run_command:, log_dir:, executor: Kernel.method(:system))
+      @combination = combination
+      @run_command = run_command
+      @log_dir = log_dir
+      @executor = executor
+    end
+
+    def build_run
+      run_success = false
+      build_success = build
+
+      if build_success
+        run_success = run
+      end
+
+      Result.new(build_success, run_success, tag)
+    end
+
+    private
+
+    attr_reader :combination, :run_command, :log_dir, :executor
+
+    def_delegators :image_tag, :tag
+
+    def build
+      executor.call("docker build %{build_args} --tag %{tag} . > #{log_path(type: "build")} 2>&1" % build_params)
+    end
+
+    def run
+      executor.call("docker run %{env_args} %{tag} %{run_command} > #{log_path(type: "run")} 2>&1" % run_params)
+    end
+
+    def build_params
+      { tag: tag, build_args: build_args }
+    end
+
+    def run_params
+      {
+        tag: tag,
+        env_args: env_args,
+        run_command: formatted_run_command,
+      }
+    end
+
+    def image_tag
+      @image_tag ||= ImageTag.new(values: combination.aspects.map(&:value))
+    end
+
+    def build_args
+      args = combination.aspects.select { |a| a[:type] == "build_arg" }
+
+      if args.empty?
+        return ""
+      end
+
+      args.map do |build_arg|
+        "--build-arg #{build_arg.name}=#{build_arg.value}"
+      end.join(" ")
+    end
+
+    def env_args
+      args = combination.aspects.select { |a| a[:type] == "env" }
+
+      if args.empty?
+        return ""
+      end
+
+      args.map do |env_arg|
+        "--env #{env_arg.name}=#{env_arg.value}"
+      end.join(" ")
+    end
+
+    def formatted_run_command
+      if run_command.nil? || run_command.empty?
+        return ""
+      end
+
+      Shellwords.join(run_command)
+    end
+
+    def log_path(type:)
+      File.join(log_dir, "#{type}-#{tag.gsub(":", "-")}.log")
+    end
+
+    Result = Struct.new(:build_success, :run_success, :tag) do
+      def success?
+        build_success && run_success
+      end
+
+      def failure?
+        !success?
+      end
+    end
+  end
+end

data/lib/dmatrix/image_tag.rb

@@ -0,0 +1,26 @@
+module Dmatrix
+  class ImageTag
+    def initialize(values:)
+      @values = values
+    end
+
+    def tag
+      "#{repo}:#{combination_tag}"
+    end
+
+    private
+
+    attr_reader :values
+
+    INVALID_CHARS = /[^\w\-]/
+    private_constant :INVALID_CHARS
+
+    def repo
+      File.split(Dir.pwd).last
+    end
+
+    def combination_tag
+      values.join("-").gsub(INVALID_CHARS, "-")
+    end
+  end
+end

data/lib/dmatrix/logger.rb

@@ -0,0 +1,37 @@
+module Dmatrix
+  class Logger
+    def initialize(std_out: STDOUT)
+      @std_out = std_out
+    end
+
+    def log_result(result)
+      message = [
+        result.tag,
+        format_status("Build", result.build_success),
+        format_status("Run", result.run_success)
+      ]
+
+      std_out.puts(message.join("\t"))
+    end
+
+    private
+
+    attr_reader :std_out
+
+    def format_status(label, success)
+      if success
+        "#{label}: #{green('success')}"
+      else
+        "#{label}: #{red('failure')}"
+      end
+    end
+
+    def red(text)
+      "\e[31m#{text}\e[0m"
+    end
+
+    def green(text)
+      "\e[32m#{text}\e[0m"
+    end
+  end
+end

data/lib/dmatrix/matrix.rb

@@ -0,0 +1,28 @@
+module Dmatrix
+  class Matrix
+    def initialize(input)
+      @input = input
+    end
+
+    def combinations
+      dimensions = []
+
+      input.each do |type, dimension_group|
+        dimension_group.each do |(name, values)|
+          dimensions << values.map { |value| Aspect.new(type, name, value) }
+        end
+      end
+
+      dimensions.first.product(*dimensions.drop(1)).map do |aspects|
+        Combination.new(aspects)
+      end
+    end
+
+    private
+
+    attr_reader :input
+
+    Combination = Struct.new(:aspects)
+    Aspect = Struct.new(:type, :name, :value)
+  end
+end

data/lib/dmatrix/runner.rb

@@ -0,0 +1,64 @@
+require "fileutils"
+require "parallel"
+require "yaml"
+
+require_relative "executor"
+require_relative "logger"
+require_relative "matrix"
+
+module Dmatrix
+  class Runner
+    def initialize(options:, run_command:, logger: Logger.new)
+      @options = options
+      @run_command = run_command
+      @logger = logger
+    end
+
+    def call
+      reset_log_dir
+
+      results = Parallel.map(combinations, in_threads: 4) do |combination|
+        Executor.new(
+          combination: combination,
+          run_command: run_command,
+          log_dir: log_dir
+        ).build_run.tap do |result|
+          logger.log_result(result)
+        end
+      end
+
+      if results.any?(&:failure?)
+        exit(1)
+      end
+    end
+
+    private
+
+    attr_reader :options, :run_command, :logger
+
+    def combinations
+      Matrix.new(input_combinations).combinations
+    end
+
+    def input_combinations
+      YAML.load_file(matrix_path).fetch("matrix")
+    end
+
+    def matrix_path
+      options.fetch(:matrix)
+    end
+
+    def reset_log_dir
+      FileUtils.mkdir_p(log_dir)
+      FileUtils.rm_r(log_files)
+    end
+
+    def log_dir
+      options.fetch(:log_dir)
+    end
+
+    def log_files
+      Dir.glob(File.join(log_dir, "*.log"))
+    end
+  end
+end

data/lib/dmatrix/version.rb

@@ -0,0 +1,3 @@
+module Dmatrix
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,162 @@
+--- !ruby/object:Gem::Specification
+name: dmatrix
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Oliver Peate
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2019-06-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: slop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.5'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: jet_black
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.8'
+description: 
+email: 
+executables:
+- dmatrix
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".matrix.yaml"
+- ".rspec"
+- CHANGELOG.md
+- Dockerfile
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- dmatrix.gemspec
+- exe/dmatrix
+- lib/dmatrix.rb
+- lib/dmatrix/executor.rb
+- lib/dmatrix/image_tag.rb
+- lib/dmatrix/logger.rb
+- lib/dmatrix/matrix.rb
+- lib/dmatrix/runner.rb
+- lib/dmatrix/version.rb
+homepage: https://github.com/odlp/dmatrix
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Docker matrix runner with parallel execution
+test_files: []