rascal 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 814d874743694d11422907bf02213100735789bb30c4f6a04d5bf57d3a686541
-  data.tar.gz: 77a43969306a5d45e84a8f7a4e3518604b09989701afc4829188ae6385ef464d
+  metadata.gz: 5cac8b4564154cac2c8e002da1cb8e2874509331ad381cb6b8cc60b196954ab0
+  data.tar.gz: a577bd8374a055453a3db8a73bf211d69973fcab2fe74a3117aadd56149e644a
 SHA512:
-  metadata.gz: f076c894c0bebe868de8f56764ce47d2bd54ed8052550aa4595c48238515db324f5582f0f0f612976b81e06f99597bd76d303f8b2171ada400b221280c7ca4ae
-  data.tar.gz: e2ac5cf785f08ab3f1adfafde4c2bd8e6e0a2bb72ebb740b279d584379341915f4e595c5b987a1b8ecb00236cc7f50321e449074c83223cb62a3618ce6d88c7a
+  metadata.gz: 7d0770a88072f36e7848b0fb46ef11599e32759d3c1e17b0de8eebb2fd298b32a97f7aedc9ea191e4d996bddc27d12b313457a5adba7905465b03e68fd11fa87
+  data.tar.gz: 2113fab40ec414ce8d724a7cf34ef94f12246f7f93d4a6509b32292c6757b96e2993286c69ba53a155bb7a1485ab5fe3a39b5d41abcd40d9763f4c7b44b0c25b

data/CHANGELOG.md

@@ -5,6 +5,13 @@ All notable changes to this project will be documented here.
 Rascal follows semantic versioning. This has little consequence pre 1.0, so expect breaking changes.
 
 
+## 0.2.0 (2019-04-10)
+
+- Add `--all` flag for `rascal clean`.
+- Add `rascal update` command.
+- Allow rascal specific per job config.
+
+
 ## 0.1.0 (2019-03-21)
 
 initial release

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rascal (0.1.0)
+    rascal (0.2.0)
       thor (~> 0.20.3)
 
 GEM

data/README.md

@@ -30,14 +30,18 @@ You need to add some extra information to your `.gitlab-ci.yml`. A working versi
 
 ```
 # settings here override job settings
-.rascal:
-  repo_dir: /repo
-  variables:
+.rascal:                                 # add rascal specific config here
+  repo_dir: /repo                        # /repo is the default
+  variables:                             # extra env variables
     BUNDLE_PATH: /cache/bundle
-  volumes:
-    cache: /cache
+  volumes:                               # mount these volumes
+    cache: /cache                        # we will always mount a /builds volume
   before_shell:
-    - bundle check
+    - bundle check                       # run this when starting a shell
+  jobs:
+    rspec:                               # override settings for a specific job
+      variables:
+        BUNDLE_GEMFILE = /repo/Gemfile
 
 .environment: &environment
   image: registry.makandra.de/makandra/ci-images/test-env:2.5
@@ -53,7 +57,6 @@ You need to add some extra information to your `.gitlab-ci.yml`. A working versi
   variables:
     BUNDLE_PATH: ./bundle/vendor
     DATABASE_URL: postgresql://pg_user@pg-db/test-db
-    DATABASE_CLEANER_ALLOW_REMOTE_DATABASE_URL: "true"
     REDIS_URL: redis://redis
     PROMPT: CI env
   cache:
@@ -73,6 +76,27 @@ rspec:
 Then, in your project root, run `rascal shell rspec`.
 
 
+### Commands
+
+#### rascal shell <job>
+
+Start a docker container (plus required services) and open an interactive shell.
+
+Currently requires a "bash" to exist.
+
+
+#### rascal clean <job> | --all [--volumes]
+
+Stop and remove all created containers, services, networks for either the given or all jobs.
+
+If `--volumes` is given, also remove all cached volumes.
+
+
+#### rascal update <job> | --all
+
+Update all images for the given job, or all jobs.
+
+
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/lib/rascal/cli.rb

@@ -2,9 +2,10 @@ require 'thor'
 
 module Rascal
   module CLI
-    autoload :Base,  'rascal/cli/base'
-    autoload :Clean, 'rascal/cli/clean'
-    autoload :Main,  'rascal/cli/main'
-    autoload :Shell, 'rascal/cli/shell'
+    autoload :Base,   'rascal/cli/base'
+    autoload :Clean,  'rascal/cli/clean'
+    autoload :Main,   'rascal/cli/main'
+    autoload :Shell,  'rascal/cli/shell'
+    autoload :Update, 'rascal/cli/update'
   end
 end

data/lib/rascal/cli/base.rb

@@ -19,19 +19,38 @@ module Rascal
       end
 
       def find_environment(environment_name)
-        if (definition = EnvironmentsDefinition.detect(config_location))
-          if (environment = definition.environment(environment_name))
-            return environment
+        definition = environment_definition
+        if (environment = definition.environment(environment_name))
+          return environment
+        else
+          available_environments = definition.available_environment_names.join(', ')
+          if environment_name
+            fail_with_error("Unknown environment #{environment_name}. Available: #{available_environments}.")
           else
-            available_environments = definition.available_environment_names.join(', ')
-            if environment_name
-              fail_with_error("Unknown environment #{environment_name}. Available: #{available_environments}.")
-            else
-              fail_with_error("Missing environment. Available: #{available_environments}.")
-            end
+            fail_with_error("Missing environment. Available: #{available_environments}.")
+          end
+        end
+      end
+
+      def each_environment(name, &block)
+        return enum_for(:each_environment) unless block_given?
+
+        if name == :all
+          definition = environment_definition
+          definition.available_environment_names.each do |environment_name|
+            yield definition.environment(environment_name)
           end
+        else
+          yield find_environment(name)
+        end
+      end
+
+      def environment_definition
+        if (definition = EnvironmentsDefinition.detect(config_location))
+          definition
         else
           fail_with_error("Could not find an environment definition in current working directory.")
+          nil
         end
       end
     end

data/lib/rascal/cli/clean.rb

@@ -4,12 +4,19 @@ module Rascal
   module CLI
     class Clean < Base
       def initialize(thor, options, environment_name)
-        @environment_name = environment_name
+        @environment_name = if options[:all]
+          fail_with_error('Cannot give --all and an environment name!') if environment_name
+          :all
+        else
+          environment_name
+        end
         super(thor, options)
       end
 
       def run
-        find_environment(@environment_name)&.clean
+        each_environment(@environment_name) do |environment|
+          environment.clean(clean_volumes: @options[:volumes])
+        end
       end
     end
   end

data/lib/rascal/cli/main.rb

@@ -31,15 +31,37 @@ module Rascal
       map "shell" => "_shell"
       desc 'shell ENVIRONMENT', 'Start a docker shell for the given environment'
       def _shell(environment_name = nil)
-        Shell.new(self, options, environment_name).run
+        handle_error do
+          Shell.new(self, options, environment_name).run
+        end
       end
 
       desc 'clean ENVIRONMENT', 'Stop and remove docker containers for the given environment'
+      method_option :volumes, type: :boolean, default: false, desc: 'Remove (cache) volumes'
+      method_option :all, type: :boolean, default: false, desc: 'Clean all environments'
       def clean(environment_name = nil)
-        Clean.new(self, options, environment_name).run
+        handle_error do
+          Clean.new(self, options, environment_name).run
+        end
+      end
+
+      desc 'update ENVIRONMENT', 'Update all docker images'
+      method_option :all, type: :boolean, default: false, desc: 'update all available environments'
+      def update(environment_name = nil)
+        handle_error do
+          Update.new(self, options, environment_name).run
+        end
       end
 
-      class_option :config_file, aliases: ['-c'], default: '.', required: true, banner: 'path to configuration file or directory containing it'
+      class_option :config_file, aliases: ['-c'], default: '.', required: true, desc: 'path to configuration file or directory containing it'
+
+      private
+
+      def handle_error
+        yield
+      rescue Rascal::Error => e
+        raise Thor::Error, e.message
+      end
     end
   end
 end

data/lib/rascal/cli/update.rb

@@ -0,0 +1,24 @@
+require 'rascal'
+
+module Rascal
+  module CLI
+    class Update < Base
+      def initialize(thor, options, environment_name)
+        @environment_name = if options[:all]
+          fail_with_error('Cannot give --all and an environment name!') if environment_name
+          :all
+        else
+          environment_name
+        end
+        super(thor, options)
+      end
+
+      def run
+        images = []
+        each_environment(@environment_name) do |environment|
+          images += environment.update(skip: images)
+        end
+      end
+    end
+  end
+end

data/lib/rascal/docker/container.rb

@@ -3,6 +3,8 @@ module Rascal
     class Container
       include IOHelper
 
+      attr_reader :image
+
       def initialize(name, image)
         @name = name
         @prefixed_name = "#{NAME_PREFIX}#{name}"
@@ -26,7 +28,7 @@ module Rascal
             'container',
             'inspect',
             id,
-            output: :json
+            output: :json,
           ).first
           !!container_info.dig('State', 'Running')
         else
@@ -56,26 +58,55 @@ module Rascal
           *(['--network', network.id] if network),
           *(['--network-alias', network_alias] if network_alias),
           @image,
-          output: :id
+          output: :id,
         )
       end
 
       def run_and_attach(*command, env: {}, network: nil, volumes: [], working_dir: nil, allow_failure: false)
-        Docker.interface.run_and_attach(@image, *command,
-          env: env,
-          stdout: stdout,
-          stderr: stderr,
-          stdin: stdin,
-          network: network&.id,
-          volumes: volumes,
-          working_dir: working_dir,
-          allow_failure: allow_failure
+        Docker.interface.run_and_attach(
+          'container',
+          'run',
+          '--rm',
+          '-a', 'STDOUT',
+          '-a', 'STDERR',
+          '-a', 'STDIN',
+          '--interactive',
+          '--tty',
+          *(['-w', working_dir] if working_dir),
+          *(volumes.flat_map { |v| ['-v', v.to_param] }),
+          *(env.flat_map { |key, value| ['-e', "#{key}=#{value}"] }),
+          *(['--network', network.id] if network),
+          @image,
+          *command,
+          redirect_io: {
+            out: stdout,
+            err: stderr,
+            in: stdin,
+          },
+          allow_failure: allow_failure,
         )
       end
 
       def clean
-        stop_container if running?
-        remove_container if exists?
+        if running?
+          say "Stopping container for #{@name}"
+          stop_container
+        end
+        if exists?
+          say "Removing container for #{@name}"
+          remove_container
+        end
+      end
+
+      def update(skip: [])
+        return if skip.include?(@image)
+        say "Updating image #{@image}"
+        Docker.interface.run(
+          'pull',
+          @image,
+          stdout: stdout,
+        )
+        @image
       end
 
       private
@@ -87,7 +118,7 @@ module Rascal
           '--all',
           '--quiet',
           '--filter', "name=^/#{@prefixed_name}$",
-          output: :id
+          output: :id,
         )
       end
 

data/lib/rascal/docker/interface.rb

@@ -6,7 +6,7 @@ module Rascal
     class Interface
       class Error < Rascal::Error; end
 
-      def run(*command, output: :ignore, stdout: nil, redirect_io: {}, allow_failure: false)
+      def run(*command, output: :ignore, stdout: nil, allow_failure: false)
         save_stdout = ''
         save_stderr = ''
         exit_status = nil
@@ -22,60 +22,11 @@ module Rascal
         unless allow_failure || exit_status.success?
           raise Error, "docker command '#{command.join(' ')}' failed with error:\n#{save_stderr}"
         end
-        case output
-        when :json
-          begin
-            JSON.parse(save_stdout)
-          rescue JSON::ParserError
-            raise Error, "could not parse output of docker command '#{command.join(' ')}':\n#{save_stdout}"
-          end
-        when :id
-          save_stdout[/[0-9a-f]+/]
-        when :ignore
-          nil
-        else
-          raise ArgumentError, 'unknown option for :output'
-        end
+        parse_output(output, save_stdout, save_stderr, command: command.join(' '))
       end
 
-      def run_and_attach(image, *command, stdout: nil, stderr: nil, stdin: nil, env: {}, network: nil, volumes: [], working_dir: nil, allow_failure: false)
-        process_redirections = {}
-        args = []
-        if stdout
-          process_redirections[:out] = stdout
-          args += ['-a', 'STDOUT']
-        end
-        if stderr
-          process_redirections[:err] = stderr
-          args += ['-a', 'STDERR']
-        end
-        if stdin
-          process_redirections[:in] = stdin
-          args += ['-a', 'STDIN', '--interactive', '--tty']
-        end
-        if working_dir
-          args += ['-w', working_dir.to_s]
-        end
-        volumes.each do |volume|
-          args += ['-v', volume.to_param]
-        end
-        env.each do |key, value|
-          args += ['-e', "#{key}=#{value}"]
-        end
-        if network
-          args += ['--network', network.to_s]
-        end
-        exit_status = spawn(
-          env,
-          'docker',
-          'container',
-          'run',
-          '--rm',
-          *args,
-          image.to_s,
-          *stringify_command(command),
-          process_redirections,
-        )
+      def run_and_attach(*command, stdout: nil, stderr: nil, stdin: nil, env: {}, network: nil, volumes: [], working_dir: nil, redirect_io: {}, allow_failure: false)
+        exit_status = spawn(env, 'docker', *stringify_command(command), redirect_io)
         unless allow_failure || exit_status.success?
           raise Error, "docker container run failed"
         end
@@ -106,6 +57,23 @@ module Rascal
         rescue IOError
         end
       end
+
+      def parse_output(output, stdout, stderr, command:)
+        case output
+        when :json
+          begin
+            JSON.parse(stdout)
+          rescue JSON::ParserError
+            raise Error, "could not parse output of docker command '#{command}':\n#{stdout}"
+          end
+        when :id
+          stdout[/[0-9a-f]+/]
+        when :ignore
+          nil
+        else
+          raise ArgumentError, 'unknown option for :output'
+        end
+      end
     end
   end
 end

data/lib/rascal/docker/volume.rb

@@ -2,6 +2,7 @@ module Rascal
   module Docker
     module Volume
       class Base
+        include IOHelper
       end
 
       class Named < Base
@@ -15,10 +16,23 @@ module Rascal
         end
 
         def clean
+          if exists?
+            say "Removing volume #{@prefixed_name}"
+            Docker.interface.run(
+              'volume',
+              'rm',
+              @prefixed_name,
+            )
+          end
+        end
+
+        def exists?
           Docker.interface.run(
             'volume',
-            'rm',
-            @prefixed_name,
+            'ls',
+            '--quiet',
+            '--filter', "name=^#{@prefixed_name}$",
+            output: :id,
           )
         end
       end

data/lib/rascal/environment.rb

@@ -1,6 +1,6 @@
 module Rascal
   class Environment
-    attr_reader :name
+    attr_reader :name, :network, :container, :env_variables, :services, :volumes, :working_dir, :before_shell
 
     def initialize(name, image:, env_variables: {}, services: [], volumes: [], before_shell: [], working_dir: nil)
       @name = name
@@ -26,10 +26,17 @@ module Rascal
       )
     end
 
-    def clean
+    def clean(clean_volumes: false)
       @services.each(&:clean)
       @network.clean
-      @volumes.each(&:clean)
+      @volumes.each(&:clean) if clean_volumes
+    end
+
+    def update(skip: [])
+      [
+        *@services.collect { |s| s.update(skip: skip) },
+        *@container.update(skip: skip),
+      ]
     end
 
     private

data/lib/rascal/environments_definition/gitlab.rb

@@ -36,6 +36,7 @@ module Rascal
         @info = parse_definition(config_path.read)
         @repo_dir = config_path.parent
         @rascal_config = @info.fetch('.rascal', {})
+        @rascal_environment_config = @rascal_config.delete('jobs') || {}
       end
 
       def environment(name)
@@ -51,20 +52,28 @@ module Rascal
       private
 
       def parse_definition(yaml)
-        YAML.safe_load(yaml, [], [], true)
+        if Psych::VERSION >= '3.1'
+          YAML.safe_load(yaml, aliases: true)
+        else
+          YAML.safe_load(yaml, [], [], true)
+        end
       end
 
       def environments
         @environments ||= begin
           @info.collect do |key, environment_config|
-            config = Config.new(deep_merge(environment_config, @rascal_config), key)
-            docker_repo_dir = config.get('repo_dir')
+            config = Config.new(deep_merge(environment_config, @rascal_config, @rascal_environment_config[key] || {}), key)
+            docker_repo_dir = config.get('repo_dir', '/repo')
             unless key.start_with?('.')
               Environment.new(key,
                 image: config.get('image'),
                 env_variables: (config.get('variables', {})),
                 services: build_services(key, config.get('services', [])),
-                volumes: [build_repo_volume(docker_repo_dir), *build_volumes(key, config.get('volumes', {}))],
+                volumes: [
+                  build_repo_volume(docker_repo_dir),
+                  build_builds_volume(key),
+                  *build_volumes(key, config.get('volumes', {}))
+                ],
                 before_shell: config.get('before_shell', []),
                 working_dir: docker_repo_dir,
               )
@@ -73,9 +82,9 @@ module Rascal
         end
       end
 
-      def deep_merge(hash1, hash2)
+      def deep_merge(hash1, hash2, *other)
+        result = {}
         if hash1.is_a?(Hash) && hash2.is_a?(Hash)
-          result = {}
           hash1.each do |key1, value1|
             if hash2.has_key?(key1)
               result[key1] = deep_merge(value1, hash2[key1])
@@ -86,9 +95,13 @@ module Rascal
           hash2.each do |key2, value2|
             result[key2] ||= value2
           end
-          result
         else
-          hash2
+          result = hash2
+        end
+        if other.any?
+          deep_merge(result, *other)
+        else
+          result
         end
       end
 
@@ -106,6 +119,10 @@ module Rascal
         Docker::Volume::Bind.new(@repo_dir, docker_repo_dir)
       end
 
+      def build_builds_volume(name)
+        Docker::Volume::Named.new("#{name}-builds", '/builds')
+      end
+
       def build_volumes(name, volume_config)
         volume_config.collect do |volume_name, docker_path|
           Docker::Volume::Named.new("#{name}-#{volume_name}", docker_path)

data/lib/rascal/service.rb

@@ -1,6 +1,6 @@
 module Rascal
   class Service
-    attr_reader :name
+    attr_reader :name, :container, :alias
 
     def initialize(name, image:, alias_name:)
       @name = name
@@ -21,5 +21,9 @@ module Rascal
     def clean
       @container.clean
     end
+
+    def update(**args)
+      @container.update(**args)
+    end
   end
 end

data/lib/rascal/version.rb

@@ -1,3 +1,3 @@
 module Rascal
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rascal
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Tobias Kraze
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-03-21 00:00:00.000000000 Z
+date: 2019-04-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -56,6 +56,7 @@ files:
 - lib/rascal/cli/clean.rb
 - lib/rascal/cli/main.rb
 - lib/rascal/cli/shell.rb
+- lib/rascal/cli/update.rb
 - lib/rascal/docker.rb
 - lib/rascal/docker/container.rb
 - lib/rascal/docker/interface.rb