rascal 0.1.0 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 814d874743694d11422907bf02213100735789bb30c4f6a04d5bf57d3a686541
-  data.tar.gz: 77a43969306a5d45e84a8f7a4e3518604b09989701afc4829188ae6385ef464d
+  metadata.gz: cd535de9c0092cf4ab69d5e279e8cb0e19ec682d74e915844a781db8e7e2bb53
+  data.tar.gz: ce84b237c9302f56f810a03b3bf95711b279f3a931687d5d29bd21bbb4f325ca
 SHA512:
-  metadata.gz: f076c894c0bebe868de8f56764ce47d2bd54ed8052550aa4595c48238515db324f5582f0f0f612976b81e06f99597bd76d303f8b2171ada400b221280c7ca4ae
-  data.tar.gz: e2ac5cf785f08ab3f1adfafde4c2bd8e6e0a2bb72ebb740b279d584379341915f4e595c5b987a1b8ecb00236cc7f50321e449074c83223cb62a3618ce6d88c7a
+  metadata.gz: d2d95fdaf90bab2efc7e3987d142f37653526f30e6026a8a2d056cf85c8bb780b47251d85e8102faac8a0d4b715459c1dcec2d7e5f20d8fb413150778872844d
+  data.tar.gz: c5f35334569ea8be40d4f474aef6734c52289f33517dc7c96cbae432a2c02fccbb5bd906c1273ff077ddb21be89dcbd29b3a729c96853f26afb07dea9439e8a0

data/.travis.yml

@@ -12,5 +12,5 @@ notifications:
   email:
     - fail@makandra.de
 install:
-  - gem install bundler:2.0.1
+  - gem install bundler:2.1.4
   - bundle install --no-deployment --jobs=3 --retry=3 --path=${BUNDLE_PATH:-vendor/bundle}

data/CHANGELOG.md

@@ -4,6 +4,32 @@ All notable changes to this project will be documented here.
 
 Rascal follows semantic versioning. This has little consequence pre 1.0, so expect breaking changes.
 
+## 0.3.2 (2020-10-23)
+
+- Fix broken `rascal shell` command.
+
+
+## 0.3.1 (2020-10-01)
+
+- Pass env variables to service containers.
+
+
+## 0.3.0 (2020-09-23)
+
+- Mount /repo into all service volumes
+
+
+## 0.2.1 (2019-04-10)
+
+- Prefix names with directory name, to avoid conflicts between projects.
+
+
+## 0.2.0 (2019-04-10)
+
+- Add `--all` flag for `rascal clean`.
+- Add `rascal update` command.
+- Allow rascal specific per job config.
+
 
 ## 0.1.0 (2019-03-21)
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rascal (0.1.0)
+    rascal (0.3.2)
       thor (~> 0.20.3)
 
 GEM
@@ -107,4 +107,4 @@ DEPENDENCIES
   rspec
 
 BUNDLED WITH
-   2.0.1
+   2.1.4

data/README.md

@@ -30,14 +30,18 @@ You need to add some extra information to your `.gitlab-ci.yml`. A working versi
 
 ```
 # settings here override job settings
-.rascal:
-  repo_dir: /repo
-  variables:
+.rascal:                                 # add rascal specific config here
+  repo_dir: /repo                        # /repo is the default
+  variables:                             # extra env variables
     BUNDLE_PATH: /cache/bundle
-  volumes:
-    cache: /cache
+  volumes:                               # mount these volumes
+    cache: /cache                        # we will always mount a /builds volume
   before_shell:
-    - bundle check
+    - bundle check                       # run this when starting a shell
+  jobs:
+    rspec:                               # override settings for a specific job
+      variables:
+        BUNDLE_GEMFILE = /repo/Gemfile
 
 .environment: &environment
   image: registry.makandra.de/makandra/ci-images/test-env:2.5
@@ -53,7 +57,6 @@ You need to add some extra information to your `.gitlab-ci.yml`. A working versi
   variables:
     BUNDLE_PATH: ./bundle/vendor
     DATABASE_URL: postgresql://pg_user@pg-db/test-db
-    DATABASE_CLEANER_ALLOW_REMOTE_DATABASE_URL: "true"
     REDIS_URL: redis://redis
     PROMPT: CI env
   cache:
@@ -73,6 +76,27 @@ rspec:
 Then, in your project root, run `rascal shell rspec`.
 
 
+### Commands
+
+#### rascal shell <job>
+
+Start a docker container (plus required services) and open an interactive shell.
+
+Currently requires a "bash" to exist.
+
+
+#### rascal clean <job> | --all [--volumes]
+
+Stop and remove all created containers, services, networks for either the given or all jobs.
+
+If `--volumes` is given, also remove all cached volumes.
+
+
+#### rascal update <job> | --all
+
+Update all images for the given job, or all jobs.
+
+
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/lib/rascal/cli.rb

@@ -2,9 +2,10 @@ require 'thor'
 
 module Rascal
   module CLI
-    autoload :Base,  'rascal/cli/base'
-    autoload :Clean, 'rascal/cli/clean'
-    autoload :Main,  'rascal/cli/main'
-    autoload :Shell, 'rascal/cli/shell'
+    autoload :Base,   'rascal/cli/base'
+    autoload :Clean,  'rascal/cli/clean'
+    autoload :Main,   'rascal/cli/main'
+    autoload :Shell,  'rascal/cli/shell'
+    autoload :Update, 'rascal/cli/update'
   end
 end

data/lib/rascal/cli/base.rb

@@ -19,19 +19,38 @@ module Rascal
       end
 
       def find_environment(environment_name)
-        if (definition = EnvironmentsDefinition.detect(config_location))
-          if (environment = definition.environment(environment_name))
-            return environment
+        definition = environment_definition
+        if (environment = definition.environment(environment_name))
+          return environment
+        else
+          available_environments = definition.available_environment_names.join(', ')
+          if environment_name
+            fail_with_error("Unknown environment #{environment_name}. Available: #{available_environments}.")
           else
-            available_environments = definition.available_environment_names.join(', ')
-            if environment_name
-              fail_with_error("Unknown environment #{environment_name}. Available: #{available_environments}.")
-            else
-              fail_with_error("Missing environment. Available: #{available_environments}.")
-            end
+            fail_with_error("Missing environment. Available: #{available_environments}.")
+          end
+        end
+      end
+
+      def each_environment(name, &block)
+        return enum_for(:each_environment) unless block_given?
+
+        if name == :all
+          definition = environment_definition
+          definition.available_environment_names.each do |environment_name|
+            yield definition.environment(environment_name)
           end
+        else
+          yield find_environment(name)
+        end
+      end
+
+      def environment_definition
+        if (definition = EnvironmentsDefinition.detect(config_location))
+          definition
         else
           fail_with_error("Could not find an environment definition in current working directory.")
+          nil
         end
       end
     end

data/lib/rascal/cli/clean.rb

@@ -4,12 +4,19 @@ module Rascal
   module CLI
     class Clean < Base
       def initialize(thor, options, environment_name)
-        @environment_name = environment_name
+        @environment_name = if options[:all]
+          fail_with_error('Cannot give --all and an environment name!') if environment_name
+          :all
+        else
+          environment_name
+        end
         super(thor, options)
       end
 
       def run
-        find_environment(@environment_name)&.clean
+        each_environment(@environment_name) do |environment|
+          environment.clean(clean_volumes: @options[:volumes])
+        end
       end
     end
   end

data/lib/rascal/cli/main.rb

@@ -31,15 +31,37 @@ module Rascal
       map "shell" => "_shell"
       desc 'shell ENVIRONMENT', 'Start a docker shell for the given environment'
       def _shell(environment_name = nil)
-        Shell.new(self, options, environment_name).run
+        handle_error do
+          Shell.new(self, options, environment_name).run
+        end
       end
 
       desc 'clean ENVIRONMENT', 'Stop and remove docker containers for the given environment'
+      method_option :volumes, type: :boolean, default: false, desc: 'Remove (cache) volumes'
+      method_option :all, type: :boolean, default: false, desc: 'Clean all environments'
       def clean(environment_name = nil)
-        Clean.new(self, options, environment_name).run
+        handle_error do
+          Clean.new(self, options, environment_name).run
+        end
+      end
+
+      desc 'update ENVIRONMENT', 'Update all docker images'
+      method_option :all, type: :boolean, default: false, desc: 'update all available environments'
+      def update(environment_name = nil)
+        handle_error do
+          Update.new(self, options, environment_name).run
+        end
       end
 
-      class_option :config_file, aliases: ['-c'], default: '.', required: true, banner: 'path to configuration file or directory containing it'
+      class_option :config_file, aliases: ['-c'], default: '.', required: true, desc: 'path to configuration file or directory containing it'
+
+      private
+
+      def handle_error
+        yield
+      rescue Rascal::Error => e
+        raise Thor::Error, e.message
+      end
     end
   end
 end

data/lib/rascal/cli/update.rb

@@ -0,0 +1,24 @@
+require 'rascal'
+
+module Rascal
+  module CLI
+    class Update < Base
+      def initialize(thor, options, environment_name)
+        @environment_name = if options[:all]
+          fail_with_error('Cannot give --all and an environment name!') if environment_name
+          :all
+        else
+          environment_name
+        end
+        super(thor, options)
+      end
+
+      def run
+        images = []
+        each_environment(@environment_name) do |environment|
+          images += environment.update(skip: images)
+        end
+      end
+    end
+  end
+end

data/lib/rascal/docker/container.rb

@@ -3,6 +3,8 @@ module Rascal
     class Container
       include IOHelper
 
+      attr_reader :image
+
       def initialize(name, image)
         @name = name
         @prefixed_name = "#{NAME_PREFIX}#{name}"
@@ -26,7 +28,7 @@ module Rascal
             'container',
             'inspect',
             id,
-            output: :json
+            output: :json,
           ).first
           !!container_info.dig('State', 'Running')
         else
@@ -38,9 +40,9 @@ module Rascal
         !!id
       end
 
-      def start(network: nil, network_alias: nil)
+      def start(network: nil, network_alias: nil, volumes: [], env: {})
         say "Starting container for #{@name}"
-        create(network: network, network_alias: network_alias) unless exists?
+        create(network: network, network_alias: network_alias, volumes: volumes, env: env) unless exists?
         Docker.interface.run(
           'container',
           'start',
@@ -48,34 +50,65 @@ module Rascal
         )
       end
 
-      def create(network: nil, network_alias: nil)
+      def create(network: nil, network_alias: nil, volumes: [], env: {})
         @id = Docker.interface.run(
           'container',
           'create',
           '--name', @prefixed_name,
+          *(volumes.flat_map { |v| ['-v', v.to_param] }),
+          *env_args(env),
           *(['--network', network.id] if network),
           *(['--network-alias', network_alias] if network_alias),
           @image,
-          output: :id
+          output: :id,
         )
       end
 
       def run_and_attach(*command, env: {}, network: nil, volumes: [], working_dir: nil, allow_failure: false)
-        Docker.interface.run_and_attach(@image, *command,
-          env: env,
-          stdout: stdout,
-          stderr: stderr,
-          stdin: stdin,
-          network: network&.id,
-          volumes: volumes,
-          working_dir: working_dir,
-          allow_failure: allow_failure
+        Docker.interface.run_and_attach(
+          'container',
+          'run',
+          '--rm',
+          '-a', 'STDOUT',
+          '-a', 'STDERR',
+          '-a', 'STDIN',
+          '--interactive',
+          '--tty',
+          *(['-w', working_dir] if working_dir),
+          *(volumes.flat_map { |v| ['-v', v.to_param] }),
+          *env_args(env),
+          *(['--network', network.id] if network),
+          @image,
+          *command,
+          redirect_io: {
+            out: stdout,
+            err: stderr,
+            in: stdin,
+          },
+          allow_failure: allow_failure,
         )
       end
 
       def clean
-        stop_container if running?
-        remove_container if exists?
+        if running?
+          say "Stopping container for #{@name}"
+          stop_container
+        end
+        if exists?
+          say "Removing container for #{@name}"
+          remove_container
+        end
+      end
+
+      def update(skip: [])
+        return if skip.include?(@image)
+        say "Updating image #{@image}"
+        Docker.interface.run(
+          'pull',
+          @image,
+          stdout: stdout,
+        )
+        @image
       end
 
       private
@@ -87,7 +120,7 @@ module Rascal
           '--all',
           '--quiet',
           '--filter', "name=^/#{@prefixed_name}$",
-          output: :id
+          output: :id,
         )
       end
 
@@ -116,6 +149,10 @@ module Rascal
           id,
         )
       end
+
+      def env_args(env)
+        env.flat_map { |key, value| ['-e', "#{key}=#{value}"] }
+      end
     end
   end
 end

data/lib/rascal/docker/interface.rb

@@ -6,7 +6,7 @@ module Rascal
     class Interface
       class Error < Rascal::Error; end
 
-      def run(*command, output: :ignore, stdout: nil, redirect_io: {}, allow_failure: false)
+      def run(*command, output: :ignore, stdout: nil, allow_failure: false)
         save_stdout = ''
         save_stderr = ''
         exit_status = nil
@@ -22,60 +22,11 @@ module Rascal
         unless allow_failure || exit_status.success?
           raise Error, "docker command '#{command.join(' ')}' failed with error:\n#{save_stderr}"
         end
-        case output
-        when :json
-          begin
-            JSON.parse(save_stdout)
-          rescue JSON::ParserError
-            raise Error, "could not parse output of docker command '#{command.join(' ')}':\n#{save_stdout}"
-          end
-        when :id
-          save_stdout[/[0-9a-f]+/]
-        when :ignore
-          nil
-        else
-          raise ArgumentError, 'unknown option for :output'
-        end
+        parse_output(output, save_stdout, save_stderr, command: command.join(' '))
       end
 
-      def run_and_attach(image, *command, stdout: nil, stderr: nil, stdin: nil, env: {}, network: nil, volumes: [], working_dir: nil, allow_failure: false)
-        process_redirections = {}
-        args = []
-        if stdout
-          process_redirections[:out] = stdout
-          args += ['-a', 'STDOUT']
-        end
-        if stderr
-          process_redirections[:err] = stderr
-          args += ['-a', 'STDERR']
-        end
-        if stdin
-          process_redirections[:in] = stdin
-          args += ['-a', 'STDIN', '--interactive', '--tty']
-        end
-        if working_dir
-          args += ['-w', working_dir.to_s]
-        end
-        volumes.each do |volume|
-          args += ['-v', volume.to_param]
-        end
-        env.each do |key, value|
-          args += ['-e', "#{key}=#{value}"]
-        end
-        if network
-          args += ['--network', network.to_s]
-        end
-        exit_status = spawn(
-          env,
-          'docker',
-          'container',
-          'run',
-          '--rm',
-          *args,
-          image.to_s,
-          *stringify_command(command),
-          process_redirections,
-        )
+      def run_and_attach(*command, stdout: nil, stderr: nil, stdin: nil, env: {}, network: nil, volumes: [], working_dir: nil, redirect_io: {}, allow_failure: false)
+        exit_status = spawn(env, 'docker', *stringify_command(command), redirect_io)
         unless allow_failure || exit_status.success?
           raise Error, "docker container run failed"
         end
@@ -106,6 +57,23 @@ module Rascal
         rescue IOError
         end
       end
+
+      def parse_output(output, stdout, stderr, command:)
+        case output
+        when :json
+          begin
+            JSON.parse(stdout)
+          rescue JSON::ParserError
+            raise Error, "could not parse output of docker command '#{command}':\n#{stdout}"
+          end
+        when :id
+          stdout[/[0-9a-f]+/]
+        when :ignore
+          nil
+        else
+          raise ArgumentError, 'unknown option for :output'
+        end
+      end
     end
   end
 end

data/lib/rascal/docker/volume.rb

@@ -2,6 +2,7 @@ module Rascal
   module Docker
     module Volume
       class Base
+        include IOHelper
       end
 
       class Named < Base
@@ -15,10 +16,23 @@ module Rascal
         end
 
         def clean
+          if exists?
+            say "Removing volume #{@prefixed_name}"
+            Docker.interface.run(
+              'volume',
+              'rm',
+              @prefixed_name,
+            )
+          end
+        end
+
+        def exists?
           Docker.interface.run(
             'volume',
-            'rm',
-            @prefixed_name,
+            'ls',
+            '--quiet',
+            '--filter', "name=^#{@prefixed_name}$",
+            output: :id,
           )
         end
       end

data/lib/rascal/environment.rb

@@ -1,22 +1,23 @@
 module Rascal
   class Environment
-    attr_reader :name
+    attr_reader :name, :network, :container, :env_variables, :services, :volumes, :working_dir, :before_shell
 
-    def initialize(name, image:, env_variables: {}, services: [], volumes: [], before_shell: [], working_dir: nil)
+    def initialize(full_name, name:, image:, env_variables: {}, services: [], volumes: [], before_shell: [], after_shell: [], working_dir: nil)
       @name = name
-      @network = Docker::Network.new(name)
-      @container = Docker::Container.new(name, image)
+      @network = Docker::Network.new(full_name)
+      @container = Docker::Container.new(full_name, image)
       @env_variables = env_variables
       @services = services
       @volumes = volumes
       @working_dir = working_dir
       @before_shell = before_shell
+      @after_shell = after_shell
     end
 
     def run_shell
       download_missing
       start_services
-      command = [*@before_shell, 'bash'].join(';')
+      command = [*@before_shell, 'bash', *@after_shell].join(';')
       @container.run_and_attach('bash', '-c', command,
         env: @env_variables,
         network: @network,
@@ -26,10 +27,17 @@ module Rascal
       )
     end
 
-    def clean
+    def clean(clean_volumes: false)
       @services.each(&:clean)
       @network.clean
-      @volumes.each(&:clean)
+      @volumes.each(&:clean) if clean_volumes
+    end
+
+    def update(skip: [])
+      [
+        *@services.collect { |s| s.update(skip: skip) },
+        *@container.update(skip: skip),
+      ]
     end
 
     private

data/lib/rascal/environments_definition/gitlab.rb

@@ -35,7 +35,9 @@ module Rascal
       def initialize(config_path)
         @info = parse_definition(config_path.read)
         @repo_dir = config_path.parent
+        @base_name = @repo_dir.basename
         @rascal_config = @info.fetch('.rascal', {})
+        @rascal_environment_config = @rascal_config.delete('jobs') || {}
       end
 
       def environment(name)
@@ -51,21 +53,34 @@ module Rascal
       private
 
       def parse_definition(yaml)
-        YAML.safe_load(yaml, [], [], true)
+        if Psych::VERSION >= '3.1'
+          YAML.safe_load(yaml, aliases: true)
+        else
+          YAML.safe_load(yaml, [], [], true)
+        end
       end
 
       def environments
         @environments ||= begin
           @info.collect do |key, environment_config|
-            config = Config.new(deep_merge(environment_config, @rascal_config), key)
-            docker_repo_dir = config.get('repo_dir')
-            unless key.start_with?('.')
-              Environment.new(key,
+            config = Config.new(deep_merge(environment_config, @rascal_config, @rascal_environment_config[key] || {}), key)
+            docker_repo_dir = config.get('repo_dir', '/repo')
+            unless key.start_with?('.') || config.get('hide', false)
+              name = config.get('name', key)
+              full_name = "#{@base_name}-#{name}"
+              shared_volumes = [build_repo_volume(docker_repo_dir), build_builds_volume(full_name)]
+              env_variables = (config.get('variables', {}))
+              Environment.new(full_name,
+                name: name,
                 image: config.get('image'),
-                env_variables: (config.get('variables', {})),
-                services: build_services(key, config.get('services', [])),
-                volumes: [build_repo_volume(docker_repo_dir), *build_volumes(key, config.get('volumes', {}))],
+                volumes: [
+                  *shared_volumes,
+                  *build_volumes(full_name, config.get('volumes', {}))
+                ],
+                env_variables: env_variables,
+                services: build_services(full_name, config.get('services', []), volumes: shared_volumes, env_variables: env_variables),
                 before_shell: config.get('before_shell', []),
+                after_shell: config.get('after_shell', []),
                 working_dir: docker_repo_dir,
               )
             end
@@ -73,9 +88,9 @@ module Rascal
         end
       end
 
-      def deep_merge(hash1, hash2)
+      def deep_merge(hash1, hash2, *other)
+        result = {}
         if hash1.is_a?(Hash) && hash2.is_a?(Hash)
-          result = {}
           hash1.each do |key1, value1|
             if hash2.has_key?(key1)
               result[key1] = deep_merge(value1, hash2[key1])
@@ -86,18 +101,24 @@ module Rascal
           hash2.each do |key2, value2|
             result[key2] ||= value2
           end
-          result
         else
-          hash2
+          result = hash2
+        end
+        if other.any?
+          deep_merge(result, *other)
+        else
+          result
         end
       end
 
-      def build_services(name, services)
+      def build_services(name, services, volumes: [], env_variables: {})
         services.collect do |service_config|
           service_alias = service_config['alias']
           Service.new("#{name}_#{service_alias}",
             alias_name: service_config['alias'],
             image: service_config['name'],
+            volumes: volumes,
+            env_variables: env_variables,
           )
         end
       end
@@ -106,6 +127,10 @@ module Rascal
         Docker::Volume::Bind.new(@repo_dir, docker_repo_dir)
       end
 
+      def build_builds_volume(name)
+        Docker::Volume::Named.new("#{name}-builds", '/builds')
+      end
+
       def build_volumes(name, volume_config)
         volume_config.collect do |volume_name, docker_path|
           Docker::Volume::Named.new("#{name}-#{volume_name}", docker_path)

data/lib/rascal/service.rb

@@ -1,11 +1,13 @@
 module Rascal
   class Service
-    attr_reader :name
+    attr_reader :name, :container, :alias, :env_variables
 
-    def initialize(name, image:, alias_name:)
+    def initialize(name, env_variables: {}, image:, alias_name:, volumes: [])
       @name = name
       @container = Docker::Container.new(name, image)
       @alias = alias_name
+      @volumes = volumes
+      @env_variables = env_variables
     end
 
     def download_missing
@@ -14,12 +16,16 @@ module Rascal
 
     def start_if_stopped(network: nil)
       unless @container.running?
-        @container.start(network: network, network_alias: @alias)
+        @container.start(network: network, network_alias: @alias, volumes: @volumes, env: @env_variables)
       end
     end
 
     def clean
       @container.clean
     end
+
+    def update(**args)
+      @container.update(**args)
+    end
   end
 end

data/lib/rascal/version.rb

@@ -1,3 +1,3 @@
 module Rascal
-  VERSION = "0.1.0"
+  VERSION = "0.3.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rascal
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.2
 platform: ruby
 authors:
 - Tobias Kraze
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-03-21 00:00:00.000000000 Z
+date: 2020-10-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -56,6 +56,7 @@ files:
 - lib/rascal/cli/clean.rb
 - lib/rascal/cli/main.rb
 - lib/rascal/cli/shell.rb
+- lib/rascal/cli/update.rb
 - lib/rascal/docker.rb
 - lib/rascal/docker/container.rb
 - lib/rascal/docker/interface.rb
@@ -90,7 +91,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Spin up CI environments locally.