djin 0.8.0 → 0.11.2

data/Vertofile

@@ -1,4 +1,4 @@
-verto_version '0.8.0'
+verto_version '0.10.0'
 
 config {
   version.prefix = 'v' # Adds a version_prefix
@@ -12,28 +12,21 @@ context(branch('master')) {
   }
 
   before_tag_creation {
-    version_changes = sh(
-      %q#git log --oneline --decorate  | grep -B 100 -m 1 "tag:" | grep "pull request" | awk '{print $1}' | xargs git show --format='%b' | grep -v Approved | grep -v "^$" | grep -E "^[[:space:]]*\[.*\]" | sed 's/^[[:space:]]*\(.*\)/ * \1/'#, output: false
-     ).output
-
-    puts "---------------------------"
-    version_changes = "## #{new_version} - #{Time.now.strftime('%d/%m/%Y')}\n#{version_changes}\n"
-    exit unless confirm("Create new Realease?\n" \
-      "---------------------------\n" \
-      "#{version_changes}" \
-      "---------------------------\n"
-    )
-
-    # CHANGELOG
-    file('CHANGELOG.md').prepend(version_changes)
+    update_changelog(with: :merged_pull_requests_with_bracketed_labels,
+                     confirmation: true,
+                     filename: 'CHANGELOG.md')
+
     git!('add CHANGELOG.md')
 
-    file('lib/djin/version.rb').replace(latest_version.to_s, new_version.to_s)
-    file('djin.yml').replace(latest_version.to_s, new_version.to_s)
-    file('examples/djin.yml').replace(latest_version.to_s, new_version.to_s)
+    files_to_change_version_once = %w[lib/djin/version.rb djin.yml] + Dir['examples/**/*.yml'] + Dir['spec/support/fixtures/**/*.yml']
+
+    files_to_change_version_once.each do |filename|
+      file(filename).replace(latest_version.to_s, new_version.to_s)
+    end
+
     file('README.md').replace_all(latest_version.to_s, new_version.to_s)
 
-    git!('add lib/djin/version.rb djin.yml examples/djin.yml README.md')
+    git!("add #{files_to_change_version_once.join(' ')} README.md")
 
     sh!('bundle install')
     sh!('rake install')

data/djin.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files         = Dir.chdir(File.expand_path(__dir__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|docker)/}) }
   end
   spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
@@ -31,6 +31,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'dry-equalizer', '~> 0.3.0'
   spec.add_dependency 'dry-struct', '~> 1.3.0'
   spec.add_dependency 'dry-validation', '~> 1.5.1'
+  spec.add_dependency 'git', '~> 1.8.1'
   spec.add_dependency 'mustache', '~> 1.1.1'
   spec.add_dependency 'vseries', '~> 0.1.0'
   spec.add_development_dependency 'bundler', '~> 2.0'
@@ -38,4 +39,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'simplecov', '~> 0.17.0'
 end

data/djin.yml

@@ -1,4 +1,4 @@
-djin_version: '0.8.0'
+djin_version: '0.11.2'
 
 _default_run_options: &default_run_options
   options: "--rm --entrypoint=''"
@@ -9,11 +9,21 @@ tasks:
     docker-compose:
       service: app
       run:
-        commands: "cd /usr/src/djin && rspec {{args}}"
+        commands: "rspec {{args}}"
         <<: *default_run_options
     aliases:
       - rspec
 
+  lint:
+    description: Lint
+    docker-compose:
+      service: app
+      run:
+        commands: "rubocop {{args}}"
+        <<: *default_run_options
+    aliases:
+      - rubocop
+
   sh:
     description: Enter app service shell
     docker-compose:
@@ -31,5 +41,6 @@ tasks:
   release:
     local:
       run:
+        - (source ~/.zshrc || true)
         - verto tag up {{args}}
         - bundle exec rake release

data/docker-compose.yml

@@ -1,6 +1,20 @@
-version: '3'
+version: "3.9"
+
 services:
   app:
-    build: .
+    build:
+      context: .
+      target: dev
+    entrypoint: 'sh docker-entrypoint.sh'
+    command: 'djin'
+    tty: true
+    stdin_open: true
     volumes:
       - .:/usr/src/djin
+    depends_on:
+      - gitserver
+
+  gitserver:
+    image: catks/gitserver-http:0.1.0
+    volumes:
+      - ./docker/git_server/repos/:/var/lib/initial/

data/docker-entrypoint.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+bundle check || bundle install
+
+exec bundle exec "$@"

data/examples/djin.yml

@@ -1,5 +1,17 @@
 ---
-djin_version: '0.8.0'
+djin_version: '0.11.2'
+
+include:
+  - file: 'djin_lib/test.yml'
+    context:
+      variables:
+        namespace: 'test:'
+
+  - file: 'djin_lib/test.yml'
+    context:
+      variables:
+        namespace: 'test2:'
+
 
 tasks:
   default:
@@ -7,12 +19,6 @@ tasks:
       image: "ruby:2.5"
       run:
         - "ruby -e 'puts \\\" Hello\\\"'"
-  test:
-    docker-compose:
-      service: app
-      run:
-        commands: rspec
-        options: "--rm"
 
   script:
     docker:

data/examples/djin_lib/test.yml

@@ -0,0 +1,12 @@
+djin_version: '0.11.2'
+
+_default_run_options: &default_run_options
+  options: "--rm --entrypoint=''"
+
+tasks:
+  "{{namespace}}unit":
+    docker-compose:
+      service: app
+      run:
+        commands: "cd /usr/src/djin && rspec {{args}}"
+        <<: *default_run_options

data/examples/local_tasks/.djin/server_tasks.yml

@@ -0,0 +1,17 @@
+djin_version: '0.8.0'
+
+tasks:
+  "{{namespace}}:ssh":
+    local:
+      run:
+        - ssh {{ssh_user}}@{{host}}
+
+  "{{namespace}}:restart":
+    local:
+      run:
+        - ssh -t {{ssh_user}}@{{host}} restart
+
+  "{{namespace}}:logs":
+    local:
+      run:
+        - ssh -t {{ssh_user}}@{{host}} tail -f /var/log/my_log

data/examples/local_tasks/djin.yml

@@ -0,0 +1,22 @@
+djin_version: '0.11.2'
+
+include:
+  - file: '.djin/server_tasks.yml'
+    context:
+      variables:
+        namespace: host1
+        host: host1.com
+        ssh_user: my_user
+
+  - file: '.djin/server_tasks.yml'
+    context:
+      variables:
+        namespace: host2
+        host: host2.com
+        ssh_user: my_user
+
+tasks:
+  hello_command:
+    local:
+      run:
+        - echo 'Hello Djin'

data/examples/remote_tasks/djin.yml

@@ -0,0 +1,9 @@
+djin_version: '0.11.2'
+
+include:
+  - git: 'https://gitserver/myrepo.git'
+    version: 'master'
+    file: 'examples/djin_lib/test.yml'
+    context:
+      variables:
+        namespace: 'remote:'

data/lib/djin.rb

@@ -8,44 +8,94 @@ require 'dry-validation'
 require 'vseries'
 require 'dry/cli'
 require 'mustache'
+require 'optparse'
+require 'git'
 require 'djin/extensions/hash_extensions'
+require 'djin/extensions/object_extensions'
 require 'djin/entities/types'
 require 'djin/entities/task'
-require 'djin/entities/file_config'
+require 'djin/entities/include_config.rb'
+require 'djin/entities/main_config'
 require 'djin/interpreter/base_command_builder'
 require 'djin/interpreter/docker_command_builder'
 require 'djin/interpreter/docker_compose_command_builder'
 require 'djin/interpreter/local_command_builder'
 require 'djin/interpreter'
+require 'djin/include_resolver'
 require 'djin/config_loader'
 require 'djin/executor'
+require 'djin/root_cli_parser'
 require 'djin/cli'
 require 'djin/task_contract'
 require 'djin/repositories/task_repository'
+require 'djin/repositories/remote_config_repository'
+require 'djin/memory_cache'
 
 module Djin
   class Error < StandardError; end
 
-  def self.load_tasks!(path = Pathname.getwd.join('djin.yml'))
-    abort 'Error: djin.yml not found' unless path.exist?
+  using Djin::ObjectExtensions
 
-    file_config = ConfigLoader.load!(path.read)
+  class << self
+    def load_tasks!(*file_paths)
+      files = file_paths.presence || RootCliParser.parse![:files] || ['djin.yml']
 
-    # TODO: Make all tasks be under 'tasks' key, passing only the tasks here
-    tasks = Interpreter.load!(file_config)
+      file_config = ConfigLoader.load_files!(*files)
 
-    @task_repository = TaskRepository.new(tasks)
-    CLI.load_tasks!(tasks)
-  rescue Djin::Interpreter::InvalidConfigurationError => e
-    error_name = e.class.name.split('::').last
-    abort("[#{error_name}] #{e.message}")
-  end
+      # TODO: Make all tasks be under 'tasks' key, passing only the tasks here
+      tasks = Interpreter.load!(file_config)
 
-  def self.tasks
-    task_repository.all
-  end
+      @task_repository = TaskRepository.new(tasks)
+
+      remote_configs = file_config.include_configs.select { |f| f.type == :remote }
+      @remote_config_repository = RemoteConfigRepository.new(remote_configs)
+
+      CLI.load_tasks!(tasks)
+    rescue Djin::Interpreter::InvalidConfigurationError => e
+      error_name = e.class.name.split('::').last
+      abort("[#{error_name}] #{e.message}")
+    end
+
+    def tasks
+      task_repository.all
+    end
+
+    def task_repository
+      @task_repository ||= TaskRepository.new
+    end
+
+    def remote_config_repository
+      @remote_config_repository ||= RemoteConfigRepository.new
+    end
+
+    def cache
+      @cache ||= MemoryCache.new
+    end
+
+    def root_path
+      Pathname.new File.expand_path(__dir__ + '/..')
+    end
+
+    def warn(message, type: 'WARNING')
+      stderr.puts "[#{type}] #{message}"
+    end
+
+    def warn_once(message, type: 'WARNING')
+      return if warnings.include?(message)
+
+      warn(message, type: type)
+
+      warnings << message
+    end
+
+    def stderr
+      $stderr
+    end
+
+    private
 
-  def self.task_repository
-    @task_repository ||= TaskRepository.new
+    def warnings
+      @warnings ||= []
+    end
   end
 end

data/lib/djin/cli.rb

@@ -28,6 +28,42 @@ module Djin
       end
     end
 
+    class File < Dry::CLI::Command
+      desc 'Specify a djin file to load (default: djin.yml)'
+      argument :filepath, required: true, desc: 'The file path to load'
+
+      def call(filename:, **)
+        # The actual behaviour is on RootCliParser
+      end
+    end
+
+    module RemoteConfig
+      class Fetch < Dry::CLI::Command
+        desc 'Fetchs missing remote configs'
+
+        def call(*)
+          Djin.remote_config_repository.fetch_all
+        end
+      end
+
+      class Clear < Dry::CLI::Command
+        desc 'clear downloaded remote configs'
+        option :all,
+               type: :boolean,
+               default: false,
+               desc: 'Remove all remote configs, not only the ones referenced in the current djin file'
+
+        def call(all:)
+          return Djin.remote_config_repository.clear_all if all
+
+          Djin.remote_config_repository.clear
+        end
+      end
+    end
+
+    register '-f', File, aliases: ['--file']
     register '--version', Version, aliases: ['-v']
+    register 'remote-config fetch', RemoteConfig::Fetch
+    register 'remote-config clear', RemoteConfig::Clear
   end
 end

data/lib/djin/config_loader.rb

@@ -4,20 +4,38 @@ module Djin
   # TODO: Refactor this class to be the Interpreter
   #       class and use the current interpreter as
   #       a TaskLoader
+
+  # rubocop:disable Metrics/ClassLength
   class ConfigLoader
     using Djin::HashExtensions
-    RESERVED_WORDS = %w[djin_version variables tasks].freeze
+    RESERVED_WORDS = %w[djin_version variables tasks include].freeze
+
+    # Change Base Error
+    FileNotFoundError = Class.new(Interpreter::InvalidConfigurationError)
+
+    def self.load_files!(*files, runtime_config: {}, base_directory: '.')
+      files.map do |file_path|
+        ConfigLoader.load!(file_path, runtime_config: runtime_config, base_directory: base_directory)
+      end&.reduce(:deep_merge)
+    end
 
-    def self.load!(template_file)
-      new(template_file).load!
+    def self.load!(template_file_path, runtime_config: {}, base_directory: '.')
+      new(template_file_path, runtime_config: runtime_config, base_directory: base_directory).load!
     end
 
-    def initialize(template_file)
-      @template_file = template_file
+    def initialize(template_file_path, runtime_config: {}, base_directory: '.')
+      @base_directory = Pathname.new(base_directory)
+      @template_file = @base_directory.join(template_file_path)
+
+      file_not_found!(@template_file) unless @template_file.exist?
+
+      @template_file_content = Djin.cache.fetch(@template_file.realpath.to_s) { @template_file.read }
+      @runtime_config = runtime_config
     end
 
     def load!
       validate_version!
+      validate_missing_config!
 
       file_config
     end
@@ -25,52 +43,38 @@ module Djin
     private
 
     def file_config
-      FileConfig.new(
+      MainConfig.new(
         djin_version: version,
         variables: variables,
         tasks: tasks,
-        raw_tasks: raw_tasks
+        raw_tasks: raw_tasks,
+        include_configs: @include_configs || []
       )
     end
 
-    def raw_djin_config
-      @raw_djin_config ||= yaml_load(@template_file)
-    rescue Psych::SyntaxError => e
-      raise Interpreter::InvalidConfigFileError, e.message
-    end
-
-    def rendered_djin_config
-      @rendered_djin_config ||= begin
-                                  locals = env.merge(variables)
-
-                                  rendered_yaml = Mustache.render(@template_file,
-                                                                  args: args.join(' '),
-                                                                  args?: args.any?,
-                                                                  **locals)
-                                  yaml_load(rendered_yaml)
-                                end
-    end
-
     def version
       # TODO: Deprecates djin_version and use version instead
       @version || raw_djin_config['djin_version']
     end
 
     def variables
-      @variables ||= raw_djin_config['variables']&.symbolize_keys || {}
+      @variables ||= included_variables.merge(raw_djin_config['variables']&.symbolize_keys || {})
     end
 
     def tasks
-      rendered_djin_config['tasks'] || legacy_tasks
+      included_tasks.merge(rendered_djin_config['tasks'] || legacy_tasks)
     end
 
     def raw_tasks
-      raw_djin_config['tasks'] || legacy_raw_tasks
+      included_raw_tasks.merge(raw_djin_config['tasks'] || legacy_raw_tasks)
     end
 
     def legacy_tasks
-      warn '[DEPRECATED] Root tasks are deprecated and will be removed in Djin 1.0.0,' \
-           ' put the tasks under \'tasks\' keyword'
+      Djin.warn_once(
+        'Root tasks are deprecated and will be removed in Djin 1.0.0,' \
+        ' put the tasks under \'tasks\' keyword',
+        type: 'DEPRECATED'
+      )
 
       rendered_djin_config.except(*RESERVED_WORDS).reject { |task| task.start_with?('_') }
     end
@@ -79,6 +83,57 @@ module Djin
       raw_djin_config.except(*RESERVED_WORDS).reject { |task| task.start_with?('_') }
     end
 
+    def included_variables
+      return {} unless included_config
+
+      included_config.variables
+    end
+
+    def included_tasks
+      return {} unless included_config
+
+      included_config.tasks
+    end
+
+    def included_raw_tasks
+      return {} unless included_config
+
+      included_config.raw_tasks
+    end
+
+    # TODO: Rename method
+    def included_config
+      @included_config ||= begin
+                             present_include_configs&.map do |present_include|
+                               ConfigLoader.load!(present_include.file, base_directory: @template_file.dirname,
+                                                                        # TODO: Rename to context_config
+                                                                        runtime_config: present_include.context)
+                             end&.reduce(:deep_merge)
+                           end
+    end
+
+    def present_include_configs
+      include_configs&.select(&:present?)
+    end
+
+    def missing_include_configs
+      include_configs&.select(&:missing?)
+    end
+
+    # TODO: Refactor to move include methods to a specific IncludeConfigLoader, maybe rename IncludeResolver
+    def include_configs
+      @include_configs ||= begin
+                             # TODO: Rename the resolved variables
+                             resolver = IncludeResolver.new(base_directory: @template_file.dirname)
+
+                             include_djin_config = raw_djin_config['include'] || []
+
+                             include_djin_config.map do |include_config|
+                               resolver.call(include_config)
+                             end
+                           end
+    end
+
     def args
       index = ARGV.index('--')
 
@@ -91,6 +146,24 @@ module Djin
       @env ||= ENV.to_h.symbolize_keys
     end
 
+    def raw_djin_config
+      @raw_djin_config ||= yaml_load(@template_file_content).deep_merge(@runtime_config)
+    rescue Psych::SyntaxError => e
+      raise Interpreter::InvalidConfigFileError, "File: #{@template_file.realpath}\n  #{e.message}"
+    end
+
+    def rendered_djin_config
+      @rendered_djin_config ||= begin
+                                  locals = env.merge(variables)
+
+                                  rendered_yaml = Mustache.render(@template_file_content,
+                                                                  args: args.join(' '),
+                                                                  args?: args.any?,
+                                                                  **locals)
+                                  yaml_load(rendered_yaml).merge(@runtime_config)
+                                end
+    end
+
     def yaml_load(text)
       YAML.safe_load(text, [], [], true)
     end
@@ -102,5 +175,26 @@ module Djin
 
       raise Interpreter::VersionNotSupportedError, "Version #{version} is not supported, use #{Djin::VERSION} or higher"
     end
+
+    def validate_missing_config!
+      missing_include_configs.each do |ic|
+        file_not_found!(ic.full_path) if ic.type == :local
+
+        missing_file_remote_error = "#{ic.git} exists but is missing %s," \
+          'if the file exists in upstream run djin remote-config fetch to fix'
+
+        file_not_found!(ic.full_path, missing_file_remote_error) if ic.type == :remote && ic.repository_fetched?
+
+        if ic.type == :remote
+          Djin.warn_once "Missing #{ic.git} with version '#{ic.version}', " \
+            'run `djin remote-config fetch` to fetch the config'
+        end
+      end
+    end
+
+    def file_not_found!(filename, message = "File '%s' not found")
+      raise FileNotFoundError, message % filename
+    end
   end
+  # rubocop:enable Metrics/ClassLength
 end