dapp 0.0.24 → 0.1.0

data/lib/dapp/atomizer.rb

@@ -1,56 +0,0 @@
-module Dapp
-  # "Transaction" journal with rollback (mainly to protect cache fill with unbuildable configuration)
-  class Atomizer
-    def initialize(builder, file_path, lock_timeout: 10)
-      @builder = builder
-      @file_path = file_path
-      @lock_timeout = lock_timeout
-      @file = open
-
-      builder.register_atomizer self
-    end
-
-    def <<(path)
-      file.puts path
-      file.fsync
-    end
-
-    def commit!
-      @file.truncate(0)
-      @file.close
-      @file = open
-    end
-
-    protected
-
-    attr_reader :file_path
-    attr_reader :builder
-
-    attr_reader :lock_timeout
-    attr_reader :file
-
-    def open
-      file = File.open(file_path, File::RDWR | File::CREAT, 0644)
-
-      file.sync = true
-
-      Timeout.timeout(lock_timeout) do
-        file.flock(File::LOCK_EX)
-      end
-
-      if (not_commited_paths = file.read.lines.map(&:strip))
-        FileUtils.rm_rf not_commited_paths
-      end
-
-      file.truncate(0)
-      file.rewind
-
-      file
-    rescue Timeout::Error
-      file.close
-
-      STDERR.puts 'Atomizer already in use! Try again later.'
-      exit 1
-    end
-  end
-end

data/lib/dapp/builder.rb

@@ -1,230 +0,0 @@
-module Dapp
-  # Main class that does all stuff
-  class Builder
-    include Chefify
-    include Centos7
-    include Ubuntu1404
-    include Ubuntu1604
-    include CiTagging
-    include ManualTagging
-    include GitTagging
-    include CascadeTagging
-    include Filelock
-
-    class << self
-      def default_opts
-        @default_opts ||= {}
-      end
-
-      def dappfiles_paths(path, pattern = '*')
-        pattern.split('-').instance_eval { count.downto(1).map { |n| slice(0, n).join('-') } }
-               .map { |p| Dir.glob(File.join([path, p, default_opts[:dappfile_name] || 'Dappfile'].compact)) }.find(&:any?) || []
-      end
-
-      def process_directory(path, pattern = '*')
-        dappfiles_paths(path, pattern).map { |dappfile_path| process_file(dappfile_path, app_filter: pattern).builded_apps }.flatten
-      end
-
-      def process_file(dappfile_path, app_filter: '*')
-        new(dappfile_path: dappfile_path, app_filter: app_filter) do |builder|
-          builder.log "Processing application #{builder.name} (#{dappfile_path})"
-
-          # indent all subsequent messages
-          builder.indent_log
-
-          # eval instructions from file
-          builder.instance_eval File.read(dappfile_path), dappfile_path
-
-          # commit atomizers
-          builder.commit_atomizers!
-        end
-      end
-    end
-
-    def log(message)
-      puts ' ' * opts[:log_indent] + ' * ' + message if opts[:log_verbose] || !opts[:log_quiet]
-    end
-
-    def shellout(*args, log_verbose: false, **kwargs)
-      kwargs[:live_stream] = STDOUT if log_verbose && opts[:log_verbose]
-      Mixlib::ShellOut.new(*args, :timeout => 3600, **kwargs).run_command.tap(&:error!)
-    end
-
-    def home_path(*path)
-      path.compact.inject(Pathname.new(opts[:home_path]), &:+).expand_path.to_s
-    end
-
-    def build_path(*path)
-      path.compact.inject(Pathname.new(opts[:build_path]), &:+).expand_path.tap do |p|
-        FileUtils.mkdir_p p.parent
-      end.to_s
-    end
-
-    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-    def initialize(**options)
-      opts.merge! self.class.default_opts
-      opts.merge! options
-
-      # default log indentation
-      opts[:log_indent] = 0
-
-      # basename
-      if opts[:name]
-        opts[:basename] = opts[:name]
-        opts[:name] = nil
-      elsif opts[:dappfile_path]
-        opts[:basename] ||= Pathname.new(opts[:dappfile_path]).expand_path.parent.basename
-      end
-
-      # home path
-      opts[:home_path] ||= Pathname.new(opts[:dappfile_path] || 'fakedir').parent.expand_path.to_s
-
-      # build path
-      opts[:build_path] = opts[:build_dir] ? opts[:build_dir] : home_path('build')
-      opts[:build_path] = build_path opts[:basename] if opts[:shared_build_dir]
-
-      # home branch
-      @home_branch = shellout("git -C #{home_path} rev-parse --abbrev-ref HEAD").stdout.strip
-
-      # atomizers
-      @atomizers = []
-
-      # account builded apps
-      @builded_apps = []
-
-      lock do
-        yield self
-      end
-    end
-    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
-
-    def indent_log
-      opts[:log_indent] += 1
-    end
-
-    attr_reader :home_branch
-
-    def builded_apps
-      @builded_apps.dup
-    end
-
-    def docker
-      docker_stack.last
-    end
-
-    def scope(&blk)
-      stack_settings(&blk)
-    end
-
-    def app(name)
-      log "Processing #{self.name}-#{name}"
-
-      name = "#{opts[:name]}-#{name}" if opts[:name]
-
-      stack_settings name: name, log_indent: opts[:log_indent] + 1 do
-        docker.name name
-
-        yield
-      end
-    end
-
-    def name
-      [opts[:basename], opts[:name]].compact.join '-'
-    end
-
-    def add_git_artifact(where_to_add, **kwargs)
-      log "Adding artifact (to #{where_to_add})"
-
-      # own repo
-      repo = GitRepo::Own.new(self)
-
-      # add artifact
-      artifact = GitArtifact.new(self, repo, where_to_add, flush_cache: opts[:flush_cache], branch: home_branch, **kwargs)
-      artifact.add_multilayer!
-    end
-
-    def add_remote_git_artifact(url, where_to_add, branch: opts[:git_artifact_branch] || home_branch, ssh_key_path: nil, **kwargs)
-      log "Adding artifact from git (#{url} to #{where_to_add}, branch: #{branch})"
-
-      # extract git repo name from url
-      repo_name = url.gsub(%r{.*?([^\/ ]+)\.git}, '\\1')
-
-      # clone or fetch
-      repo = GitRepo::Remote.new(self, repo_name, url: url, ssh_key_path: ssh_key_path)
-      repo.fetch!(branch)
-
-      # add artifact
-      artifact = GitArtifact.new(self, repo, where_to_add, flush_cache: opts[:flush_cache], branch: branch, **kwargs)
-      artifact.add_multilayer!
-    end
-
-    # rubocop:disable Metrics/AbcSize
-    def build(**_kwargs)
-      # check app name
-      unless !opts[:app_filter] || File.fnmatch("#{opts[:app_filter]}*", name)
-        log "Skipped (does not match filter: #{opts[:app_filter]})!"
-        return false
-      end
-
-      # build image
-      log 'Building'
-      image_id = docker.build
-
-      # apply tagging
-      %w(ci manual git cascade).each do |strategy|
-        send "tag_#{strategy}", image_id
-      end
-
-      # push to repo
-      if opts[:docker_repo]
-        log 'Pushing to docker repo'
-        docker.push name: name, repo: opts[:docker_repo]
-      end
-
-      # count it
-      @builded_apps << name
-
-      image_id
-    end
-    # rubocop:enable Metrics/AbcSize
-
-    def register_atomizer(atomizer)
-      atomizers << atomizer
-    end
-
-    def commit_atomizers!
-      atomizers.each(&:commit!)
-    end
-
-    protected
-
-    attr_reader :atomizers
-
-    def opts
-      opts_stack.last
-    end
-
-    def opts_stack
-      @opts_stack ||= [{}]
-    end
-
-    def docker_stack
-      @docker_stack ||= [Docker.new(self)]
-    end
-
-    def stack_settings(**options)
-      opts_stack.push opts.merge(options).dup
-      docker_stack.push docker.dup
-
-      yield
-    ensure
-      docker_stack.pop
-      opts_stack.pop
-    end
-
-    def lock(**kwargs, &block)
-      filelock(build_path("#{home_branch}.lock"), error_message: "Application #{opts[:basename]} (#{home_branch}) in use! Try again later.", **kwargs,
-               &block)
-    end
-  end
-end

data/lib/dapp/builder/cascade_tagging.rb

@@ -1,48 +0,0 @@
-module Dapp
-  class Builder
-    # Cascade tagging strategy
-    module CascadeTagging
-      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-      def tag_cascade(image_id)
-        return unless opts[:tag_cascade]
-
-        log 'Applying cascade tagging'
-
-        opts[:build_history_length] ||= 10
-
-        spec = {
-          name: name,
-          tag: home_branch,
-          repo: opts[:docker_repo]
-        }
-
-        # return if nothing changed
-        return if image_id == docker.image_id(spec)
-
-        # remove excess tags
-        tags_to_remove = docker.images(name: spec[:name], repo: spec[:repo])
-        tags_to_remove.map! { |image| image[:tag] }
-        tags_to_remove.select! { |tag| tag.start_with?("#{spec[:tag]}_") && tag.sub(/^#{spec[:tag]}_/, '').to_i >= opts[:build_history_length] }
-        tags_to_remove.each do |tag_to_remove|
-          docker.rmi spec.merge(tag: tag_to_remove)
-        end
-
-        # shift old images: 1 -> 2, 2 -> 3, ..., n -> n+1
-        (opts[:build_history_length] - 1).downto(1).each do |n|
-          origin = spec.merge(tag: "#{spec[:tag]}_#{n}")
-
-          if docker.image_exist?(**origin)
-            docker.tag origin, spec.merge(tag: "#{spec[:tag]}_#{n + 1}")
-          end
-        end
-
-        # shift top -> 1
-        docker.tag spec, spec.merge(tag: "#{spec[:tag]}_1") if docker.image_exist?(**spec)
-
-        # tag top
-        docker.tag image_id, spec
-      end
-      # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
-    end
-  end
-end

data/lib/dapp/builder/centos7.rb

@@ -1,47 +0,0 @@
-module Dapp
-  class Builder
-    # Centos7 support
-    module Centos7
-      # rubocop:disable Metrics/MethodLength
-      def from_centos7
-        # use centos7
-        docker.from 'centos:7'
-
-        # add real systemd
-        docker.env container: 'docker', step: :begining
-        docker.run(
-          'yum -y update; yum clean all',
-          '(cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done)',
-          'rm -f /lib/systemd/system/multi-user.target.wants/*',
-          'rm -f /etc/systemd/system/*.wants/*',
-          'rm -f /lib/systemd/system/local-fs.target.wants/*',
-          'rm -f /lib/systemd/system/sockets.target.wants/*udev*',
-          'rm -f /lib/systemd/system/sockets.target.wants/*initctl*',
-          'rm -f /lib/systemd/system/basic.target.wants/*',
-          'rm -f /lib/systemd/system/anaconda.target.wants/*',
-          'rm -f /lib/systemd/system/systemd-remount-fs.service',
-          'rm -f /lib/systemd/system/sys-fs-fuse-connections.mount',
-          step: :begining
-        )
-        docker.volume '/sys/fs/cgroup', step: :begining
-        docker.cmd '/usr/sbin/init', step: :begining
-
-        # cache yum fastestmirror
-        docker.run 'yum makecache', step: :begining
-
-        # TERM & utf8
-        docker.run 'localedef -c -f UTF-8 -i en_US en_US.UTF-8', step: :begining
-        docker.env TERM: 'xterm', LANG: 'en_US.UTF-8', LANGUAGE: 'en_US:en', LC_ALL: 'en_US.UTF-8', step: :begining
-
-        # centos hacks
-        docker.run(
-          'sed \'s/\(-\?session\s\+optional\s\+pam_systemd\.so.*\)/#\1/g\' -i /etc/pam.d/system-auth',
-          'yum install -y sudo git',
-          'echo \'Defaults:root !requiretty\' >> /etc/sudoers',
-          step: :begining
-        )
-      end
-      # rubocop:enable Metrics/MethodLength
-    end
-  end
-end

data/lib/dapp/builder/chefify.rb

@@ -1,107 +0,0 @@
-module Dapp
-  class Builder
-    # Build using chef "dapp" cookbooks
-    module Chefify
-      def dappit(*extra_dapps, chef_version: '12.5.1', **_kwargs)
-        log 'Adding dapp chef cookbook artifact and chef solo run'
-
-        setup_dapp_chef chef_version
-
-        # run chef solo
-        [:prepare, :build, :setup].each do |step|
-          # run chef-solo for extra dapps
-          extra_dapps.each do |extra_dapp|
-            if dapp_chef_cookbooks_artifact.exist_in_step? "cookbooks/mdapp-#{extra_dapp}/recipes/#{step}.rb", step
-              docker.run "chef-solo -c /usr/share/dapp/chef_solo.rb -o mdapp-#{extra_dapp}::#{step},dapp-#{opts[:basename]}::void", step: step
-            end
-          end
-
-          # run chef-solo for app
-          recipe = [opts[:name], step].compact.join '-'
-          if dapp_chef_cookbooks_artifact.exist_in_step? "cookbooks/dapp-#{opts[:basename]}/recipes/#{recipe}.rb", step
-            docker.run "chef-solo -c /usr/share/dapp/chef_solo.rb -o dapp-#{opts[:basename]}::#{recipe}", step: step
-          end
-        end
-      end
-
-      def build_dapp(*args, extra_dapps: [], **kwargs, &blk)
-        stack_settings do
-          dappit(*extra_dapps, **kwargs)
-
-          build(*args, **kwargs, &blk)
-        end
-      end
-
-      protected
-
-      def dapp_chef_cookbooks_artifact
-        unless @dapp_chef_cookbooks_artifact
-          # init cronicler
-          repo = GitRepo::Chronicler.new(self, 'dapp_cookbooks', build_path: home_branch)
-
-          # warmup berks cache
-          Dir.mktmpdir('dapp_berks_warmup') do |tmpdir_path|
-            FileUtils.cp home_path('Berksfile'), tmpdir_path
-            FileUtils.cp home_path('metadata.rb'), tmpdir_path
-            shellout 'berks install', cwd: tmpdir_path
-          end
-
-          # verify berks lock
-          shellout "berks verify --berksfile=#{home_path 'Berksfile'}"
-
-          # vendor cookbooks
-          shellout "berks vendor --berksfile=#{home_path 'Berksfile'} #{repo.chronodir_path 'cookbooks'}", log_verbose: true
-
-          # create void receipt
-          FileUtils.touch repo.chronodir_path 'cookbooks', "dapp-#{opts[:basename]}", 'recipes', 'void.rb'
-
-          # commit (if smth changed)
-          repo.commit!
-
-          # init artifact
-          @dapp_chef_cookbooks_artifact = GitArtifact.new(self, repo, '/usr/share/dapp/chef_repo/cookbooks',
-                                                          cwd: 'cookbooks', build_path: home_branch, flush_cache: opts[:flush_cache])
-        end
-
-        @dapp_chef_cookbooks_artifact
-      end
-
-      def install_chef_and_setup_chef_solo(chef_version)
-        docker.run(
-          "curl -L https://www.opscode.com/chef/install.sh | bash -s -- -v #{chef_version}",
-          'mkdir -p /usr/share/dapp/chef_repo /var/cache/dapp/chef',
-          'echo file_cache_path \\"/var/cache/dapp/chef\\" > /usr/share/dapp/chef_solo.rb',
-          'echo cookbook_path \\"/usr/share/dapp/chef_repo/cookbooks\\" >> /usr/share/dapp/chef_solo.rb',
-          step: :begining
-        )
-      end
-
-      def run_chef_solo_for_mdapp_common
-        [:prepare, :build, :setup].each do |step|
-          if dapp_chef_cookbooks_artifact.exist_in_step? "cookbooks/mdapp-common/recipes/#{step}.rb", step
-            docker.run "chef-solo -c /usr/share/dapp/chef_solo.rb -o mdapp-common::#{step},dapp-#{opts[:basename]}::void", step: step
-          end
-        end
-      end
-
-      def setup_dapp_chef(chef_version)
-        if opts[:dapp_chef_version]
-          raise "dapp chef version mismatch, version #{opts[:dapp_chef_version]} already installed" if opts[:dapp_chef_version] != chef_version
-          return
-        end
-
-        # install chef, setup chef_solo
-        install_chef_and_setup_chef_solo(chef_version)
-
-        # add cookbooks
-        dapp_chef_cookbooks_artifact.add_multilayer!
-
-        # mark chef as installed
-        opts[:dapp_chef_version] = chef_version
-
-        # run chef solo for mdapp-common
-        run_chef_solo_for_mdapp_common
-      end
-    end
-  end
-end