dapp 0.0.1

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    YTQzOThlMTg4MDA3YzQ0MjE4ODgyZjgyMzQ4M2RiMTgzMmRlN2ZjZg==
+  data.tar.gz: !binary |-
+    ODQ2ODVjOTY3OTE0M2I1NzQ4MjU1ZDczODE3MTFmNDJmYTI4NDQ2OQ==
+SHA512:
+  metadata.gz: !binary |-
+    YzU1NWRjMDAyOWNjMGRiN2RkZGQyMmFhYzhlODQwYTc1ZGJiMzI0MDJkODkw
+    Zjk5YTA2YTg1MDlmN2M0N2E5MWM1YmVhNmVmZTQ1MTlhMDRkY2IxOGI5OGQ1
+    NTRkNzU0OTU4ZDM3ZDVlNjE3Y2U1N2VhODk3ODA0NThiYTE2Yzc=
+  data.tar.gz: !binary |-
+    ZGQ1ZTQ2ZGJmMmEwMWZjYWQ3MDYxMjg5MGNiNGQyNDc5NmQ0YWFmMWU3NmUx
+    MjkyMjllNjczNDAzMzZkYzNiY2E3YTk5MjVkODMyY2Q2NDZlMTVlMDIxZTI3
+    Y2U0YzBmNDIxOGQwYzE2N2EyMTY5MjdmZTFlYTI0NjUyZTgxMjU=

data/bin/dapp

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+require 'rubygems'
+require 'dapp'
+
+Dapp::CLI.new.run

data/lib/dapp.rb

@@ -0,0 +1,21 @@
+require 'pathname'
+require 'fileutils'
+require 'tmpdir'
+require 'digest'
+require 'timeout'
+require 'base64'
+require 'mixlib/shellout'
+
+require 'dapp/version'
+require 'dapp/cli'
+require 'dapp/builder/chefify'
+require 'dapp/builder/centos7'
+require 'dapp/builder/cascade_tagging'
+require 'dapp/filelock'
+require 'dapp/builder'
+require 'dapp/docker'
+require 'dapp/atomizer'
+require 'dapp/git_repo/base'
+require 'dapp/git_repo/chronicler'
+require 'dapp/git_repo/remote'
+require 'dapp/git_artifact'

data/lib/dapp/atomizer.rb

@@ -0,0 +1,53 @@
+module Dapp
+  # "Transaction" journal with rollback (mainly to protect cache fill with unbuildable configuration)
+  class Atomizer
+    def initialize(builder, file_path)
+      @builder = builder
+      @file_path = file_path
+      @file = open
+
+      builder.register_atomizer self
+    end
+
+    def <<(path)
+      file.puts path
+    end
+
+    def commit!
+      @file.truncate(0)
+      @file.close
+      @file = open
+    end
+
+    protected
+
+    attr_reader :file_path
+    attr_reader :builder
+
+    attr_reader :file
+
+    def open
+      file = File.open(file_path, File::RDWR | File::CREAT, 0644)
+
+      file.sync = true
+
+      Timeout.timeout(10) do
+        file.flock(File::LOCK_EX)
+      end
+
+      if (not_commited_paths = file.read.lines.map(&:strip))
+        FileUtils.rm_rf not_commited_paths
+      end
+
+      file.truncate(0)
+      file.rewind
+
+      file
+    rescue Timeout::Error
+      file.close
+
+      STDERR.puts 'Atomizer already in use! Try again later.'
+      exit 1
+    end
+  end
+end

data/lib/dapp/builder.rb

@@ -0,0 +1,217 @@
+module Dapp
+  # Main class that does all stuff
+  class Builder
+    include Chefify
+    include Centos7
+    include CascadeTagging
+    include Filelock
+
+    class << self
+      def default_opts
+        @default_opts ||= {}
+      end
+
+      def dappfiles_paths(path, pattern = '*')
+        pattern.split('-').instance_eval { count.downto(1).map { |n| slice(0, n).join('-') } }
+               .map { |p| Dir.glob(File.join([path, p, default_opts[:dappfile_name] || 'Dappfile'].compact)) }.find(&:any?) || []
+      end
+
+      def process_directory(path, pattern = '*')
+        dappfiles_paths(path, pattern).map { |dappfile_path| process_file(dappfile_path, app_filter: pattern).builded_apps }.flatten
+      end
+
+      def process_file(dappfile_path, app_filter: '*')
+        new(dappfile_path: dappfile_path, app_filter: app_filter) do |builder|
+          builder.log "Processing application #{builder.name} (#{dappfile_path})"
+
+          # indent all subsequent messages
+          builder.indent_log
+
+          # eval instructions from file
+          builder.instance_eval File.read(dappfile_path), dappfile_path
+
+          # commit atomizers
+          builder.commit_atomizers!
+        end
+      end
+    end
+
+    def log(message)
+      puts ' ' * opts[:log_indent] + ' * ' + message if opts[:log_verbose] || !opts[:log_quiet]
+    end
+
+    def shellout(*args, log_verbose: false, **kwargs)
+      kwargs[:live_stream] = STDOUT if log_verbose && opts[:log_verbose]
+      Mixlib::ShellOut.new(*args, :timeout => 3600, **kwargs).run_command.tap(&:error!)
+    end
+
+    def home_path(*path)
+      path.compact.inject(Pathname.new(opts[:home_path]), &:+).expand_path.to_s
+    end
+
+    def build_path(*path)
+      path.compact.inject(Pathname.new(opts[:build_path]), &:+).expand_path.tap do |p|
+        FileUtils.mkdir_p p.parent
+      end.to_s
+    end
+
+    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+    def initialize(**options)
+      opts.merge! self.class.default_opts
+      opts.merge! options
+
+      # default log indentation
+      opts[:log_indent] = 0
+
+      # basename
+      if opts[:name]
+        opts[:basename] = opts[:name]
+        opts[:name] = nil
+      elsif opts[:dappfile_path]
+        opts[:basename] ||= Pathname.new(opts[:dappfile_path]).expand_path.parent.basename
+      end
+
+      # home path
+      opts[:home_path] ||= Pathname.new(opts[:dappfile_path] || 'fakedir').parent.expand_path.to_s
+
+      # build path
+      opts[:build_path] = opts[:build_dir] ? opts[:build_dir] : home_path('build')
+      opts[:build_path] = build_path opts[:basename] if opts[:shared_build_dir]
+
+      # home branch
+      @home_branch = shellout("git -C #{home_path} rev-parse --abbrev-ref HEAD").stdout.strip
+
+      # atomizers
+      @atomizers = []
+
+      # account builded apps
+      @builded_apps = []
+
+      lock do
+        yield self
+      end
+    end
+    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+
+    def indent_log
+      opts[:log_indent] += 1
+    end
+
+    attr_reader :home_branch
+
+    def builded_apps
+      @builded_apps.dup
+    end
+
+    def docker
+      docker_stack.last
+    end
+
+    def scope(&blk)
+      stack_settings(&blk)
+    end
+
+    def app(name)
+      log "Processing #{self.name}-#{name}"
+
+      name = "#{opts[:name]}-#{name}" if opts[:name]
+
+      stack_settings name: name, log_indent: opts[:log_indent] + 1 do
+        docker.name name
+
+        yield
+      end
+    end
+
+    def name
+      [opts[:basename], opts[:name]].compact.join '-'
+    end
+
+    def add_artifact_from_git(url, where_to_add, branch: opts[:git_artifact_branch], ssh_key_path: nil, **kwargs)
+      log "Adding artifact from git (#{url} to #{where_to_add}, branch: #{branch})"
+
+      # extract git repo name from url
+      repo_name = url.gsub(%r{.*?([^\/ ]+)\.git}, '\\1')
+
+      # clone or fetch
+      repo = GitRepo::Remote.new(self, repo_name, url: url, ssh_key_path: ssh_key_path)
+      repo.fetch!(branch)
+
+      # add artifact
+      artifact = GitArtifact.new(self, repo, where_to_add, flush_cache: opts[:flush_cache], branch: branch, **kwargs)
+      artifact.add_multilayer!
+    end
+
+    def build(**_kwargs)
+      # check app name
+      unless !opts[:app_filter] || File.fnmatch("#{opts[:app_filter]}*", name)
+        log "Skipped (does not match filter: #{opts[:app_filter]})!"
+        return false
+      end
+
+      # build image
+      log 'Building'
+      image_id = docker.build
+
+      # apply cascade tagging
+      tag_cascade image_id
+
+      # push to registry
+      if opts[:docker_registry]
+        log 'Pushing to registry'
+        docker.push name: name, registry: opts[:docker_registry]
+      end
+
+      # count it
+      @builded_apps << name
+
+      image_id
+    end
+
+    def tag(image_id, name: nil, tag: nil, registry: nil)
+      return unless name && tag
+
+      new = { name: name, tag: tag, registry: registry }
+      docker.tag image_id, new
+    end
+
+    def register_atomizer(atomizer)
+      atomizers << atomizer
+    end
+
+    def commit_atomizers!
+      atomizers.each(&:commit!)
+    end
+
+    protected
+
+    attr_reader :atomizers
+
+    def opts
+      opts_stack.last
+    end
+
+    def opts_stack
+      @opts_stack ||= [{}]
+    end
+
+    def docker_stack
+      @docker_stack ||= [Docker.new(self)]
+    end
+
+    def stack_settings(**options)
+      opts_stack.push opts.merge(options).dup
+      docker_stack.push docker.dup
+
+      yield
+    ensure
+      docker_stack.pop
+      opts_stack.pop
+    end
+
+    def lock(**kwargs, &block)
+      filelock(build_path("#{home_branch}.lock"), error_message: "Application #{opts[:basename]} (#{home_branch}) in use! Try again later.", **kwargs,
+               &block)
+    end
+  end
+end

data/lib/dapp/builder/cascade_tagging.rb

@@ -0,0 +1,48 @@
+module Dapp
+  class Builder
+    # Cascade tagging strategy
+    module CascadeTagging
+      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      def tag_cascade(image_id)
+        return unless opts[:cascade_tagging]
+
+        log 'Applying cascade tagging'
+
+        opts[:build_history_length] ||= 10
+
+        i = {
+          name: name,
+          tag: home_branch,
+          registry: opts[:docker_registry]
+        }
+
+        # return if nothing changed
+        return if image_id == docker.image_id(i)
+
+        # remove excess tags
+        tags_to_remove = docker.images(name: i[:name], registry: i[:registry])
+                               .map { |image| image[:tag] }
+                               .select { |tag| tag.start_with?("#{i[:tag]}_") && tag.sub(/^#{i[:tag]}_/, '').to_i >= opts[:build_history_length] }
+        tags_to_remove.each do |tag_to_remove|
+          docker.rmi i.merge(tag: tag_to_remove)
+        end
+
+        # shift old images: 1 -> 2, 2 -> 3, ..., n -> n+1
+        (opts[:build_history_length] - 1).downto(1).each do |n|
+          origin = i.merge(tag: "#{i[:tag]}_#{n}")
+
+          if docker.image_exists?(**origin)
+            docker.tag origin, i.merge(tag: "#{i[:tag]}_#{n + 1}")
+          end
+        end
+
+        # shift top -> 1
+        docker.tag i, i.merge(tag: "#{i[:tag]}_1") if docker.image_exists?(**i)
+
+        # tag top
+        docker.tag image_id, i
+      end
+      # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+    end
+  end
+end

data/lib/dapp/builder/centos7.rb

@@ -0,0 +1,46 @@
+module Dapp
+  class Builder
+    # Centos7 support
+    module Centos7
+      # rubocop:disable Metrics/MethodLength:
+      def from_centos7
+        # use centos7
+        docker.from 'centos:7'
+
+        # add real systemd
+        docker.env container: 'docker', step: :begining
+        docker.run 'yum -y swap -- remove systemd-container systemd-container-libs -- install systemd systemd-libs', step: :begining
+        docker.run(
+          'yum -y update; yum clean all',
+          '(cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done)',
+          'rm -f /lib/systemd/system/multi-user.target.wants/*',
+          'rm -f /etc/systemd/system/*.wants/*',
+          'rm -f /lib/systemd/system/local-fs.target.wants/*',
+          'rm -f /lib/systemd/system/sockets.target.wants/*udev*',
+          'rm -f /lib/systemd/system/sockets.target.wants/*initctl*',
+          'rm -f /lib/systemd/system/basic.target.wants/*',
+          'rm -f /lib/systemd/system/anaconda.target.wants/*',
+          step: :begining
+        )
+        docker.volume '/sys/fs/cgroup', step: :begining
+        docker.cmd '/usr/sbin/init', step: :begining
+
+        # cache yum fastestmirror
+        docker.run 'yum makecache', step: :begining
+
+        # TERM & utf8
+        docker.run 'localedef -c -f UTF-8 -i en_US en_US.UTF-8', step: :begining
+        docker.env TERM: 'xterm', LANG: 'en_US.UTF-8', LANGUAGE: 'en_US:en', LC_ALL: 'en_US.UTF-8', step: :begining
+
+        # centos hacks
+        docker.run(
+          'sed \'s/\(-\?session\s\+optional\s\+pam_systemd\.so.*\)/#\1/g\' -i /etc/pam.d/system-auth',
+          'yum install -y sudo git',
+          'echo \'Defaults:root !requiretty\' >> /etc/sudoers',
+          step: :begining
+        )
+      end
+      # rubocop:enable Metrics/MethodLength
+    end
+  end
+end

data/lib/dapp/builder/chefify.rb

@@ -0,0 +1,101 @@
+module Dapp
+  class Builder
+    # Build using chef "dapp" cookbooks
+    module Chefify
+      def dappit(*extra_dapps, chef_version: '12.4.3', **_kwargs)
+        log 'Adding dapp chef cookbook artifact and chef solo run'
+
+        setup_dapp_chef chef_version
+
+        # run chef solo
+        [:prepare, :build, :setup].each do |step|
+          # run chef-solo for extra dapps
+          extra_dapps.each do |extra_dapp|
+            if dapp_chef_cookbooks_artifact.exists_in_step? "cookbooks/#{extra_dapp}/recipes/#{step}.rb", step
+              # FIXME: env ???
+              docker.run "chef-solo -c /usr/share/dapp/chef_solo.rb -o #{extra_dapp}::#{step},env-#{opts[:basename]}::void", step: step
+            end
+          end
+
+          # run chef-solo for app
+          recipe = [opts[:name], step].compact.join '-'
+          # FIXME: env ???
+          if dapp_chef_cookbooks_artifact.exists_in_step? "cookbooks/env-#{opts[:basename]}/recipes/#{recipe}.rb", step
+            docker.run "chef-solo -c /usr/share/dapp/chef_solo.rb -o env-#{opts[:basename]}::#{recipe}", step: step
+          end
+        end
+      end
+
+      def build_dapp(*args, extra_dapps: [], **kwargs, &blk)
+        stack_settings do
+          dappit(*extra_dapps, **kwargs)
+
+          build(*args, **kwargs, &blk)
+        end
+      end
+
+      protected
+
+      def dapp_chef_cookbooks_artifact
+        unless @dapp_chef_cookbooks_artifact
+          # init cronicler
+          repo = GitRepo::Chronicler.new(self, 'dapp_cookbooks', build_path: home_branch)
+
+          # vendor cookbooks
+          shellout "berks vendor --berksfile=#{home_path 'Berksfile'} #{repo.chronodir_path 'cookbooks'}", log_verbose: true
+
+          # create void receipt
+          # FIXME: env ???
+          FileUtils.touch repo.chronodir_path 'cookbooks', "env-#{opts[:basename]}", 'recipes', 'void.rb'
+
+          # commit (if smth changed)
+          repo.commit!
+
+          # init artifact
+          @dapp_chef_cookbooks_artifact = GitArtifact.new(self, repo, '/usr/share/dapp/chef_repo/cookbooks',
+                                                          cwd: 'cookbooks', build_path: home_branch, flush_cache: opts[:flush_cache])
+        end
+
+        @dapp_chef_cookbooks_artifact
+      end
+
+      def install_chef_and_setup_chef_solo(chef_version)
+        docker.run(
+          "curl -L https://www.opscode.com/chef/install.sh | bash -s -- -v #{chef_version}",
+          'mkdir -p /usr/share/dapp/chef_repo /var/cache/dapp/chef',
+          'echo file_cache_path \\"/var/cache/dapp/chef\\" > /usr/share/dapp/chef_solo.rb',
+          'echo cookbook_path \\"/usr/share/dapp/chef_repo/cookbooks\\" >> /usr/share/dapp/chef_solo.rb',
+          step: :begining
+        )
+      end
+
+      def run_chef_solo_for_dapp_common
+        [:prepare, :build, :setup].each do |step|
+          if dapp_chef_cookbooks_artifact.exists_in_step? "cookbooks/dapp-common/recipes/#{step}.rb", step
+            # FIXME: env ???
+            docker.run "chef-solo -c /usr/share/dapp/chef_solo.rb -o dapp-common::#{step},env-#{opts[:basename]}::void", step: step
+          end
+        end
+      end
+
+      def setup_dapp_chef(chef_version)
+        if opts[:dapp_chef_version]
+          raise "dapp chef version mismatch, version #{opts[:dapp_chef_version]} already installed" if opts[:dapp_chef_version] != chef_version
+          return
+        end
+
+        # install chef, setup chef_solo
+        install_chef_and_setup_chef_solo(chef_version)
+
+        # add cookbooks
+        dapp_chef_cookbooks_artifact.add_multilayer!
+
+        # mark chef as installed
+        opts[:dapp_chef_version] = chef_version
+
+        # run chef solo for dapp-common
+        run_chef_solo_for_dapp_common
+      end
+    end
+  end
+end