dapp 0.0.1

data/lib/dapp/cli.rb

@@ -0,0 +1,133 @@
+require 'mixlib/cli'
+
+module Dapp
+  # CLI
+  class CLI
+    include Mixlib::CLI
+
+    banner <<BANNER.freeze
+Version: #{Dapp::VERSION}
+
+Usage:
+  dappit [options] [PATTERN ...]
+
+    PATTERN                     Applications to process [default: *].
+
+Options:
+BANNER
+
+    option :version,
+           long: '--version',
+           description: 'Show version',
+           on: :tail,
+           boolean: true,
+           proc: proc { puts "Version: #{Dapp::VERSION}" },
+           exit: 0
+
+    option :quiet,
+           short: '-q',
+           long: '--quiet',
+           description: 'Suppress logging',
+           on: :tail,
+           boolean: true,
+           proc: proc { Dapp::Builder.default_opts[:log_quiet] = true }
+
+    option :verbose,
+           long: '--verbose',
+           description: 'Enable verbose output',
+           on: :tail,
+           boolean: true,
+           proc: proc { Dapp::Builder.default_opts[:log_verbose] = true }
+
+    option :help,
+           short: '-h',
+           long: '--help',
+           description: 'Show this message',
+           on: :tail,
+           boolean: true,
+           show_options: true,
+           exit: 0
+
+    option :build_dir,
+           long: '--build-dir PATH',
+           description: 'Build directory',
+           proc: proc { |p| Dapp::Builder.default_opts[:build_dir] = p }
+
+    option :dir,
+           long: '--dir PATH',
+           description: 'Change to directory',
+           on: :head
+
+    option :dappfile_name,
+           long: '--dappfile-name NAME',
+           description: 'Name of Dappfile',
+           proc: proc { |n| Dapp::Builder.default_opts[:dappfile_name] = n },
+           on: :head
+
+    option :flush_cache,
+           long: '--flush-cache',
+           description: 'Flush cache',
+           boolean: true,
+           proc: proc { Dapp::Builder.default_opts[:flush_cache] = true }
+
+    option :docker_registry,
+           long: '--docker-registry REGISTRY',
+           description: 'Docker registry',
+           proc: proc { |r| Dapp::Builder.default_opts[:docker_registry] = r }
+
+    option :cascade_tagging,
+           long: '--cascade_tagging',
+           description: 'Use cascade tagging',
+           boolean: true,
+           proc: proc { Dapp::Builder.default_opts[:cascade_tagging] = true }
+
+    option :git_artifact_branch,
+           long: '--git-artifact-branch BRANCH',
+           description: 'Default branch to archive artifacts from',
+           proc: proc { |b| Dapp::Builder.default_opts[:git_artifact_branch] = b }
+
+    def dappfile_path
+      @dappfile_path ||= File.join [config[:dir], 'Dappfile'].compact
+    end
+
+    def patterns
+      @patterns ||= cli_arguments
+    end
+
+    def run(argv = ARGV)
+      begin
+        parse_options(argv)
+      rescue OptionParser::InvalidOption => e
+        STDERR.puts "Error: #{e.message}"
+        exit 1
+      end
+
+      patterns << '*' unless patterns.any?
+
+      if File.exist? dappfile_path
+        process_file
+      else
+        process_directory
+      end
+    end
+
+    def process_file
+      patterns.each do |pattern|
+        unless Dapp::Builder.process_file(dappfile_path, app_filter: pattern).builded_apps.any?
+          STDERR.puts "Error! No such app: '#{pattern}' in #{dappfile_path}"
+          exit 1
+        end
+      end
+    end
+
+    def process_directory
+      Dapp::Builder.default_opts[:shared_build_dir] = true
+      patterns.each do |pattern|
+        unless Dapp::Builder.process_directory(config[:dir], pattern).any?
+          STDERR.puts "Error! No such app '#{pattern}'"
+          exit 1
+        end
+      end
+    end
+  end
+end

data/lib/dapp/docker.rb

@@ -0,0 +1,188 @@
+module Dapp
+  # Dockerfile builder and docker binary wrapper
+  class Docker
+    def initialize(builder)
+      @builder = builder
+    end
+
+    def name(name = nil)
+      @name = name || @name
+    end
+
+    def from(from = nil)
+      @from = from || @from
+    end
+
+    def build_path(*path)
+      builder.build_path(builder.home_branch, [name, 'docker'].compact.join('.'), *path)
+    end
+
+    def run(*command, step: :build)
+      add_instruction step, :run, command.join(' && ')
+    end
+
+    def workdir(directory, step: :build)
+      add_instruction step, :workdir, directory
+    end
+
+    def copy(what, where, step: :build)
+      add_instruction step, :copy, what, where
+    end
+
+    def add(what, where, step: :build)
+      add_instruction step, :add, what, where
+    end
+
+    def add_artifact(file_path, filename, where, step: :build)
+      add_instruction step, :add_artifact, file_path, filename, where
+    end
+
+    def expose(*ports, step: :build)
+      add_instruction step, :expose, ports
+    end
+
+    def env(step: :build, **env)
+      add_instruction step, :env, env
+    end
+
+    def volume(*paths, step: :build)
+      add_instruction step, :volume, paths
+    end
+
+    def cmd(*commands, step: :build)
+      add_instruction step, :cmd, commands
+    end
+
+    def initialize_dup(other)
+      super
+
+      @name = @name.dup if @name
+      @from = @from.dup if @from
+
+      @instructions = @instructions.dup
+      @instructions.each do |step, step_instructions|
+        @instructions[step] = step_instructions.dup
+      end
+    end
+
+    def build
+      # prepare
+      generate_dockerfile
+
+      # build
+      res = docker("build --force-rm=true --rm=true #{build_path}", log_verbose: true)
+
+      # return image id
+      res.stdout.lines.grep(/^Successfully built ([0-9a-f]+)\n$/).first.strip.split.last
+    end
+
+    def image_exists?(**kwargs)
+      !image_id(**kwargs).nil?
+    end
+
+    def image_id(**kwargs)
+      image = images(**kwargs).first
+      image ? image[:id] : nil
+    end
+
+    def images(name:, tag: nil, registry: nil)
+      all_images.select { |i| i[:name] == pad_image_name(name: name, registry: registry) && (!tag || i[:tag] == tag) }
+    end
+
+    def all_images
+      docker('images').stdout.lines.drop(1).map(&:strip)
+                      .map { |line| Hash[[:name, :tag, :id].zip(line.strip.split(/\s{2,}/)[0..3])] }
+    end
+
+    def tag(origin, new, force: true)
+      docker "tag#{' -f' if force} #{origin.is_a?(String) ? origin : pad_image_name(**origin)} #{pad_image_name(**new)}"
+    end
+
+    def rmi(**kwargs)
+      docker "rmi #{pad_image_name(**kwargs)}"
+    end
+
+    def push(name:, tag: nil, registry: nil)
+      docker "push #{pad_image_name name: name, tag: tag, registry: registry}", log_verbose: true
+    end
+
+    protected
+
+    def pad_image_name(name:, tag: nil, registry: nil)
+      name = "#{registry}/#{name}" if registry
+      name = "#{name}:#{tag}" if tag
+      name
+    end
+
+    attr_reader :builder
+
+    def instructions(step)
+      (@instructions ||= {})[step] ||= []
+    end
+
+    def add_instruction(step, *args)
+      instructions(step) << args
+    end
+
+    def docker(command, **kwargs)
+      builder.shellout "docker #{command}", **kwargs
+    end
+
+    def dockerfile_path
+      build_path 'Dockerfile'
+    end
+
+    def generate_dockerfile_run(dockerfile, *params)
+      dockerfile.puts 'RUN ' + params[0]
+    end
+
+    def generate_dockerfile_copy(dockerfile, *params)
+      dockerfile.puts "COPY #{params[0]} #{params[1]}"
+    end
+
+    def generate_dockerfile_add(dockerfile, *params)
+      dockerfile.puts "ADD #{params[0]} #{params[1]}"
+    end
+
+    def generate_dockerfile_add_artifact(dockerfile, *params)
+      FileUtils.link params[0], build_path(params[1]), force: true
+      dockerfile.puts "ADD #{params[1]} #{params[2]}"
+    end
+
+    def generate_dockerfile_expose(dockerfile, *params)
+      dockerfile.puts 'EXPOSE ' + params[0].map(&:to_s).join(' ')
+    end
+
+    def generate_dockerfile_env(dockerfile, *params)
+      dockerfile.puts 'ENV ' + params[0].map { |k, v| %(#{k}="#{v}") }.join(' ')
+    end
+
+    def generate_dockerfile_volume(dockerfile, *params)
+      dockerfile.puts 'VOLUME ' + params[0].join(' ')
+    end
+
+    def generate_dockerfile_workdir(dockerfile, *params)
+      dockerfile.puts "WORKDIR  #{params[0]}"
+    end
+
+    def generate_dockerfile_cmd(dockerfile, *params)
+      dockerfile.puts 'CMD ' + params[0].join(' ')
+    end
+
+    def generate_dockerfile_step(dockerfile, step)
+      instructions(step).each do |directive, *params|
+        send :"generate_dockerfile_#{directive}", dockerfile, *params
+      end
+    end
+
+    def generate_dockerfile
+      File.open dockerfile_path, 'w' do |dockerfile|
+        dockerfile.puts 'FROM ' + from
+
+        [:begining, :prepare, :build, :setup].each do |step|
+          generate_dockerfile_step(dockerfile, step)
+        end
+      end
+    end
+  end
+end

data/lib/dapp/filelock.rb

@@ -0,0 +1,42 @@
+module Dapp
+  # File Monitor
+  module Filelock
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    # ClassMethods
+    module ClassMethods
+      def filelocks
+        @filelocks ||= Hash.new(false)
+      end
+    end
+
+    def filelock(filelock, error_message: 'Already in use!', timeout: 10, &blk)
+      return yield if self.class.filelocks[filelock]
+
+      begin
+        self.class.filelocks[filelock] = true
+        filelock_lockfile(filelock, error_message: error_message, timeout: timeout, &blk)
+        yield
+      ensure
+        self.class.filelocks[filelock] = false
+      end
+    end
+
+    protected
+
+    def filelock_lockfile(filelock, error_message: 'Already in use!', timeout: 10)
+      File.open(build_path(filelock), File::RDWR | File::CREAT, 0644) do |file|
+        Timeout.timeout(timeout) do
+          file.flock(File::LOCK_EX)
+        end
+
+        yield
+      end
+    rescue Timeout::Error
+      STDERR.puts error_message
+      exit 1
+    end
+  end
+end

data/lib/dapp/git_artifact.rb

@@ -0,0 +1,330 @@
+module Dapp
+  # Artifact from Git repo
+  class GitArtifact
+    # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength
+    def initialize(builder, repo, where_to_add, name: nil, branch: 'master', cwd: nil, paths: nil, owner: nil, group: nil,
+                   interlayer_period: 7 * 24 * 3600, build_path: nil, flush_cache: false)
+      @builder = builder
+      @repo = repo
+      @name = name
+
+      @where_to_add = where_to_add
+      @branch = branch
+      @cwd = cwd
+      @paths = paths
+      @owner = owner
+      @group = group
+
+      @interlayer_period = interlayer_period
+
+      @build_path = build_path || []
+
+      @atomizer = Atomizer.new builder, build_path(filename('.atomizer'))
+
+      # check params hash
+      lock do
+        cleanup! unless !flush_cache && File.exist?(paramshash_path) && File.read(paramshash_path) == paramshash
+        File.write paramshash_path, paramshash
+      end
+    end
+    # rubocop:enable Metrics/ParameterLists, Metrics/MethodLength
+
+    def build_path(*paths)
+      builder.build_path(*@build_path, *paths)
+    end
+
+    def add_multilayer!
+      lock_with_repo do
+        # create and add archive
+        create_and_add_archive
+        return if archive_commit == repo_latest_commit
+
+        # add layer patches
+        latest_layer = add_layer_patches
+        if latest_layer
+          latest_layer_commit = layer_commit(latest_layer)
+          return if latest_layer_commit == repo_latest_commit
+        end
+
+        # empty changes
+        unless any_changes?(latest_layer_commit || archive_commit)
+          remove_latest!
+          return
+        end
+
+        # create and add last patch
+        create_and_add_last_patch(latest_layer, latest_layer_commit)
+      end
+    end
+
+    def cleanup!
+      lock do
+        FileUtils.rm_f [
+          paramshash_path,
+          archive_path,
+          archive_commitfile_path,
+          Dir.glob(layer_patch_path('*')),
+          Dir.glob(layer_commitfile_path('*')),
+          latest_patch_path,
+          latest_commitfile_path
+        ].flatten
+      end
+    end
+
+    def exists_in_commit?(path, commit)
+      repo.git_bare("cat-file -e #{commit}:#{path}", returns: [0, 128]).status.success?
+    end
+
+    def exists_in_step?(path, step)
+      exists_in_commit?(path, commit_by_step(step))
+    end
+
+    def prepare_step_commit
+      archive_commit
+    end
+
+    def build_step_commit
+      layer_commit(layers.last) || archive_commit
+    end
+
+    def setup_step_commit
+      latest_commit || layer_commit(layers.last) || archive_commit
+    end
+
+    def commit_by_step(step)
+      send :"#{step}_step_commit"
+    end
+
+    def any_changes?(from)
+      !repo.git_bare("diff --quiet #{from}..#{repo_latest_commit}#{" --relative=#{cwd}" if cwd} #{paths(true)}", returns: [0, 1]).status.success?
+    end
+
+    protected
+
+    attr_reader :builder
+    attr_reader :repo
+    attr_reader :name
+    attr_reader :where_to_add
+    attr_reader :branch
+    attr_reader :cwd
+    attr_reader :owner
+    attr_reader :group
+    attr_reader :interlayer_period
+    attr_reader :atomizer
+
+    def lock_with_repo(&blk)
+      lock do
+        repo.lock(&blk)
+      end
+    end
+
+    def create_and_add_archive
+      create_archive! unless archive_exists?
+      add_archive
+    end
+
+    def add_layer_patches
+      latest_layer = nil
+      layers.each do |layer|
+        add_layer_patch layer
+        latest_layer = layer
+      end
+
+      latest_layer
+    end
+
+    def create_and_add_last_patch_as_layer_patch(latest_layer, latest_layer_commit)
+      remove_latest!
+      layer = latest_layer.to_i + 1
+      create_layer_patch!(latest_layer_commit || archive_commit, layer)
+      add_layer_patch layer
+    end
+
+    def create_and_add_last_patch_as_latest_patch(_latest_layer, latest_layer_commit)
+      if latest_commit != repo_latest_commit
+        create_latest_patch!(latest_layer_commit || archive_commit)
+      end
+      add_latest_patch
+    end
+
+    def create_and_add_last_patch(latest_layer, latest_layer_commit)
+      if (Time.now - repo.commit_at(latest_layer_commit || archive_commit)) > interlayer_period
+        create_and_add_last_patch_as_layer_patch(latest_layer, latest_layer_commit)
+      else
+        create_and_add_last_patch_as_latest_patch(latest_layer, latest_layer_commit)
+      end
+    end
+
+    def paths(with_cwd = false)
+      [@paths].flatten.compact.map { |path| (with_cwd && cwd ? "#{cwd}/#{path}" : path).gsub(%r{^\/*|\/*$}, '') }.join(' ') if @paths
+    end
+
+    def repo_latest_commit
+      repo.latest_commit(branch)
+    end
+
+    def filename(ending)
+      "#{repo.name}#{name ? "_#{name}" : nil}.#{branch}#{ending}"
+    end
+
+    def paramshash_filename
+      filename '.paramshash'
+    end
+
+    def paramshash_path
+      build_path paramshash_filename
+    end
+
+    def paramshash
+      Digest::SHA256.hexdigest [cwd, paths, owner, group].map(&:to_s).join(':::')
+    end
+
+    def archive_filename
+      filename '.tar.gz'
+    end
+
+    def archive_path
+      build_path archive_filename
+    end
+
+    def archive_commitfile_path
+      build_path filename '.commit'
+    end
+
+    def archive_commit
+      File.read archive_commitfile_path
+    end
+
+    def create_arhive_with_owner_substitution!
+      Dir.mktmpdir('change_archive_owner', build_path) do |tmpdir_path|
+        atomizer << tmpdir_path
+        repo.git_bare "archive #{repo_latest_commit}:#{cwd} #{paths} | /bin/tar --extract --directory #{tmpdir_path}"
+        builder.shellout("/usr/bin/find #{tmpdir_path} -maxdepth 1 -mindepth 1 -printf '%P\\n' | /bin/tar -czf #{archive_path} -C #{tmpdir_path}" \
+                         " -T - --owner=#{owner || 'root'} --group=#{group || 'root'}")
+      end
+    end
+
+    def create_simple_archive!
+      repo.git_bare "archive --format tar.gz #{repo_latest_commit}:#{cwd} -o #{archive_path} #{paths}"
+    end
+
+    def create_archive!
+      atomizer << archive_path
+      atomizer << archive_commitfile_path
+
+      if owner || group
+        create_arhive_with_owner_substitution!
+      else
+        create_simple_archive!
+      end
+
+      File.write archive_commitfile_path, repo_latest_commit
+    end
+
+    def archive_exists?
+      File.exist? archive_commitfile_path
+    end
+
+    def add_archive
+      builder.docker.add_artifact archive_path, archive_filename, where_to_add, step: :prepare
+    end
+
+    def sudo_format_user(user)
+      user.to_i.to_s == user ? "\\\##{user}" : user
+    end
+
+    def sudo
+      sudo = ''
+
+      if owner || group
+        sudo = 'sudo '
+        sudo += "-u #{sudo_format_user(owner)} " if owner
+        sudo += "-g #{sudo_format_user(group)} " if group
+      end
+
+      sudo
+    end
+
+    def add_patch(filename, step:)
+      builder.docker.add_artifact(build_path(filename), filename, '/tmp', step: step)
+
+      builder.docker.run [
+        "zcat /tmp/#{filename} | #{sudo}git apply --whitespace=nowarn --directory=#{where_to_add}",
+        "rm /tmp/#{filename}"
+      ], step: step
+    end
+
+    def create_patch!(from, filename, commitfile_path)
+      atomizer << build_path(filename)
+      atomizer << commitfile_path
+
+      repo.git_bare "diff --binary #{from}..#{repo_latest_commit}#{" --relative=#{cwd}" if cwd} #{paths(true)} | gzip > #{build_path filename}"
+      File.write commitfile_path, repo_latest_commit
+    end
+
+    def layer_filename(layer, ending)
+      filename "_layer_#{layer.is_a?(Fixnum) ? format('%04d', layer) : layer}#{ending}"
+    end
+
+    def layer_patch_filename(layer)
+      layer_filename(layer, '.patch.gz')
+    end
+
+    def layer_patch_path(layer)
+      build_path layer_patch_filename(layer)
+    end
+
+    def layer_commitfile_path(layer)
+      build_path layer_filename(layer, '.commit')
+    end
+
+    def layer_commit(layer)
+      File.read layer_commitfile_path(layer) if File.exist? layer_commitfile_path(layer)
+    end
+
+    def layers
+      Dir.glob(layer_commitfile_path('*')).map { |path| Integer(path.gsub(/.*_(\d+)\.commit$/, '\\1')) }.sort
+    end
+
+    def create_layer_patch!(from, layer)
+      create_patch! from, layer_patch_filename(layer), layer_commitfile_path(layer)
+    end
+
+    def add_layer_patch(layer)
+      add_patch layer_patch_filename(layer), step: :build
+    end
+
+    def latest_patch_filename
+      filename '_latest.patch.gz'
+    end
+
+    def latest_patch_path
+      build_path latest_patch_filename
+    end
+
+    def latest_commitfile_path
+      build_path filename '_latest.commit'
+    end
+
+    def latest_commit
+      File.read latest_commitfile_path if File.exist? latest_commitfile_path
+    end
+
+    def create_latest_patch!(from)
+      create_patch! from, latest_patch_filename, latest_commitfile_path
+    end
+
+    def add_latest_patch
+      add_patch latest_patch_filename, step: :setup
+    end
+
+    def remove_latest!
+      FileUtils.rm_f [latest_patch_path, latest_commitfile_path]
+    end
+
+    def lock(**kwargs, &block)
+      builder.filelock(build_path(filename('.lock')), error_message: "Branch #{branch} of artifact #{name ? " #{name}" : nil} #{repo.name}" \
+                       " (#{repo.dir_path}) in use! Try again later.", **kwargs, &block)
+    end
+  end
+end