dapp 0.6.1 → 0.6.2

data/lib/dapp/build/stage/setup/ga_post_setup_patch.rb

@@ -16,7 +16,7 @@ module Dapp
           end
 
           def next_g_a_stage
-            next_stage # GALatestPatch
+            next_stage.next_stage
           end
         end # GAPostSetupPatch
       end

data/lib/dapp/build/stage/setup/ga_pre_setup_patch.rb

@@ -12,11 +12,11 @@ module Dapp
           end
 
           def prev_g_a_stage
-            super.prev_stage # GAPostInstallPatch
+            super.prev_stage.prev_stage # GAPostInstallPatch
           end
 
           def next_g_a_stage
-            super.next_stage # GAPostSetupPatch
+            super.next_stage # GAPostSetupPatch || GAArtifactPatch
           end
         end # GAPrePatch
       end

data/lib/dapp/build/stage/setup/ga_pre_setup_patch_dependencies.rb

@@ -7,18 +7,12 @@ module Dapp
           include Mod::Group
 
           def initialize(application, next_stage)
-            @prev_stage = BeforeSetup.new(application, self)
+            @prev_stage = BeforeSetupArtifact.new(application, self)
             super
           end
 
           def dependencies
-            [setup_dependencies_files_checksum, application.builder.setup_checksum]
-          end
-
-          private
-
-          def setup_dependencies_files_checksum
-            @setup_files_checksum ||= dependencies_files_checksum(application.config._setup_dependencies)
+            next_stage.next_stage.context # Setup
           end
         end # GAPreSetupPatchDependencies
       end

data/lib/dapp/build/stage/setup/setup.rb

@@ -12,17 +12,22 @@ module Dapp
           end
 
           def empty?
-            super && !application.builder.setup?
+            !application.builder.setup?
           end
 
-          def dependencies
-            prev_stage.prev_stage.dependencies # GAPreSetupPatchDependencies
+          def context
+            [setup_dependencies_files_checksum, application.builder.setup_checksum]
           end
 
-          def image
-            super do |image|
-              application.builder.setup(image)
-            end
+          def prepare_image
+            super
+            application.builder.setup(image)
+          end
+
+          private
+
+          def setup_dependencies_files_checksum
+            @setup_files_checksum ||= dependencies_files_checksum(application.config._setup_dependencies)
           end
         end # Setup
       end

data/lib/dapp/builder/base.rb

@@ -8,7 +8,7 @@ module Dapp
         @application = application
       end
 
-      def before_application_export
+      def before_application_should_be_built_check
       end
 
       def before_install?
@@ -59,6 +59,18 @@ module Dapp
         raise
       end
 
+      def build_artifact?
+        false
+      end
+
+      def build_artifact(_image)
+        raise
+      end
+
+      def build_artifact_checksum
+        raise
+      end
+
       def chef_cookbooks(_image)
       end
 

data/lib/dapp/builder/chef.rb

@@ -11,31 +11,24 @@ module Dapp
 
       DEFAULT_CHEFDK_IMAGE = 'dappdeps/chefdk:0.17.3-1'.freeze # TODO: config, DSL, DEFAULT_CHEFDK_IMAGE
 
-      def before_application_export
-        super
-
-        %i(before_install install before_setup setup chef_cookbooks).each do |stage|
-          raise ::Dapp::Error::Application, code: :cookbooks_stage_checksum_not_caclculated,
-                                            data: { stage: stage } unless stage_cookbooks_checksum_path(stage).exist?
+      %i(before_install install before_setup setup build_artifact).each do |stage|
+        define_method("#{stage}_checksum") do
+          application.hashsum [stage_cookbooks_checksum(stage),
+                               stage_attributes_raw(stage),
+                               *stage_cookbooks_runlist(stage)]
         end
-      end
-
-      %i(before_install install before_setup setup).each do |stage|
-        define_method("#{stage}_checksum") { stage_cookbooks_checksum(stage) }
 
         define_method("#{stage}?") { !stage_empty?(stage) }
 
         define_method(stage.to_s) do |image|
           unless stage_empty?(stage)
             image.add_volumes_from(chefdk_container)
-            image.add_command 'export PATH=/.dapp/deps/chefdk/bin:$PATH',
-                              "export DAPP_BUILD_STAGE=#{stage}"
-
             image.add_volume "#{stage_build_path(stage)}:#{container_stage_build_path(stage)}:ro"
-            image.add_command ['chef-solo',
+            image.add_command ['/.dapp/deps/chefdk/bin/chef-solo',
                                '--legacy-mode',
-                               "-c #{container_stage_config_path(stage)}",
-                               "-o #{stage_cookbooks_runlist(stage).join(',')}"].join(' ')
+                               "--config #{container_stage_config_path(stage)}",
+                               "--json-attributes #{container_stage_json_attributes_path(stage)}",
+                               "--override-runlist #{stage_cookbooks_runlist(stage).join(',')}"].join(' ')
           end
         end
       end
@@ -45,14 +38,23 @@ module Dapp
       end
 
       def chef_cookbooks(image)
-        image.add_volume "#{cookbooks_vendor_path}:#{application.container_dapp_path('chef_cookbooks')}"
+        image.add_volume "#{cookbooks_vendor_path(chef_cookbooks_stage: true)}:#{application.container_dapp_path('chef_cookbooks')}"
         image.add_command(
-          'mkdir -p /usr/share/dapp/chef_repo',
-          ["cp -a #{application.container_dapp_path('chef_cookbooks')} ",
+          "#{application.project.mkdir_path} -p /usr/share/dapp/chef_repo",
+          ["#{application.project.cp_path} -a #{application.container_dapp_path('chef_cookbooks')} ",
            '/usr/share/dapp/chef_repo/cookbooks'].join
         )
       end
 
+      def before_application_should_be_built_check
+        super
+
+        %i(before_install install before_setup setup build_artifact chef_cookbooks).each do |stage|
+          raise ::Dapp::Error::Application, code: :cookbooks_stage_checksum_not_calculated,
+                                            data: { stage: stage } unless stage_cookbooks_checksum_path(stage).exist?
+        end
+      end
+
       private
 
       def enabled_modules
@@ -63,6 +65,14 @@ module Dapp
         application.config._chef._recipes
       end
 
+      def stage_attributes(stage)
+        application.config._chef.send("_#{stage}_attributes")
+      end
+
+      def stage_attributes_raw(stage)
+        JSON.dump stage_attributes(stage)
+      end
+
       def project_name
         cookbook_metadata.name
       end
@@ -84,7 +94,12 @@ module Dapp
       end
 
       def cookbook_metadata
-        @cookbook_metadata ||= CookbookMetadata.new(cookbook_metadata_path)
+        @cookbook_metadata ||= CookbookMetadata.new(cookbook_metadata_path).tap do |metadata|
+          metadata.depends.each do |dependency|
+            raise Error, code: :mdapp_dependency_in_metadata_forbidden,
+                         data: { dependency: dependency } if dependency.start_with? 'mdapp-'
+          end
+        end
       end
 
       def berksfile_lock_checksum
@@ -100,11 +115,11 @@ module Dapp
           stage_cookbooks_checksum_path(stage).read.strip
         else
           checksum = if stage == :chef_cookbooks
-                       paths = Dir[cookbooks_vendor_path('**/*')].map(&Pathname.method(:new))
+                       paths = Dir[cookbooks_vendor_path('**/*', chef_cookbooks_stage: true)].map(&Pathname.method(:new))
 
                        application.hashsum [
                          application.paths_content_hashsum(paths),
-                         *paths.map { |p| p.relative_path_from(cookbooks_vendor_path).to_s }.sort
+                         *paths.map { |p| p.relative_path_from(cookbooks_vendor_path(chef_cookbooks_stage: true)).to_s }.sort
                        ]
                      else
                        paths = Dir[stage_cookbooks_path(stage, '**/*')].map(&Pathname.method(:new))
@@ -112,7 +127,6 @@ module Dapp
                        application.hashsum [
                          application.paths_content_hashsum(paths),
                          *paths.map { |p| p.relative_path_from(stage_cookbooks_path(stage)).to_s }.sort,
-                         *enabled_modules,
                          stage == :before_install ? chefdk_image : nil
                        ].compact
                      end
@@ -158,8 +172,8 @@ module Dapp
       def chefdk_container
         @chefdk_container ||= begin
           if application.project.shellout("docker inspect #{chefdk_container_name}").exitstatus.nonzero?
-            application.project.log_secondary_process(application.project.t(code: 'process.chefdk_loading'), short: true) do
-              application.project.shellout(
+            application.project.log_secondary_process(application.project.t(code: 'process.chefdk_container_loading'), short: true) do
+              application.project.shellout!(
                 ['docker create',
                  "--name #{chefdk_container_name}",
                  "--volume /.dapp/deps/chefdk #{chefdk_image}"].join(' ')
@@ -172,14 +186,19 @@ module Dapp
       end
 
       # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
-      def install_cookbooks
-        volumes_from = chefdk_container
-        application.project.log_secondary_process(application.project.t(code: 'process.berks_vendor')) do
+      def install_cookbooks(dest_path, chef_cookbooks_stage: false)
+        volumes_from = [application.project.base_container, chefdk_container]
+        process_code = [
+          'process',
+          chef_cookbooks_stage ? 'chef_cookbooks_stage_berks_vendor' : 'berks_vendor'
+        ].compact.join('.')
+
+        application.project.log_secondary_process(application.project.t(code: process_code)) do
           before_vendor_commands = [].tap do |commands|
-            unless application.project.cli_options[:dev]
+            unless application.project.cli_options[:dev] || chef_cookbooks_stage
               commands.push(
                 ['if [ ! -f Berksfile.lock ] ; then ',
-                 'echo "Berksfile.lock not found" 1>&2 ; ',
+                 "#{application.project.echo_path} \"Berksfile.lock not found\" 1>&2 ; ",
                  'exit 1 ; ',
                  'fi'].join
               )
@@ -189,14 +208,15 @@ module Dapp
           after_vendor_commands = [].tap do |commands|
             if application.project.cli_options[:dev]
               commands.push(
-                ["install -o #{Process.uid} -g #{Process.gid} --mode $(stat -c %a Berksfile.lock) ",
+                ["#{application.project.install_path} -o #{Process.uid} -g #{Process.gid} ",
+                 "--mode $(#{application.project.stat_path} -c %a Berksfile.lock) ",
                  "Berksfile.lock #{berksfile_lock_path}"].join
               )
-            else
+            elsif !chef_cookbooks_stage
               commands.push(
-                "export LOCKDIFF=$(diff -u1 Berksfile.lock #{berksfile_lock_path})",
+                "export LOCKDIFF=$(#{application.project.diff_path} -u1 Berksfile.lock #{berksfile_lock_path})",
                 ['if [ "$LOCKDIFF" != "" ] ; then ',
-                 'echo -e "Bad Berksfile.lock\n$LOCKDIFF" 1>&2 ; ',
+                 "#{application.project.echo_path} -e \"Bad Berksfile.lock\n$LOCKDIFF\" 1>&2 ; ",
                  'exit 1 ; ',
                  'fi'].join
               )
@@ -204,50 +224,62 @@ module Dapp
           end
 
           vendor_commands = [
-            'mkdir -p ~/.ssh',
-            'echo "Host *" >> ~/.ssh/config',
-            'echo "    StrictHostKeyChecking no" >> ~/.ssh/config',
+            "#{application.project.mkdir_path} -p ~/.ssh",
+            "#{application.project.echo_path} \"Host *\" >> ~/.ssh/config",
+            "#{application.project.echo_path} \"    StrictHostKeyChecking no\" >> ~/.ssh/config",
             *berksfile.local_cookbooks
                       .values
-                      .map { |cookbook| "rsync --archive --relative #{cookbook[:path]} /tmp/local_cookbooks" },
+                      .map { |cookbook| "#{application.project.rsync_path} --archive --relative #{cookbook[:path]} /tmp/local_cookbooks" },
             "cd /tmp/local_cookbooks/#{berksfile_path.parent}",
             *before_vendor_commands,
             '/.dapp/deps/chefdk/bin/berks vendor /tmp/cookbooks',
             *after_vendor_commands,
-            ["find /tmp/cookbooks -type d -exec bash -ec '",
-             "install -o #{Process.uid} -g #{Process.gid} --mode $(stat -c %a {}) -d ",
-             "#{_cookbooks_vendor_path}/$(echo {} | sed -e \"s/^\\/tmp\\/cookbooks//\")' \\;"].join,
-            ["find /tmp/cookbooks -type f -exec bash -ec '",
-             "install -o #{Process.uid} -g #{Process.gid} --mode $(stat -c %a {}) {} ",
-             "#{_cookbooks_vendor_path}/$(echo {} | sed -e \"s/\\/tmp\\/cookbooks//\")' \\;"].join,
-            "install -o #{Process.uid} -g #{Process.gid} --mode 0644 <(date +%s.%N) #{_cookbooks_vendor_path.join('.created_at')}"
+            ["#{application.project.find_path} /tmp/cookbooks -type d -exec #{application.project.bash_path} -ec '",
+             "#{application.project.install_path} -o #{Process.uid} -g #{Process.gid} --mode $(#{application.project.stat_path} -c %a {}) -d ",
+             "#{dest_path}/$(#{application.project.echo_path} {} | #{application.project.sed_path} -e \"s/^\\/tmp\\/cookbooks//\")' \\;"].join,
+            ["#{application.project.find_path} /tmp/cookbooks -type f -exec #{application.project.bash_path} -ec '",
+             "#{application.project.install_path} -o #{Process.uid} -g #{Process.gid} --mode $(#{application.project.stat_path} -c %a {}) {} ",
+             "#{dest_path}/$(#{application.project.echo_path} {} | #{application.project.sed_path} -e \"s/\\/tmp\\/cookbooks//\")' \\;"].join,
+            "#{application.project.install_path} -o #{Process.uid} -g #{Process.gid} --mode 0644 <(#{application.project.date_path} +%s.%N) #{dest_path.join('.created_at')}"
           ]
 
           application.project.shellout!(
             ['docker run --rm',
-             "--volumes-from #{volumes_from}",
+             volumes_from.map { |container| "--volumes-from #{container}" }.join(' '),
              *berksfile.local_cookbooks
                        .values
                        .map { |cookbook| "--volume #{cookbook[:path]}:#{cookbook[:path]}" },
              ("--volume #{application.project.ssh_auth_sock}:#{application.project.ssh_auth_sock}" if application.project.ssh_auth_sock),
-             "--volume #{_cookbooks_vendor_path.tap(&:mkpath)}:#{_cookbooks_vendor_path}",
+             "--volume #{dest_path.tap(&:mkpath)}:#{dest_path}",
              ("--env SSH_AUTH_SOCK=#{application.project.ssh_auth_sock}" if application.project.ssh_auth_sock),
-             "dappdeps/berksdeps:0.1.0 bash -ec '#{application.project.shellout_pack(vendor_commands.join(' && '))}'"].compact.join(' '),
-            log_verbose: application.project.log_verbose?
+             ('--env DAPP_CHEF_COOKBOOKS_VENDORING=1' if chef_cookbooks_stage),
+             "dappdeps/berksdeps:0.1.0 #{application.project.bash_path} -ec '#{application.project.shellout_pack(vendor_commands.join(' && '))}'"].compact.join(' '),
+             log_verbose: application.project.log_verbose?
           )
         end
       end
       # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
 
-      def _cookbooks_vendor_path
-        application.build_path.join('cookbooks', cookbooks_checksum)
+      def _cookbooks_vendor_path(chef_cookbooks_stage: false)
+        application.build_path.join(
+          ['cookbooks', chef_cookbooks_stage ? 'chef_cookbooks_stage' : nil].compact.join('.'),
+          cookbooks_checksum
+        )
       end
 
-      def cookbooks_vendor_path(*path)
-        _cookbooks_vendor_path.tap do |cookbooks_path|
-          application.project.lock("#{application.config._basename}.cookbooks.#{cookbooks_checksum}", default_timeout: 120) do
-            @install_cookbooks ||= begin
-              install_cookbooks unless _cookbooks_vendor_path.join('.created_at').exist? && !application.project.cli_options[:dev]
+      def cookbooks_vendor_path(*path, chef_cookbooks_stage: false)
+        _cookbooks_vendor_path(chef_cookbooks_stage: chef_cookbooks_stage).tap do |_cookbooks_path|
+          lock_name = [
+            application.config._basename,
+            'cookbooks',
+            chef_cookbooks_stage ? 'chef_cookbooks_stage' : nil,
+            cookbooks_checksum
+          ].compact.join('.')
+
+          application.project.lock(lock_name, default_timeout: 120) do
+            @install_cookbooks ||= {}
+            @install_cookbooks[chef_cookbooks_stage] ||= begin
+              install_cookbooks(_cookbooks_path, chef_cookbooks_stage: chef_cookbooks_stage) unless _cookbooks_path.join('.created_at').exist? && !application.project.cli_options[:dev]
               true
             end
           end
@@ -256,13 +288,11 @@ module Dapp
 
       # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
       def install_stage_cookbooks(stage)
-        select_existing_paths = ->(cookbook_path, paths) do
+        select_existing_paths = proc do |cookbook_path, paths|
           paths.select { |from, _| cookbook_path.join(from).exist? }
         end
 
-        common_paths = [['metadata.json', 'metadata.json'],
-                        ["files/#{stage}", 'files/default'],
-                        ["templates/#{stage}", 'templates/default']]
+        common_paths = [['metadata.json', 'metadata.json']]
 
         install_paths = Dir[cookbooks_vendor_path('*')]
                         .map(&Pathname.method(:new))
@@ -274,35 +304,39 @@ module Dapp
           mdapp_enabled = is_mdapp && enabled_modules.include?(mdapp_name)
 
           paths = if is_project
+                    common_dapp_paths = select_existing_paths.call(cookbook_path, [
+                                                                     *common_paths,
+                                                                     ["files/#{stage}/common", 'files/default'],
+                                                                     ["templates/#{stage}/common", 'templates/default'],
+                                                                     *enabled_recipes.flat_map do |recipe|
+                                                                       [["files/#{stage}/#{recipe}", 'files/default'],
+                                                                        ["templates/#{stage}/#{recipe}", 'templates/default']]
+                                                                     end
+                                                                   ])
+
                     recipe_paths = enabled_recipes.map { |recipe| ["recipes/#{stage}/#{recipe}.rb", "recipes/#{recipe}.rb"] }
                                                   .select { |from, _| cookbook_path.join(from).exist? }
-
-                    common_project_paths = select_existing_paths.call(
-                      cookbook_path, [*common_paths,
-                                      ['attributes/common', 'attributes'],
-                                      ["attributes/#{stage}", 'attributes']]
-                    )
-
                     if recipe_paths.any?
-                      [*recipe_paths, *common_project_paths]
+                      [*recipe_paths, *common_dapp_paths]
                     else
-                      [nil, *common_project_paths]
+                      [nil, *common_dapp_paths]
                     end
                   elsif is_mdapp && mdapp_enabled
-                    recipe_path = "recipes/#{stage}.rb"
-
-                    common_mdapp_paths = select_existing_paths.call(
-                      cookbook_path, [*common_paths,
-                                      ['attributes/common.rb', 'attributes/common.rb'],
-                                      ["attributes/#{stage}.rb", "attributes/#{stage}.rb"]]
-                    )
+                    common_mdapp_paths = select_existing_paths.call(cookbook_path, [
+                                                                      *common_paths,
+                                                                      ["files/#{stage}", 'files/default'],
+                                                                      ['files/common', 'files/default'],
+                                                                      ["templates/#{stage}", 'templates/default'],
+                                                                      ['templates/common', 'templates/default']
+                                                                    ])
 
+                    recipe_path = "recipes/#{stage}.rb"
                     if cookbook_path.join(recipe_path).exist?
                       [[recipe_path, recipe_path], *common_mdapp_paths]
                     else
                       [nil, *common_mdapp_paths]
                     end
-                  else
+                  elsif !is_mdapp
                     [['.', '.']]
                   end
 
@@ -351,11 +385,11 @@ module Dapp
         @stage_cookbooks_runlist[stage] ||= begin
           res = []
 
-          does_entry_exist = lambda do |cookbook, entrypoint|
+          does_entry_exist = proc do |cookbook, entrypoint|
             stage_cookbooks_path(stage, cookbook, 'recipes', "#{entrypoint}.rb").exist?
           end
 
-          format_entry = lambda do |cookbook, entrypoint|
+          format_entry = proc do |cookbook, entrypoint|
             entrypoint = 'void' if entrypoint.nil?
             "#{cookbook}::#{entrypoint}"
           end
@@ -418,6 +452,18 @@ module Dapp
         container_stage_build_path(stage, 'config.rb', *path)
       end
 
+      def install_json_attributes(stage)
+        @install_json_attributes ||= {}
+        @install_json_attributes[stage] ||= true.tap do
+          stage_build_path(stage, 'attributes.json').write "#{stage_attributes_raw(stage)}\n"
+        end
+      end
+
+      def container_stage_json_attributes_path(stage, *path)
+        install_json_attributes(stage)
+        container_stage_build_path(stage, 'attributes.json', *path)
+      end
+
       def stage_build_path(stage, *path)
         application.tmp_path(application.config._name, stage).join(*path)
       end