vendorificator 0.3.0 → 0.4.0

data/lib/vendorificator/vendor/archive.rb

@@ -10,12 +10,69 @@ require 'vendorificator/vendor'
 module Vendorificator
   class Vendor::Archive < Vendor
     arg_reader :url, :strip_root, :type, :checksum, :filename, :basename, :extname, :unpack
-    attr_reader :conjured_checksum
+    attr_reader :conjured_checksum, :conjured_filesize
 
-    def initialize(environment, name, args={}, &block)
+    def conjure!
+      shell.say_status :download, url
+      archive = download_file
+      shell.say_status :unpack, filename
+      unpack_file archive
+      add_archive_metadata
+      super
+    ensure
+      if archive
+        archive.close
+        archive.unlink
+      end
+    end
+
+    def upstream_version
+      filename
+    end
+
+    def conjure_commit_message
+      rv = "Conjured archive #{name} from #{filename}\nOrigin: #{url}\nChecksum: #{conjured_checksum}\n"
+      rv << "Version: #{version}\n" if version
+      rv
+    end
+
+    private
+
+    def download_file
+      archive = Tempfile.new([basename, extname])
+      archive.write( open(url).read )
+      @conjured_filesize = archive.size
+      archive.close
+      @conjured_checksum = Digest::SHA256.file(archive.path).hexdigest
+      raise RuntimeError, "Checksum error" if checksum && checksum != conjured_checksum
+
+      archive
+    end
+
+    def unpack_file(archive)
+      system "#{unpack} #{Escape.shell_single_word archive.path}"
+      if Dir.entries('.').length == 3 && !args[:no_strip_root]
+        root = (Dir.entries('.') - %w(.. .)).first
+        root_entries = Dir.entries(root) - %w(.. .)
+        while root_entries.include?(root)
+          FileUtils::mv root, root+"~"
+          root << "~"
+        end
+        FileUtils::mv root_entries.map { |e| File.join(root, e) }, '.'
+        FileUtils::rmdir root
+      end
+    end
+
+    def add_archive_metadata
+      @metadata[:archive_checksum] = conjured_checksum
+      @metadata[:archive_filesize] = conjured_filesize
+      @metadata[:archive_url] = url
+    end
+
+    def parse_initialize_args(args = {})
       no_url_given = !args[:url]
 
-      args[:url] ||= name
+      args[:url] ||= @name
       args[:filename] ||= URI::parse(args[:url]).path.split('/').last
 
       case args[:filename]
@@ -47,44 +104,9 @@ module Vendorificator
       args[:basename] ||= $`
       args[:extname] ||= $&
 
-      name = args[:basename] if no_url_given
-
-      super(environment, name, args, &block)
-    end
+      @name = args[:basename] if no_url_given
 
-    def conjure!
-      shell.say_status :download, url
-      archive = Tempfile.new([basename, extname])
-      archive.write( open(url).read )
-      archive.close
-      @conjured_checksum = Digest::SHA256.file(archive.path).hexdigest
-      raise RuntimeError, "Checksum error" if checksum && checksum!=conjured_checksum
-      shell.say_status :unpack, filename
-      system "#{unpack} #{Escape.shell_single_word archive.path}"
-      if Dir.entries('.').length == 3 && !args[:no_strip_root]
-        root = (Dir.entries('.') - %w(.. .)).first
-        root_entries = Dir.entries(root) - %w(.. .)
-        while root_entries.include?(root)
-          FileUtils::mv root, root+"~"
-          root << "~"
-        end
-        FileUtils::mv root_entries.map { |e| File.join(root, e) }, '.'
-        FileUtils::rmdir root
-      end
-      super
-    ensure
-      archive.close
-      archive.unlink
-    end
-
-    def upstream_version
-      filename
-    end
-
-    def conjure_commit_message
-      rv = "Conjured archive #{name} from #{filename}\nOrigin: #{url}\nChecksum: #{conjured_checksum}\n"
-      rv << "Version: #{version}\n" if version
-      rv
+      super args
     end
   end
 

data/lib/vendorificator/vendor/download.rb

@@ -8,16 +8,7 @@ require 'vendorificator/vendor'
 module Vendorificator
   class Vendor::Download < Vendor
     arg_reader :url
-    attr_reader :conjured_checksum
-
-    def initialize(environment, name, args={}, &block)
-      no_url_given = !args[:url]
-
-      args[:url] ||= name
-      name = URI::parse(args[:url]).path.split('/').last if no_url_given
-
-      super(environment, name, args, &block)
-    end
+    attr_reader :conjured_checksum, :conjured_filesize
 
     def path
       args[:path] || category
@@ -29,6 +20,8 @@ module Vendorificator
         outf.write( open(url).read )
       end
       @conjured_checksum = Digest::SHA256.file(name).hexdigest
+      @conjured_filesize = File.size(name)
+      add_download_metadata
     end
 
     def upstream_version
@@ -40,6 +33,23 @@ module Vendorificator
       rv << "Version: #{args[:version]}" if args[:version]
       rv
     end
+
+    private
+
+    def parse_initialize_args(args = {})
+      unless args[:url]
+        args[:url] ||= @name
+        @name = URI::parse(args[:url]).path.split('/').last
+      end
+
+      args
+    end
+
+    def add_download_metadata
+      @metadata[:download_checksum] = conjured_checksum
+      @metadata[:download_filesize] = conjured_filesize
+      @metadata[:download_url] = url
+    end
   end
 
   class Config

data/lib/vendorificator/vendor/git.rb

@@ -6,19 +6,6 @@ module Vendorificator
     arg_reader :repository, :revision, :tag, :branch
     attr_reader :conjured_revision
 
-    def initialize(environment, name, args={}, &block)
-      args[:version] ||= args[:tag] if args[:tag]
-      if [:revision, :tag, :branch].select { |key| args.key?(key) }.count > 1
-        raise ArgumentError, "You can provide only one of: :revision, :tag, :branch"
-      end
-
-      unless args.include?(:repository)
-        args[:repository] = name
-        name = name.split('/').last.sub(/\.git$/, '')
-      end
-      super(environment, name, args, &block)
-    end
-
     def conjure!
       shell.say_status :clone, repository
       MiniGit.git :clone, repository, '.'
@@ -33,6 +20,7 @@ module Vendorificator
       super
 
       @conjured_revision = local_git.capturing.rev_parse('HEAD').strip
+      add_git_metadata
       FileUtils::rm_rf '.git'
     end
 
@@ -48,6 +36,25 @@ module Vendorificator
       rv << "at revision #{conjured_revision}"
       rv
     end
+
+    private
+
+    def add_git_metadata
+      @metadata[:git_revision] = conjured_revision
+    end
+
+    def parse_initialize_args(args = {})
+      args[:version] ||= args[:tag] if args[:tag]
+      if [:revision, :tag, :branch].select { |key| args.key?(key) }.count > 1
+        raise ArgumentError, "You can provide only one of: :revision, :tag, :branch"
+      end
+
+      unless args.include?(:repository)
+        args[:repository] = @name
+        @name = @name.split('/').last.sub(/\.git$/, '')
+      end
+      super args
+    end
   end
 
   class Config

data/lib/vendorificator/vendor/tool.rb

@@ -0,0 +1,60 @@
+require 'fileutils'
+require 'tempfile'
+
+require 'vendorificator/vendor'
+
+module Vendorificator
+  class Vendor::Tool < Vendor
+    arg_reader :specs, :command
+
+    def before_conjure!
+      upstream_version # to cache the version in instance attribute,
+                       # it will be needed when we don't have the
+                       # specs
+    end
+
+    def conjure!
+      specs.each do |spec|
+        src = File.join(environment.git.git_work_tree, spec)
+        if File.exist?(src)
+          FileUtils.install File.join(environment.git.git_work_tree, spec),
+                            File.join(git.git_work_tree, spec),
+                            verbose: true
+        end
+        Dir.chdir(git.git_work_tree) do
+          system self.command or raise RuntimeError, "Command failed"
+        end
+      end
+    end
+
+    def git_add_extra_paths
+      specs.inject(super) do |rv, path|
+        rv << path
+      end
+    end
+
+    def upstream_version
+      @upstream_version ||= git.capturing.
+        log({:n => 1, :pretty => 'format:%ad-%h', :date => 'short'}, *specs).
+        strip
+    end
+  end
+
+  class Config
+    register_module :tool, Vendor::Tool
+
+    def rubygems_bundler
+      tool 'rubygems',
+           :path => 'cache', # Hardcoded, meh
+           :specs => [ 'Gemfile', 'Gemfile.lock' ],
+           :command => 'bundle package --all'
+    end
+
+    def chef_berkshelf
+      tool 'cookbooks',
+           :path => 'cookbooks',
+           :specs => [ 'Berksfile', 'Berksfile.lock' ],
+           :command => 'berks install --path vendor/cookbooks'
+    end
+  end
+end

data/lib/vendorificator/vendor.rb

@@ -1,7 +1,7 @@
 require 'fileutils'
-
+require 'tmpdir'
 require 'thor/shell/basic'
-
+require 'yaml'
 require 'vendorificator/config'
 
 module Vendorificator
@@ -22,20 +22,16 @@ module Vendorificator
     attr_reader :environment, :name, :args, :block
     arg_reader :version
 
-    def initialize(environment, name, args={}, &block)
+    def initialize(environment, name, args = {}, &block)
       @environment = environment
-      @category = args.delete(:category) if args.key?(:category)
-
-      unless (hooks = Array(args.delete(:hooks))).empty?
-        hooks.each do |hook|
-          hook_module = hook.is_a?(Module) ? hook : ::Vendorificator::Hooks.const_get(hook)
-          klass = class << self; self; end;
-          klass.send :include, hook_module
-        end
-      end
       @name = name
-      @args = args
       @block = block
+      @metadata = {
+        :module_name => @name,
+        :unparsed_args => args.clone
+      }
+      @metadata[:parsed_args] = @args = parse_initialize_args(args)
+      @metadata[:module_annotations] = @args[:annotate] if @args[:annotate]
 
       @environment.vendor_instances << self
     end
@@ -73,7 +69,7 @@ module Vendorificator
     end
 
     def work_dir
-      _join(config[:root_dir], work_subdir)
+      _join(git.git_work_tree, environment.relative_root_dir, work_subdir)
     end
 
     def head
@@ -82,6 +78,10 @@ module Vendorificator
       nil
     end
 
+    def tag_name
+      _join(tag_name_base, version)
+    end
+
     def merged
       unless @_has_merged
         if ( head = self.head )
@@ -91,6 +91,9 @@ module Vendorificator
         @_has_merged = true
       end
       @merged
+    rescue MiniGit::GitError
+      @_has_merged = true
+      @merged = nil
     end
 
     def merged_tag
@@ -111,6 +114,13 @@ module Vendorificator
       merged_tag && merged_tag[(1+tag_name_base.length)..-1]
     end
 
+    # Public: Get git vendor notes of the merged commit.
+    #
+    # Returns the Hash of git vendor notes.
+    def merged_notes
+      Commit.new(merged, git).notes?
+    end
+
     def version
       @args[:version] || (!config[:use_upstream_version] && merged_version) || upstream_version
     end
@@ -151,48 +161,33 @@ module Vendorificator
     end
 
     def in_branch(options={}, &block)
-      orig_branch = environment.current_branch
-      stash_message = "vendorificator-#{git.capturing.rev_parse('HEAD').strip}-#{branch_name}-#{Time.now.to_i}"
+      branch_exists = !!self.head
+      Dir.mktmpdir("vendor-#{category}-#{name}") do |tmpdir|
+        clone_opts = {:shared => true, :no_checkout => true}
+        clone_opts[:branch] = branch_name if branch_exists
+        MiniGit.git(:clone, clone_opts, git.git_dir, tmpdir)
+        tmpgit = MiniGit::new(tmpdir)
+        tmpgit.checkout({orphan: true}, branch_name) unless branch_exists
+        tmpgit.rm( { :r => true, :f => true, :q => true, :ignore_unmatch => true }, '.') if options[:clean] || !branch_exists
 
-      # We want to be in repository's root now, as we may need to
-      # remove stuff and don't want to have removed directory as cwd.
-      Dir::chdir git.git_work_tree do
         begin
-          # Stash all local changes
-          git.stash :save, {:all => true, :quiet => true}, stash_message
-
-          # If our branch exists, check it out; otherwise, create a new
-          # orphaned branch.
-          if self.head
-            git.checkout branch_name
-            git.rm( { :r => true, :f => true, :q => true, :ignore_unmatch => true }, '.') if options[:clean]
-          else
-            git.checkout( { :orphan => true }, branch_name )
-            git.rm( { :r => true, :f => true, :q => true, :ignore_unmatch => true }, '.')
+          @git = tmpgit
+          Dir.chdir(tmpdir) do
+            yield
           end
-
-          yield
         ensure
-          # We should try to ensure we're back on original branch and
-          # local changes have been applied
-          begin
-            git.checkout orig_branch
-            stash = git.capturing.
-              stash(:list, {:grep => stash_message, :fixed_strings => true}).lines.map(&:strip)
-            if stash.length > 1
-              shell.say_status 'WARNING', "more than one stash matches #{stash_message}, it's weird", :yellow
-              stash.each { |ln| shell.say_status '-', ln, :yellow }
-            end
-            git.stash :pop, {:quiet => true}, stash.first.sub(/:.*/, '') unless stash.empty?
-          rescue => e
-            shell.say_status 'ERROR', "Cannot revert branch from #{self.head} back to #{orig_branch}: #{e}", :red
-            raise
-          end
+          @git = nil
         end
+
+        git.fetch(tmpdir)
+        git.fetch({tags: true}, tmpdir)
+        git.fetch(tmpdir,
+          "+refs/heads/#{branch_name}:refs/heads/#{branch_name}",
+          "+refs/notes/vendor:refs/notes/vendor")
       end
     end
 
-    def run!
+    def run!(options = {})
       case status
 
       when :up_to_date
@@ -201,12 +196,14 @@ module Vendorificator
       when :unpulled, :unmerged
         shell.say_status 'merging', self.to_s, :yellow
         git.merge({:no_edit => true, :no_ff => true}, tagged_sha1)
+        postprocess! if self.respond_to? :postprocess!
         compute_dependencies!
 
       when :outdated, :new
         shell.say_status 'fetching', self.to_s, :yellow
         begin
           shell.padding += 1
+          before_conjure!
           in_branch(:clean => true) do
             FileUtils::mkdir_p work_dir
 
@@ -223,15 +220,11 @@ module Vendorificator
               make_subdir_root subdir if subdir && !subdir.empty?
             end
 
-
-            # Commit and tag the conjured module
-            git.add work_dir
-            git.commit :m => conjure_commit_message
-            git.tag( { :a => true, :m => tag_message }, tag_name )
-            shell.say_status :tag, tag_name
+            commit_and_annotate(options[:metadata])
           end
           # Merge back to the original branch
           git.merge( {:no_edit => true, :no_ff => true}, branch_name )
+          postprocess! if self.respond_to? :postprocess!
           compute_dependencies!
         ensure
           shell.padding -= 1
@@ -242,35 +235,57 @@ module Vendorificator
       end
     end
 
-    def tag_name_base
-      _join('vendor', category, name)
+    def conjure!
+      block.call(self) if block
     end
 
-    def tag_name
-      _join(tag_name_base, version)
-    end
+    #
+    # Hook points
+    def git_add_extra_paths ; [] ; end
+    def before_conjure! ; end
+    def compute_dependencies! ; end
 
-    def conjure_commit_message
-      "Conjured vendor module #{name} version #{version}"
+    def pushable_refs
+      created_tags.
+        map { |tag| '+' << tag }.
+        unshift("+refs/heads/#{branch_name}")
     end
 
-    def tag_message
-      conjure_commit_message
+    def metadata
+      default = {
+        :module_version => version,
+        :module_category => @category,
+      }
+      default.merge @metadata
     end
 
-    def conjure!
-      block.call(self) if block
+    private
+
+    def parse_initialize_args(args = {})
+      @category = args.delete(:category) if args.key?(:category)
+
+      unless (hooks = Array(args.delete(:hooks))).empty?
+        hooks.each do |hook|
+          hook_module = hook.is_a?(Module) ? hook : ::Vendorificator::Hooks.const_get(hook)
+          klass = class << self; self; end;
+          klass.send :include, hook_module
+        end
+      end
+
+      args
     end
 
-    def compute_dependencies! ; end
+    def tag_name_base
+      _join('vendor', category, name)
+    end
 
-    def pushable_refs
-      branch = "+refs/heads/#{branch_name}"
-      tags = created_tags.map{ |tag| '+' + tag }
-      [branch, tags].flatten
+    def conjure_commit_message
+      "Conjured vendor module #{name} version #{version}"
     end
 
-    private
+    def tag_message
+      conjure_commit_message
+    end
 
     def tagged_sha1
       @tagged_sha1 ||= git.capturing.rev_parse({:verify => true, :quiet => true}, "refs/tags/#{tag_name}^{commit}").strip
@@ -279,12 +294,12 @@ module Vendorificator
     end
 
     def created_tags
-      git.capturing.show_ref.split("\n").map{ |line| line.split(' ')[1] }.
-        select{ |ref| ref =~ /\Arefs\/tags\/#{tag_name_base}/ }
+      git.capturing.show_ref.lines.map{ |line| line.split(' ')[1] }.
+        select{ |ref| ref =~ /\Arefs\/tags\/#{tag_name_base}\// }
     end
 
     def git
-      environment.git
+      @git || environment.git
     end
 
     def config
@@ -309,6 +324,30 @@ module Vendorificator
       Dir.chdir(curdir.to_s) if curdir.exist?
     end
 
+    # Private: Commits and annotates the conjured module.
+    #
+    # environment_metadata - Hash with environment metadata where vendor was run
+    #
+    # Returns nothing.
+    def commit_and_annotate(environment_metadata = {})
+      git.add work_dir, *git_add_extra_paths
+      git.commit :m => conjure_commit_message
+      git.notes({:ref => 'vendor'}, 'add', {:m => conjure_note(environment_metadata)}, 'HEAD')
+      git.tag( { :a => true, :m => tag_message }, tag_name )
+      shell.say_status :tag, tag_name
+    end
+
+    # Private: Merges all the data we use for the commit note.
+    #
+    # environment_metadata - Hash with environment metadata where vendor was run
+    #
+    # Returns: The note in the YAML format.
+    def conjure_note(environment_metadata = {})
+      config.metadata.
+        merge(environment_metadata).
+        merge(metadata).
+        to_yaml
+    end
   end
 
   Config.register_module :vendor, Vendor