vendorificator 0.0.1

data/lib/vendorificator/cli.rb

@@ -0,0 +1,233 @@
+require 'thor'
+
+require 'vendorificator'
+
+module Vendorificator
+  class CLI < Thor
+    include Vendorificator
+
+    check_unknown_options! :except => [:git, :diff, :log]
+    stop_on_unknown_option! :git, :diff, :log
+
+    default_task :sync
+
+    class_option :file, :aliases => '-f', :type => :string, :banner => 'PATH'
+    class_option :debug, :aliases => '-d', :type => :boolean, :default => false
+    class_option :quiet, :aliases => ['-q'], :default => false, :type => :boolean
+    class_option :modules, :type => :string, :default => '',
+      :banner => 'mod1,mod2,...,modN',
+      :desc => 'Run only for specified modules (name or path, comma separated)'
+
+    def initialize(*args)
+      super
+      Grit.debug = true if options[:debug]
+      Vendorificator::Config.from_file(find_vendorfile)
+      Vendorificator::Config[:shell] = shell
+
+      class << shell
+        # Make say_status always say it.
+        def quiet?
+          false
+        end
+      end
+    end
+
+    desc :sync, "Download new or updated vendor files"
+    def sync
+      ensure_clean_repo!
+      Vendorificator::Config.each_module(*modules) do |mod|
+        say_status :module, mod.name
+        begin
+          shell.padding += 1
+          mod.run!
+        ensure
+          shell.padding -= 1
+        end
+      end
+    end
+
+    desc "status", "List known vendor modules and their status"
+    def status
+      say_status 'WARNING', 'Git repository is not clean', :red unless repo.clean?
+      Vendorificator::Config.each_module(*modules) do |mod|
+        status_line = mod.to_s
+
+        updatable = mod.updatable?
+        if updatable
+          if updatable == true
+            status_line << ' (updatable)'
+          else
+            status_line << " (updatable to #{updatable.name})"
+          end
+        end
+
+        say_status( mod.status.to_s.gsub('_', ' '), status_line,
+                    ( mod.status==:up_to_date ? :green : :yellow ) )
+      end
+    end
+
+    desc :pull, "Pull upstream branches from a remote repository"
+    method_option :remote, :aliases => ['-r'], :default => nil
+    method_option :dry_run, :aliases => ['-n'], :default => false, :type => :boolean
+    def pull
+      ensure_clean_repo!
+      remotes = options[:remote] ? options[:remote].split(',') : conf[:remotes]
+      remotes.each do |remote|
+        indent 'remote', remote do
+          repo.pull(remote, options)
+        end
+      end
+    end
+    desc "git GIT_COMMAND [GIT_ARGS [...]]",
+         "Run a git command for specified modules"
+    long_desc <<EOF
+  Run a git command for specified modules. Within GIT_ARGS arguments,
+  you can use @MERGED@ and @PATH@ tags, which will be substituted with
+  mo#dule's most recently merged revision and full path of its work
+  directory.
+
+  The 'diff' and 'log' commands are simple aliases for 'git' command.
+
+  Examples:
+    vendor git log @MERGED@..HEAD -- @PATH@    # basic 'vendor log'
+    vendor git diff --stat @MERGED@ -- @PATH@  # 'vendor diff', as diffstat
+EOF
+    method_option :only_changed, :default => false, :type => :boolean
+    def git(command, *args)
+      Vendorificator::Config.each_module(*modules) do |mod|
+        unless mod.merged
+          say_status 'unmerged', mod.to_s, :red unless options[:only_changed]
+          next
+        end
+
+        actual_args = args.dup.map do |arg|
+          arg.
+            gsub('@MERGED@', mod.merged).
+            gsub('@PATH@', mod.work_dir)
+        end
+
+        output = repo.git.native(command, {}, *actual_args)
+        if output.empty?
+          say_status 'unchanged', mod.to_s, :green unless options[:only_changed]
+        else
+          say_status 'changed', mod.to_s, :yellow
+        end
+        puts output unless options[:quiet] || output.empty?
+      end
+    end
+
+    desc "diff [OPTIONS] [GIT OPTIONS]",
+         "Show differences between work tree and upstream module(s)"
+    method_option :only_changed, :default => false, :type => :boolean
+    def diff(*args)
+      invoke :git, %w'diff' + args + %w'@MERGED@ -- @PATH@'
+    end
+
+    desc "log [OPTIONS] [GIT OPTIONS]",
+         "Show git log of commits added to upstream module(s)"
+    method_option :only_changed, :default => false, :type => :boolean
+    def log(*args)
+      invoke :git, %w'log' + args + %w'@MERGED@..HEAD -- @PATH@'
+    end
+
+    desc :pry, 'Pry into the binding', :hide => true
+    def pry
+      require 'pry'
+      binding.pry
+    end
+
+    def self.start
+      # Make --git-options always quoted
+      if i = ARGV.index('--git-options')
+        ARGV[i+1,0] = '--'
+      end
+
+      if ENV['FIXTURES_DIR']
+        require 'vcr'
+        VCR.configure do |c|
+          c.cassette_library_dir = File.join(ENV['FIXTURES_DIR'], 'vcr_cassettes')
+          c.default_cassette_options = { :record => :new_episodes }
+          c.hook_into :fakeweb
+        end
+        VCR.use_cassette(ENV['VCR_CASSETTE'] || 'vendorificator') do
+          super
+        end
+      else
+        super
+      end
+    end
+
+    private
+
+    def split_git_options(args)
+      case i = args.index('--git-options')
+      when nil then [ args, [] ]
+      when 0 then [ [], args[1..-1] ]
+      else [ args[0..(i-1)], args[(i+1)..-1] ]
+      end
+    end
+
+    def modules
+      options[:modules].split(',').map(&:strip)
+    end
+
+    def conf
+      Vendorificator::Config
+    end
+
+    def repo
+      Vendorificator::Config.repo
+    end
+
+    def fail!(message, exception_message='I give up.')
+      say_status('FATAL', message, :red)
+      raise Thor::Error, 'I give up.'
+    end
+
+    def indent(*args, &block)
+      say_status *args unless args.empty?
+      shell.padding += 1
+      yield
+    ensure
+      shell.padding -= 1
+    end
+
+    # Find proper Vendorfile
+    def find_vendorfile
+      given = options.file || ENV['VENDORFILE']
+      return Pathname.new(given).expand_path if given && !given.empty?
+
+      Pathname.pwd.ascend do |dir|
+        vf = dir.join('Vendorfile')
+        return vf if vf.exist?
+
+        vf = dir.join('config/vendor.rb')
+        return vf if vf.exist?
+
+        # avoid stepping above the tmp directory when testing
+        if ENV['VENDORIFICATOR_SPEC_RUN'] &&
+            dir.join('vendorificator.gemspec').exist?
+          raise RuntimeError, "Vendorfile not found"
+        end
+      end
+
+      raise RuntimeError, "Vendorfile not found"
+    end
+
+    def ensure_clean_repo!
+      unless repo.clean?
+        fail!('Repository is not clean.')
+      end
+    end
+  end
+end
+
+# Monkey patch over https://github.com/wycats/thor/pull/298
+class Thor::Options
+  alias_method :_orig_current_is_switch?, :current_is_switch?
+  def current_is_switch?
+    rv = _orig_current_is_switch?
+    @parsing_options = false if !rv[0] && @stop_on_unknown && @parsing_options
+    rv
+  end
+end

data/lib/vendorificator/config.rb

@@ -0,0 +1,72 @@
+require 'pathname'
+
+require 'mixlib/config'
+
+require 'vendorificator/repo'
+
+module Vendorificator
+  class Config
+    extend Mixlib::Config
+
+    configure do |c|
+      c[:root] = Pathname.getwd
+      c[:basedir] = 'vendor'
+      c[:branch_prefix] = 'vendor'
+      c[:modules] = []
+      c[:remotes] = %w(origin)
+    end
+
+    def self.from_file(filename)
+      pathname = Pathname.new(filename).cleanpath.expand_path
+      self[:root_dir] =
+        if ( pathname.basename.to_s == 'vendor.rb' &&
+               pathname.dirname.basename.to_s == 'config' )
+          # Correctly recognize root dir if main config is 'config/vendor.rb'
+          pathname.dirname.dirname
+        else
+          pathname.dirname
+        end
+      self[:vendorfile_path] = pathname
+      self[:lockfile_path] = pathname.dirname.join(pathname.basename.to_s + '.lock')
+      super(pathname.to_s)
+    end
+
+    def self.repo
+      @repo ||= begin
+                  git_root_path = self[:repo_dir] || _find_git_root
+                  raise "Can't find Git repository" unless git_root_path
+                  Vendorificator::Repo.new( git_root_path.to_s )
+                end
+    end
+
+    def self.each_module(*modules)
+      module_paths = modules.map { |m| File.expand_path(m) }
+
+      # We don't use self[:modules].each here, because mod.run! is
+      # explicitly allowed to append to Config[:modules], and #each
+      # fails to catch up on some Ruby implementations.
+      i = 0
+      while true
+        break if i >= Vendorificator::Config[:modules].length
+        mod = Vendorificator::Config[:modules][i]
+        yield mod if
+          modules.empty? ||
+          modules.include?(mod.name) ||
+          module_paths.include?(mod.work_dir)
+        i += 1
+
+        # Add dependencies
+        work_dirs = Vendorificator::Config[:modules].map(&:work_dir)
+        Vendorificator::Config[:modules] +=
+          mod.dependencies.reject { |dep| work_dirs.include?(dep.work_dir) }
+      end
+    end
+
+    def self._find_git_root
+      self[:root_dir].ascend do |dir|
+        return dir if dir.join('.git').exist?
+      end
+    end
+    private_class_method :_find_git_root
+  end
+end

data/lib/vendorificator/hooks/chef_cookbook.rb

@@ -0,0 +1,23 @@
+require 'chef/cookbook/metadata'
+
+module Vendorificator::Hooks
+  module ChefCookbookDependencies
+    # Add required Chef cookbooks to vendor modules
+    def dependencies
+      ignored = Vendorificator::Config[:chef_cookbook_ignore_dependencies] || []
+      metadata = File.join(self.work_dir, 'metadata.rb')
+
+      unless File.exist?(metadata)
+        shell.say_status 'WARNING', "Metadata of #{name} does not exist at #{metadata}, could not gather dependencies", :red
+        return super
+      end
+
+      cbmd = Chef::Cookbook::Metadata.new
+      cbmd.from_file(metadata)
+
+      super + cbmd.dependencies.
+        reject { |name, version| ignored.include?(name) }.
+        map { |name, version| Vendorificator::Vendor::ChefCookbook.new(name) }
+    end
+  end
+end

data/lib/vendorificator/repo.rb

@@ -0,0 +1,69 @@
+require 'grit'
+
+module Vendorificator
+  class Repo < Grit::Repo
+    # True if repository doesn't contain uncommitted changes.
+    def clean?
+      # copy code from http://stackoverflow.com/a/3879077/16390
+      git.native :update_index, {}, '-q', '--ignore-submodules', '--refresh'
+      git.native :diff_files, {:raise => true}, '--quiet', '--ignore-submodules', '--'
+      git.native :diff_index, {:raise => true}, '--cached', '--quiet', 'HEAD', '--ignore-submodules', '--'
+      true
+    rescue Grit::Git::CommandFailed
+      false
+    end
+
+    # Update vendor branches & tags from an upstream repository
+    def pull(remote, options={})
+      raise RuntimeError, "Unknown remote #{remote}" unless remote_list.include?(remote)
+
+      git.fetch({}, remote)
+      git.fetch({:tags => true}, remote)
+
+      ref_rx = /^#{Regexp.quote(remote)}\//
+      remote_branches = Hash[remotes.map{|r| [$',r] if r.name =~ ref_rx }.compact]
+
+      # FIXME: should we depend on Vendorificator::Config here?
+      Vendorificator::Config.each_module do |mod|
+        remote_head = remote_branches[mod.branch_name]
+        ours = mod.head && mod.head.commit.sha
+        theirs = remote_head && remote_head.commit.sha
+
+        if remote_head
+          if not mod.head
+            say_status 'new', mod.branch_name, :yellow
+            git.branch({:track=>true}, mod.branch_name, remote_head.name) unless options[:dry_run]
+          elsif ours == theirs
+            say_status 'unchanged', mod.branch_name
+          elsif fast_forwardable?(theirs, ours)
+            say_status 'updated', mod.name, :yellow
+            unless options[:dry_run]
+              mod.in_branch do
+                git.merge({:ff_only => true}, remote_head.name)
+              end
+            end
+          elsif fast_forwardable?(ours, theirs)
+            say_status 'older', mod.branch_name
+          else
+            say_status 'complicated', mod.branch_name, :red
+            indent do
+              say 'Merge it yourself.'
+            end
+          end
+        else
+          say_status 'unknown', mod.branch_name
+        end
+      end
+
+      private
+
+      def conf
+        Vendorificator::Config
+      end
+
+      def say_status(*args)
+        conf[:shell].say_status(*args) if conf[:shell]
+      end
+    end
+  end
+end

data/lib/vendorificator/vendor/archive.rb

@@ -0,0 +1,90 @@
+require 'digest'
+require 'open-uri'
+require 'tempfile'
+require 'uri'
+
+require 'escape'
+
+require 'vendorificator/vendor'
+
+class Vendorificator::Vendor::Archive < Vendorificator::Vendor
+  arg_reader :url, :strip_root, :type, :checksum, :filename, :basename, :extname, :unpack
+  attr_reader :conjured_checksum
+
+  def initialize(name, args={}, &block)
+    no_url_given = !args[:url]
+
+    args[:url] ||= name
+    args[:filename] ||= URI::parse(args[:url]).path.split('/').last
+
+    case args[:filename]
+    when /\.(tar\.|t)gz$/
+      args[:type] ||= :targz
+      args[:unpack] ||= 'tar -xzf'
+    when /\.tar\.bz2$/
+      args[:type] ||= :tarbz2
+      args[:unpack] ||= 'tar -xjf'
+    when /\.zip$/
+      args[:type] ||= :zip
+      args[:unpack] ||= 'unzip'
+    when /\.[^\.][^\.]?[^\.]?[^\.]?$/
+      args[:type] ||=
+        begin
+          unless args[:unpack]
+            raise RuntimeError,
+            "Unknown file type #{$&.inspect}, please provide :unpack argument"
+          end
+          $&
+        end
+    else
+      args[:basename] ||= args[:filename]
+      args[:extname] ||= ''
+      unless args[:unpack] || [:targz, :tarbz2, :zip].include?(args[:type])
+        raise RuntimeError, "Unknown file type for #{args[:filename].inspect}, please provide :unpack or :type argument"
+      end
+    end
+    args[:basename] ||= $`
+    args[:extname] ||= $&
+
+    name = args[:basename] if no_url_given
+
+    super(name, args, &block)
+  end
+
+  def conjure!
+    shell.say_status :download, url
+    archive = Tempfile.new([basename, extname])
+    archive.write( open(url).read )
+    archive.close
+    @conjured_checksum = Digest::SHA256.file(archive.path).hexdigest
+    raise RuntimeError, "Checksum error" if checksum && checksum!=conjured_checksum
+    shell.say_status :unpack, filename
+    system "#{unpack} #{Escape.shell_single_word archive.path}"
+    if Dir.entries('.').length == 3 && !args[:no_strip_root]
+      root = (Dir.entries('.') - %w(.. .)).first
+      root_entries = Dir.entries(root) - %w(.. .)
+      while root_entries.include?(root)
+        FileUtils::mv root, root+"~"
+        root << "~"
+      end
+      FileUtils::mv root_entries.map { |e| File.join(root, e) }, '.'
+      FileUtils::rmdir root
+    end
+    super
+  ensure
+    archive.close
+    archive.unlink
+  end
+
+  def upstream_version
+    filename
+  end
+
+  def conjure_commit_message
+    rv = "Conjured archive #{name} from #{filename}\nOrigin: #{url}\nChecksum: #{conjured_checksum}\n"
+    rv << "Version: #{version}\n" if version
+    rv
+  end
+
+  install!
+end

data/lib/vendorificator/vendor/chef_cookbook.rb

@@ -0,0 +1,58 @@
+require 'net/http'
+require 'uri'
+require 'json'
+
+require 'vendorificator/vendor/archive'
+require 'vendorificator/hooks/chef_cookbook'
+
+class Vendorificator::Vendor::ChefCookbook < Vendorificator::Vendor::Archive
+  include Vendorificator::Hooks::ChefCookbookDependencies
+
+  @method_name = :chef_cookbook
+  @category = :cookbooks
+
+  API_PREFIX = 'http://cookbooks.opscode.com/api/v1/cookbooks/'
+
+  def initialize(name, args={}, &block)
+    args[:url] ||= true         # to avoid having name treated as url
+    args[:filename] ||= "#{name}.tgz"
+
+    super(name, args, &block)
+  end
+
+  def api_data(v=nil)
+    v = v.gsub(/[^0-9]/, '_') if v
+    @api_data ||= {}
+    @api_data[v] ||=
+      begin
+        url = "#{API_PREFIX}#{name}"
+        url << "/versions/#{v}" if v
+        JSON::load(Net::HTTP.get_response(URI.parse(url)).body)
+      end
+  end
+
+  def cookbook_data
+    @cookbook_data ||= api_data(version)
+  end
+
+  def upstream_version
+    URI::parse(api_data['latest_version']).path.split('/').last.gsub('_', '.')
+  end
+
+  def url
+    cookbook_data['file']
+  end
+
+  def conjure!
+    super
+    # Some Opscode Community tarballs include a confusing .git file,
+    # we don't want this.
+    FileUtils::rm_f '.git'
+  end
+
+  def conjure_commit_message
+    "Conjured cookbook #{name} version #{version}\nOrigin: #{url}\nChecksum: #{conjured_checksum}\n"
+  end
+
+  install!
+end

data/lib/vendorificator/vendor/git.rb

@@ -0,0 +1,47 @@
+require 'fileutils'
+
+require 'grit'
+
+require 'vendorificator/vendor'
+
+class Vendorificator::Vendor::Git < Vendorificator::Vendor
+  arg_reader :repository, :revision, :branch
+  attr_reader :module_repo, :conjured_revision
+
+  def initialize(name, args={}, &block)
+    unless args.include?(:repository)
+      args[:repository] = name
+      name = name.split('/').last.sub(/\.git$/, '')
+    end
+    super(name, args, &block)
+  end
+
+  def conjure!
+    shell.say_status :clone, repository
+    Grit::Git.new('.').clone({}, repository, '.')
+    @module_repo = Grit::Repo.new('.')
+
+    if revision
+      module_repo.git.checkout({:b => 'vendorified'}, revision)
+    elsif branch
+      module_repo.git.checkout({:b => 'vendorified'}, "origin/#{branch}")
+    end
+
+    super
+    @conjured_revision = module_repo.head.commit.id
+    FileUtils::rm_rf '.git'
+  end
+
+  def upstream_version
+    conjured_revision
+  end
+
+  def conjure_commit_message
+    rv = "Conjured git module #{name} "
+    rv << "version #{version} " if version
+    rv << "revision #{conjured_revision}"
+    rv
+  end
+
+  install!
+end