modulesync 0.6.1 → 0.7.2

data/lib/modulesync.rb

@@ -1,14 +1,25 @@
 require 'fileutils'
+require 'pathname'
 require 'modulesync/cli'
 require 'modulesync/constants'
 require 'modulesync/git'
 require 'modulesync/hook'
 require 'modulesync/renderer'
+require 'modulesync/settings'
 require 'modulesync/util'
 
 module ModuleSync
   include Constants
 
+  def self.config_defaults
+    {
+      :project_root         => 'modules/',
+      :managed_modules_conf => 'managed_modules.yml',
+      :configs              => '.',
+      :tag_pattern          => '%s'
+    }
+  end
+
   def self.local_file(config_path, file)
     "#{config_path}/#{MODULE_FILES_DIR}/#{file}"
   end
@@ -18,8 +29,8 @@ module ModuleSync
   end
 
   def self.local_files(path)
-    if File.exists?(path)
-      local_files = Find.find(path).collect { |file| file if !File.directory?(file) }.compact
+    if File.exist?(path)
+      local_files = Find.find(path).collect { |file| file unless File.directory?(file) }.compact
     else
       puts "#{path} does not exist. Check that you are working in your module configs directory or that you have passed in the correct directory with -c."
       exit
@@ -30,65 +41,100 @@ module ModuleSync
     local_files.map { |file| file.sub(/#{path}/, '') }
   end
 
-  def self.managed_modules(path, filter)
+  def self.managed_modules(path, filter, negative_filter)
     managed_modules = Util.parse_config(path)
     if managed_modules.empty?
       puts "No modules found at #{path}. Check that you specified the right configs directory containing managed_modules.yml."
       exit
     end
     managed_modules.select! { |m| m =~ Regexp.new(filter) } unless filter.nil?
+    managed_modules.select! { |m| m !~ Regexp.new(negative_filter) } unless negative_filter.nil?
     managed_modules
   end
 
-  def self.run(args)
-    cli = CLI.new
-    cli.parse_opts(args)
-    options  = cli.options
-    if options[:command] == 'update'
-      defaults = Util.parse_config("#{options[:configs]}/#{CONF_FILE}")
-
-      path = "#{options[:configs]}/#{MODULE_FILES_DIR}"
-      local_files = self.local_files(path)
-      module_files = self.module_files(local_files, path)
-
-      managed_modules = self.managed_modules("#{options[:configs]}/managed_modules.yml", options[:filter])
-
-      # managed_modules is either an array or a hash
-      managed_modules.each do |puppet_module, opts|
-        puts "Syncing #{puppet_module}"
-        unless options[:offline]
-          git_base = "#{options[:git_base]}#{options[:namespace]}"
-          Git.pull(git_base, puppet_module, options[:branch], options[:project_root], opts || {})
-        end
-        module_configs = Util.parse_config("#{options[:project_root]}/#{puppet_module}/#{MODULE_CONF_FILE}")
-        global_defaults = defaults[GLOBAL_DEFAULTS_KEY] || {}
-        module_defaults = module_configs[GLOBAL_DEFAULTS_KEY] || {}
-        files_to_manage = (module_files | defaults.keys | module_configs.keys) - [GLOBAL_DEFAULTS_KEY]
-        files_to_delete = []
-        files_to_manage.each do |file|
-          file_configs = global_defaults.merge(defaults[file] || {}).merge(module_defaults).merge(module_configs[file] || {})
-          file_configs[:puppet_module] = puppet_module
-          if file_configs['unmanaged']
-            puts "Not managing #{file} in #{puppet_module}"
-            files_to_delete << file
-          elsif file_configs['delete']
-            Renderer.remove(module_file(options['project_root'], puppet_module, file))
-          else
-            erb = Renderer.build(local_file(options[:configs], file))
-            template = Renderer.render(erb, file_configs)
-            Renderer.sync(template, "#{options[:project_root]}/#{puppet_module}/#{file}")
-          end
-        end
-        files_to_manage -= files_to_delete
-        if options[:noop]
-          Git.update_noop(puppet_module, options)
-        elsif not options[:offline]
-          Git.update(puppet_module, files_to_manage, options)
-        end
+  def self.module_name(module_name, default_namespace)
+    return [default_namespace, module_name] unless module_name.include?('/')
+    ns, mod = module_name.split('/')
+  end
+
+  def self.hook(options)
+    hook = Hook.new(HOOK_FILE, options)
+
+    case options[:hook]
+    when 'activate'
+      hook.activate
+    when 'deactivate'
+      hook.deactivate
+    end
+  end
+
+  def self.manage_file(filename, settings, options)
+    module_name = settings.additional_settings[:puppet_module]
+    configs = settings.build_file_configs(filename)
+    if configs['delete']
+      Renderer.remove(module_file(options[:project_root], module_name, filename))
+    else
+      templatename = local_file(options[:configs], filename)
+      begin
+        erb = Renderer.build(templatename)
+        template = Renderer.render(erb, configs)
+        Renderer.sync(template, module_file(options[:project_root], module_name, filename))
+      rescue
+        STDERR.puts "Error while rendering #{filename}"
+        raise
       end
-    elsif options[:command] == 'hook'
-      Hook.hook(args[1], options)
     end
   end
 
+  def self.manage_module(puppet_module, module_files, module_options, defaults, options)
+    puts "Syncing #{puppet_module}"
+    namespace, module_name = self.module_name(puppet_module, options[:namespace])
+    unless options[:offline]
+      git_base = options[:git_base]
+      git_uri = "#{git_base}#{namespace}"
+      Git.pull(git_uri, module_name, options[:branch], options[:project_root], module_options || {})
+    end
+    module_configs = Util.parse_config("#{options[:project_root]}/#{module_name}/#{MODULE_CONF_FILE}")
+    settings = Settings.new(defaults[GLOBAL_DEFAULTS_KEY] || {},
+                            defaults,
+                            module_configs[GLOBAL_DEFAULTS_KEY] || {},
+                            module_configs,
+                            :puppet_module => module_name,
+                            :git_base => git_base,
+                            :namespace => namespace)
+    settings.unmanaged_files(module_files).each do |filename|
+      puts "Not managing #{filename} in #{module_name}"
+    end
+
+    files_to_manage = settings.managed_files(module_files)
+    files_to_manage.each { |filename| manage_file(filename, settings, options) }
+
+    if options[:noop]
+      Git.update_noop(module_name, options)
+    elsif !options[:offline]
+      Git.update(module_name, files_to_manage, options)
+    end
+  end
+
+  def self.update(options)
+    options = config_defaults.merge(options)
+    defaults = Util.parse_config("#{options[:configs]}/#{CONF_FILE}")
+
+    path = "#{options[:configs]}/#{MODULE_FILES_DIR}"
+    local_files = self.local_files(path)
+    module_files = self.module_files(local_files, path)
+
+    managed_modules = self.managed_modules("#{options[:configs]}/managed_modules.yml", options[:filter], options[:negative_filter])
+
+    # managed_modules is either an array or a hash
+    managed_modules.each do |puppet_module, module_options|
+      begin
+        manage_module(puppet_module, module_files, module_options, defaults, options)
+      rescue
+        STDERR.puts "Error while updating #{puppet_module}"
+        raise unless options[:skip_broken]
+        puts "Skipping #{puppet_module} as update process failed"
+      end
+    end
+  end
 end

data/lib/modulesync/cli.rb

@@ -1,124 +1,64 @@
-require 'optparse'
+require 'thor'
+require 'modulesync'
 require 'modulesync/constants'
 require 'modulesync/util'
 
 module ModuleSync
   class CLI
-    include Constants
+    class Hook < Thor
+      class_option :project_root, :aliases => '-c', :desc => 'Path used by git to clone modules into. Defaults to "modules"', :default => 'modules'
+      class_option :hook_args, :aliases => '-a', :desc => 'Arguments to pass to msync in the git hook'
 
-    def defaults
-      {
-        :namespace            => 'puppetlabs',
-        :branch               => 'master',
-        :git_base             => 'git@github.com:',
-        :managed_modules_conf => 'managed_modules.yml',
-        :configs              => '.',
-        :tag_pattern          => '%s',
-        :project_root         => './modules',
-      }
-    end
+      desc 'activate', 'Activate the git hook.'
+      def activate
+        config = { :command => 'hook' }.merge(options)
+        config[:hook] = 'activate'
+        ModuleSync.hook(config)
+      end
 
-    def commands_available
-      [
-        'update',
-        'hook',
-      ]
+      desc 'deactivate', 'Deactivate the git hook.'
+      def deactivate
+        config = { :command => 'hook' }.merge(options)
+        config[:hook] = 'deactivate'
+        ModuleSync.hook(config)
+      end
     end
 
-    def fail(message)
-      puts @options[:help]
-      puts message
-      exit
-    end
+    class Base < Thor
+      include Constants
 
-    def parse_opts(args)
-      @options = defaults
-      @options.merge!(Hash.transform_keys_to_symbols(Util.parse_config(MODULESYNC_CONF_FILE)))
-      @options[:command] = args[0] if commands_available.include?(args[0])
-      opt_parser = OptionParser.new do |opts|
-        opts.banner = "Usage: msync update [-m <commit message>] [-c <directory> ] [--offline] [--noop] [--bump] [--changelog] [--tag] [--tag-pattern <tag_pattern>] [-p <project_root> [-n <namespace>] [-b <branch>] [-r <branch>] [-f <filter>] | hook activate|deactivate [-c <directory> ] [-n <namespace>] [-b <branch>]"
-        opts.on('-m', '--message <msg>',
-                'Commit message to apply to updated modules') do |msg|
-          @options[:message] = msg
-        end
-        opts.on('-n', '--namespace <url>',
-                'Remote github namespace (user or organization) to clone from and push to. Defaults to puppetlabs') do |namespace|
-          @options[:namespace] = namespace
-        end
-        opts.on('-c', '--configs <directory>',
-                'The local directory or remote repository to define the list of managed modules, the file templates, and the default values for template variables.') do |configs|
-          @options[:configs] = configs
-        end
-        opts.on('-b', '--branch <branch>',
-                'Branch name to make the changes in. Defaults to "master"') do |branch|
-          @options[:branch] = branch
-        end
-        opts.on('-p', '--project-root <path>',
-                'Path used by git to clone modules into. Defaults to "modules"') do |project_root|
-          @options[:project_root] = project_root
-        end
-        opts.on('-r', '--remote-branch <branch>',
-                'Remote branch name to push the changes to. Defaults to the branch name') do |branch|
-          @options[:remote_branch] = branch
-        end
-        opts.on('-f', '--filter <filter>',
-                'A regular expression to filter repositories to update.') do |filter|
-          @options[:filter] = filter
-        end
-        opts.on('--amend',
-                'Amend previous commit') do |msg|
-          @options[:amend] = true
-        end
-        opts.on('--force',
-                'Force push amended commit') do |msg|
-          @options[:force] = true
-        end
-        opts.on('--noop',
-                'No-op mode') do |msg|
-          @options[:noop] = true
-        end
-        opts.on('--offline',
-                'Do not run git command. Helpful if you have existing repositories locally.') do |msg|
-          @options[:offline] = true
-        end
-        opts.on('--bump',
-                'Bump module version to the next minor') do |msg|
-          @options[:bump] = true
-        end
-        opts.on('--changelog',
-                'Update CHANGELOG.md if version was bumped') do |msg|
-          @options[:changelog] = true
-        end
-        opts.on('--tag',
-                'Git tag with the current module version') do |msg|
-          @options[:tag] = true
-        end
-        opts.on('--tag-pattern',
-                'The pattern to use when tagging releases.') do |pattern|
-          @options[:tag_pattern] = pattern
-        end
-        @options[:help] = opts.help
-      end.parse!
+      class_option :project_root, :aliases => '-c', :desc => 'Path used by git to clone modules into. Defaults to "modules"', :default => 'modules'
+      class_option :git_base, :desc => 'Specify the base part of a git URL to pull from', :default => 'git@github.com:'
+      class_option :namespace, :aliases => '-n', :desc => 'Remote github namespace (user or organization) to clone from and push to. Defaults to puppetlabs', :default => 'puppetlabs'
+      class_option :filter, :aliases => '-f', :desc => 'A regular expression to select repositories to update.'
+      class_option :negative_filter, :aliases => '-x', :desc => 'A regular expression to skip repositories.'
+      class_option :branch, :aliases => '-b', :desc => 'Branch name to make the changes in. Defaults to master.', :default => 'master'
 
-      @options.fetch(:message) do
-        if @options[:command] == 'update' && ! @options[:noop] && ! @options[:amend] && ! @options[:offline]
-          fail("A commit message is required unless using noop or offline.")
-        end
-      end
+      desc 'update', 'Update the modules in managed_modules.yml'
+      option :message, :aliases => '-m', :desc => 'Commit message to apply to updated modules. Required unless running in noop mode.'
+      option :configs, :aliases => '-c', :desc => 'The local directory or remote repository to define the list of managed modules, the file templates, and the default values for template variables.'
+      option :remote_branch, :aliases => '-r', :desc => 'Remote branch name to push the changes to. Defaults to the branch name.'
+      option :skip_broken, :type => :boolean, :aliases => '-s', :desc => 'Process remaining modules if an error is found', :default => false
+      option :amend, :type => :boolean, :desc => 'Amend previous commit', :default => false
+      option :force, :type => :boolean, :desc => 'Force push amended commit', :default => false
+      option :noop, :type => :boolean, :desc => 'No-op mode', :default => false
+      option :offline, :type => :boolean, :desc => 'Do not run any Git commands. Allows the user to manage Git outside of ModuleSync.', :default => false
+      option :bump, :type => :boolean, :desc => 'Bump module version to the next minor', :default => false
+      option :changelog, :type => :boolean, :desc => 'Update CHANGELOG.md if version was bumped', :default => false
+      option :tag, :type => :boolean, :desc => 'Git tag with the current module version', :default => false
+      option :tag_pattern, :desc => 'The pattern to use when tagging releases.'
 
-      @options.fetch(:command) do
-        fail("A command is required.")
+      def update
+        config = { :command => 'update' }.merge(options)
+        config.merge!(Util.parse_config(MODULESYNC_CONF_FILE))
+        config = Util.symbolize_keys(config)
+        raise Thor::Error, 'No value provided for required option "--message"' unless config[:noop] || config[:message] || config[:offline]
+        config[:git_opts] = { 'amend' => config[:amend], 'force' => config[:force] }
+        ModuleSync.update(config)
       end
 
-      if @options[:command] == 'hook' &&
-           (! args.include?('activate') && ! args.include?('deactivate'))
-        fail("You must activate or deactivate the hook.")
-      end
-
-    end
-
-    def options
-      @options
+      desc 'hook', 'Activate or deactivate a git hook.'
+      subcommand 'hook', ModuleSync::CLI::Hook
     end
   end
 end

data/lib/modulesync/constants.rb

@@ -1,10 +1,10 @@
 module ModuleSync
   module Constants
-    MODULE_FILES_DIR     = 'moduleroot/'
-    CONF_FILE            = 'config_defaults.yml'
-    MODULE_CONF_FILE     = '.sync.yml'
-    MODULESYNC_CONF_FILE = 'modulesync.yml'
-    HOOK_FILE            = '.git/hooks/pre-push'
+    MODULE_FILES_DIR     = 'moduleroot/'.freeze
+    CONF_FILE            = 'config_defaults.yml'.freeze
+    MODULE_CONF_FILE     = '.sync.yml'.freeze
+    MODULESYNC_CONF_FILE = 'modulesync.yml'.freeze
+    HOOK_FILE            = '.git/hooks/pre-push'.freeze
     GLOBAL_DEFAULTS_KEY  = :global
   end
 end

data/lib/modulesync/git.rb

@@ -6,38 +6,41 @@ module ModuleSync
     include Constants
 
     def self.remote_branch_exists?(repo, branch)
-      repo.branches.remote.collect { |b| b.name }.include?(branch)
+      repo.branches.remote.collect(&:name).include?(branch)
     end
 
     def self.local_branch_exists?(repo, branch)
-      repo.branches.local.collect { |b| b.name }.include?(branch)
+      repo.branches.local.collect(&:name).include?(branch)
+    end
+
+    def self.remote_branch_differ?(repo, local_branch, remote_branch)
+      !remote_branch_exists?(repo, remote_branch) ||
+        repo.diff("#{local_branch}..origin/#{remote_branch}").any?
     end
 
     def self.switch_branch(repo, branch)
-      unless repo.branch.name == branch
-        if local_branch_exists?(repo, branch)
-          puts "Switching to branch #{branch}"
-          repo.checkout(branch)
-        elsif remote_branch_exists?(repo, branch)
-          puts "Creating local branch #{branch} from origin/#{branch}"
-          repo.checkout("origin/#{branch}")
-          repo.branch(branch).checkout
-        else
-          repo.checkout('origin/master')
-          puts "Creating new branch #{branch}"
-          repo.branch(branch).checkout
-        end
+      return if repo.branch.name == branch
+
+      if local_branch_exists?(repo, branch)
+        puts "Switching to branch #{branch}"
+        repo.checkout(branch)
+      elsif remote_branch_exists?(repo, branch)
+        puts "Creating local branch #{branch} from origin/#{branch}"
+        repo.checkout("origin/#{branch}")
+        repo.branch(branch).checkout
+      else
+        repo.checkout('origin/master')
+        puts "Creating new branch #{branch}"
+        repo.branch(branch).checkout
       end
     end
 
     def self.pull(git_base, name, branch, project_root, opts)
-      if ! Dir.exists?(project_root)
-        Dir.mkdir(project_root)
-      end
+      Dir.mkdir(project_root) unless Dir.exist?(project_root)
 
       # Repo needs to be cloned in the cwd
-      if ! Dir.exists?("#{project_root}/#{name}") || ! Dir.exists?("#{project_root}/#{name}/.git")
-        puts "Cloning repository fresh"
+      if !Dir.exist?("#{project_root}/#{name}") || !Dir.exist?("#{project_root}/#{name}/.git")
+        puts 'Cloning repository fresh'
         remote = opts[:remote] || (git_base.start_with?('file://') ? "#{git_base}/#{name}" : "#{git_base}/#{name}.git")
         local = "#{project_root}/#{name}"
         puts "Cloning from #{remote}"
@@ -52,20 +55,18 @@ module ModuleSync
           repo.fetch
           repo.reset_hard
           switch_branch(repo, branch)
-          if remote_branch_exists?(repo, branch)
-            repo.pull('origin', branch)
-          end
+          repo.pull('origin', branch) if remote_branch_exists?(repo, branch)
         end
       end
     end
 
     def self.update_changelog(repo, version, message, module_root)
       changelog = "#{module_root}/CHANGELOG.md"
-      if File.exists?(changelog)
+      if File.exist?(changelog)
         puts "Updating #{changelog} for version #{version}"
         changes = File.readlines(changelog)
         File.open(changelog, 'w') do |f|
-          date = Time.now.strftime("%Y-%m-%d")
+          date = Time.now.strftime('%Y-%m-%d')
           f.puts "## #{date} - Release #{version}\n\n"
           f.puts "#{message}\n\n"
           # Add old lines again
@@ -73,15 +74,15 @@ module ModuleSync
         end
         repo.add('CHANGELOG.md')
       else
-        puts "No CHANGELOG.md file found, not updating."
+        puts 'No CHANGELOG.md file found, not updating.'
       end
     end
 
-    def self.bump(repo, m, message, module_root, changelog=false)
+    def self.bump(repo, m, message, module_root, changelog = false)
       new = m.bump!
       puts "Bumped to version #{new}"
       repo.add('metadata.json')
-      self.update_changelog(repo, new, message, module_root) if changelog
+      update_changelog(repo, new, message, module_root) if changelog
       repo.commit("Release version #{new}")
       repo.push
       new
@@ -98,43 +99,40 @@ module ModuleSync
     def self.update(name, files, options)
       module_root = "#{options[:project_root]}/#{name}"
       message = options[:message]
-      if options[:remote_branch]
-        branch = "#{options[:branch]}:#{options[:remote_branch]}"
-      else
-        branch = options[:branch]
-      end
       repo = ::Git.open(module_root)
       repo.branch(options[:branch]).checkout
       files.each do |file|
         if repo.status.deleted.include?(file)
           repo.remove(file)
-        else
+        elsif File.exist?("#{module_root}/#{file}")
           repo.add(file)
         end
       end
       begin
         opts_commit = {}
         opts_push = {}
-        if options[:amend]
-          opts_commit = {:amend => true}
-        end
-        if options[:force]
-          opts_push = {:force => true}
-        end
+        opts_commit = { :amend => true } if options[:amend]
+        opts_push = { :force => true } if options[:force]
         if options[:pre_commit_script]
           script = "#{File.dirname(File.dirname(__FILE__))}/../contrib/#{options[:pre_commit_script]}"
-          %x[#{script} #{module_root}]
+          `#{script} #{module_root}`
         end
         repo.commit(message, opts_commit)
-        repo.push('origin', branch, opts_push)
+        if options[:remote_branch]
+          if remote_branch_differ?(repo, options[:branch], options[:remote_branch])
+            repo.push('origin', "#{options[:branch]}:#{options[:remote_branch]}", opts_push)
+          end
+        else
+          repo.push('origin', options[:branch], opts_push)
+        end
         # Only bump/tag if pushing didn't fail (i.e. there were changes)
         m = Blacksmith::Modulefile.new("#{module_root}/metadata.json")
         if options[:bump]
-          new = self.bump(repo, m, message, module_root, options[:changelog])
-          self.tag(repo, new, options[:tag_pattern]) if options[:tag]
+          new = bump(repo, m, message, module_root, options[:changelog])
+          tag(repo, new, options[:tag_pattern]) if options[:tag]
         end
       rescue ::Git::GitExecuteError => git_error
-        if git_error.message.include? "nothing to commit, working directory clean"
+        if git_error.message =~ /working (directory|tree) clean/
           puts "There were no files to update in #{name}. Not committing."
         else
           puts git_error
@@ -148,12 +146,8 @@ module ModuleSync
     # https://github.com/schacon/ruby-git/issues/130
     def self.untracked_unignored_files(repo)
       ignore_path = "#{repo.dir.path}/.gitignore"
-      if File.exists?(ignore_path)
-        ignored = File.open(ignore_path).read.split
-      else
-        ignored = []
-      end
-      repo.status.untracked.keep_if{|f,_| !ignored.any?{|i| File.fnmatch(i, f)}}
+      ignored = File.exist?(ignore_path) ? File.open(ignore_path).read.split : []
+      repo.status.untracked.keep_if { |f, _| !ignored.any? { |i| File.fnmatch(i, f) } }
     end
 
     def self.update_noop(name, options)
@@ -162,13 +156,13 @@ module ModuleSync
       repo = ::Git.open("#{options[:project_root]}/#{name}")
       repo.branch(options[:branch]).checkout
 
-      puts "Files changed: "
+      puts 'Files changed:'
       repo.diff('HEAD', '--').each do |diff|
         puts diff.patch
       end
 
-      puts "Files added: "
-      untracked_unignored_files(repo).each do |file,_|
+      puts 'Files added:'
+      untracked_unignored_files(repo).each do |file, _|
         puts file
       end