modulesync 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5b33cdfbc0df1b30570ffadbd78e6bbcec9ba8b6
+  data.tar.gz: 5044a6754a320ec00e0ebf4bb7cba71087da9e02
+SHA512:
+  metadata.gz: 529d0ed35ec73c722cb607980133637851e1cfda52d564542ec3d0393bae5bcce62a5c72dff96b430654c2fff69ee263c17f4d3a9572924d3cd1342f89aacf74
+  data.tar.gz: 563cf2c60a67b435a9ff7481f37288236166ea1fa66d30087becf15ef932335ceda65685aecfd8d8625172a4594e8265a34ece2c2ab4d6fe555cf85099d78ddf

data/.gitignore

@@ -0,0 +1,2 @@
+modules/
+*.gem

data/Gemfile

@@ -0,0 +1,3 @@
+source ENV['GEM_SOURCE'] || 'https://rubygems.org'
+
+gem 'git'

data/Gemfile.lock

@@ -0,0 +1,10 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    git (1.2.7)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  git

data/LICENSE

@@ -0,0 +1,17 @@
+   Puppet Module Sync
+
+   Copyright (C) 2014 Puppet Labs Inc
+
+   Puppet Labs can be contacted at: info@puppetlabs.com
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,220 @@
+ModuleSync
+===========
+
+Table of Contents
+-----------------
+
+1. [Usage TLDR](#usage-tldr)
+2. [Overview](#overview)
+3. [How it works](#how-it-works)
+4. [Installing](#installing)
+5. [Workflow](#workflow)
+6. [The Templates](#the-templates)
+
+Usage TLDR
+----------
+
+```
+gem install modulesync
+msync --help
+```
+
+Overview
+--------
+
+ModuleSync was written as a simple script with ERB templates to help the
+Puppet Labs module engineers manage the zoo of Puppet modules on GitHub, and
+has now been restructured and generalized to be used within other
+organizations. Puppet modules within an organization tend to have a number of
+meta-files that are identical or very similar between modules, such as the
+Gemfile, .travis.yml, .gitignore, or spec\_helper.rb. If a file needs to
+change in one module, it likely needs to change in the same way in every other
+module that the organization manages.
+
+One approach to this problem is to use sed in a bash for loop on the modules to
+make a single change across every module. This approach falls short if there is
+a single file that is purposefully different than the others, but still needs
+to be managed. Moreover, this approach does not help if two files are
+structured differently but need to be changed with the same meaning; for
+instance, if the .travis.yml of one module uses a list of environments to
+include, and another uses a matrix of includes with a list of excludes, adding
+a test environment to both modules will require entirely different approaches.
+
+ModuleSync provides the advantage of defining a standard template for each
+file to follow, so it becomes clear what a file is supposed to look like. Two
+files with the same semantics should also have the same syntax. A difference
+between two files should have clear reasons, and old cruft should not be left
+in files of one module as the files of another module march forward.
+
+Another advantage of ModuleSync is the ability to run in no-op mode, which
+makes local changes and shows the diffs, but does not make permanent changes in
+the remote repository.
+
+How It Works
+------------
+
+ModuleSync is a gem that uses the GitHub workflow to clone, update, and push module
+repositories. It expects to be activated from a directory containing
+configuration for modulesync and the modules, or you can pass it the location
+of this configuration directory. [The configuration for the Puppet Labs
+modules](https://github.com/puppetlabs/modulesync\_configs), can be used as an
+example for your own configuration. The configuration directory contains a
+directory called moduleroot which mirrors the structure of a module. The files
+in the moduleroot could be flat files or ERB templates. The templates are
+rendered using values from a file called config\_defaults.yml in the root (not
+moduleroot) of the configuration directory. The default values can be
+overridden or extended by adding a file called .sync.yml to the module itself.
+This allows us to, for example, have a set of "required" gems that are added
+to all Gemfiles, and a set of "optional" gems that a single module might add.
+
+The list of modules to manage is in managed\_modules.yml in the configuration
+directory. This lists just the GitHub names of the modules to be managed.
+
+ModuleSync can be called from the command line with parameters to change the
+branch you're working on or the remote to clone from and push to. You can also
+define these parameters in a file named modulesync.yml in the configuration
+directory.
+
+Installing
+----------
+
+```
+gem install modulesync
+```
+
+For developers:
+
+```
+gem build modulesync.gemspec
+gem install modulesync-*.gem
+```
+
+Workflow
+--------
+
+### Default mode
+
+With no additional arguments, ModuleSync clones modules from the puppetlabs
+github organization and pushes to the master branch.
+
+#### Make changes
+
+Make changes to a file in the moduleroot. For sanity's sake you should commit
+and push these changes, but in this mode the update will be rendered from the
+state of the files locally. Run `msync update` from the root of the
+configuration directory (not moduleroot), or use -c <relative path> to point
+it to the location of the configuration directory.
+
+#### Dry-run
+
+Do a dry-run to see what files will be changed, added and removed. This clones
+the modules to `modules/<namespace>-<modulename>` in the current working, or if
+the modules are already cloned, does an effective `git fetch origin; git
+checkout master; git reset --hard origin/master` on the modules. Don't run
+modulesync if the current working directory contains a modules/ directory with
+changes you want to keep. The dry-run makes local changes there, but does not
+commit or push changes. It is still destructive in that it overwrites local
+changes.
+
+```
+msync update --noop 
+```
+
+#### Damage mode
+
+Make changes for real and push them back to master. This operates on the
+pre-cloned modules from the dry-run or clones them fresh if the modules aren't
+found.
+
+```
+msync update
+```
+
+#### Automating Updates
+
+You can install a pre-push git hook to automatically clone, update, and push
+modules upon pushing changes to the configuration directory. This does not
+include a noop mode.
+
+```
+msync hook activate
+```
+
+If you have activated the hook but want to make changes to the configuration
+directory (such as changes to managed_modules.yml or modulesync.yml) without
+touching the modules, you can deactivate the hook.
+
+```
+msync hook deactivate
+```
+
+### Using Forks and Non-master branches
+
+The default functionality is to run ModuleSync on the puppetlabs modules, but
+you can use this on your own organization's modules. This functionality also
+applies if you want to work on a fork of the puppetlabs modules or work on a
+non-master branch of any organization's modules. ModuleSync does not support
+cloning from one remote and pushing to another, you are expected to fork
+manually. It does not yet support automating pull requests (coming soon).
+
+#### Dry-run
+
+If you dry-run before doing the live update, you need to specify what namespace
+to clone from because the live update will not re-clone if the modules are
+already cloned. The namespace should be your fork, not the upstream module. The
+format should be the SSH or HTTP prefix of the full URL minus the module name
+itself.
+
+```
+msync update -n git@github.com:puppetlabs --noop
+```
+
+#### Damage mode
+
+You don't technically need to specify the namespace if the modules are already
+cloned from the dry-run, but it doesn't hurt. You do need to specify the
+namespace if the modules are not pre-cloned. You need to specify a branch to
+push to if you are not pushing to master.
+
+```
+msync update -n git@github.com:puppetlabs -b sync_branch
+```
+
+#### Configuring ModuleSync defaults
+
+If you're not using the puppetlabs modules or only ever pushing to a fork of
+them, then specifying the namespace and branch every time you use ModuleSync
+probably seems excessive. You can create a file called modulesync.yml in the
+configuration directory that provides these arguments automatically. This file
+has a form such as:
+
+```
+---
+namespace: mygithubusername
+branch: modulesyncbranch
+```
+
+Then you can run ModuleSync without extra arguments:
+
+```
+msync update --noop
+msync update
+```
+
+#### Automating updates
+
+If you install a git hook, you need to tell it what remote and branch to push
+to. This may not work properly if you already have the modules cloned from a
+different remote. The hook will also look in modulesync.yml for default
+arguments.
+
+```
+msync hook activate -n git@github.com:puppetlabs -b sync_branch
+```
+
+The Templates
+-------------
+
+See the [modulesync\_configs](https://github.com/puppetlabs/modulesync_configs)
+repository for an explanation of the templates that Puppet Labs uses on its
+modules.

data/bin/msync

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+lib = File.expand_path('../../lib', __FILE__)
+$:.unshift(lib) unless $:.include?(lib)
+
+require 'modulesync'
+
+ModuleSync.run(ARGV)

data/lib/modulesync/cli.rb

@@ -0,0 +1,81 @@
+require 'optparse'
+require 'modulesync/constants'
+require 'modulesync/util'
+
+module ModuleSync
+  class CLI
+    include Constants
+
+    def defaults
+      {
+        :namespace            => 'puppetlabs',
+        :branch               => 'master',
+        :managed_modules_conf => 'managed_modules.yml',
+        :configs              => '.',
+      }
+    end
+
+    def commands_available
+      [
+        'update',
+        'hook',
+      ]
+    end
+
+    def fail(message)
+      puts @options[:help]
+      puts message
+      exit
+    end
+
+    def parse_opts(args)
+      @options = defaults
+      @options.merge!(Hash.transform_keys_to_symbols(Util.parse_config(MODULESYNC_CONF_FILE)))
+      @options[:command] = args[0] if commands_available.include?(args[0])
+      opt_parser = OptionParser.new do |opts|
+        opts.banner = "Usage: msync update [-m <commit message>] [-c <directory> ] [--noop] [-n <namespace>] [-b <branch>] | hook activate|deactivate [-c <directory> ] [-n <namespace>] [-b <branch>]"
+        opts.on('-m', '--message <msg>',
+                'Commit message to apply to updated modules') do |msg|
+          @options[:message] = msg
+        end
+        opts.on('-n', '--namespace <url>',
+                'Remote github namespace (user or organization) to clone from and push to. Defaults to puppetlabs') do |namespace|
+          @options[:namespace] = namespace
+        end
+        opts.on('-c', '--configs <directory>',
+                'The local directory or remote repository to define the list of managed modules, the file templates, and the default values for template variables.') do |configs|
+          @options[:configs] = configs
+        end
+        opts.on('-b', '--branch <branch>',
+                'Branch name to make the changes in. Defaults to "master"') do |branch|
+          @options[:branch] = branch
+        end
+        opts.on('--noop',
+                'No-op mode') do |msg|
+          @options[:noop] = true
+        end
+        @options[:help] = opts.help
+      end.parse!
+
+      @options.fetch(:message) do
+        if @options[:command] == 'update' && ! @options[:noop]
+          fail("A commit message is required unless using noop.")
+        end
+      end
+
+      @options.fetch(:command) do
+        fail("A command is required.")
+      end
+
+      if @options[:command] == 'hook' &&
+           (! args.include?('activate') && ! args.include?('deactivate'))
+        fail("You must activate or deactivate the hook.")
+      end
+
+    end
+
+    def options
+      @options
+    end
+  end
+end

data/lib/modulesync/constants.rb

@@ -0,0 +1,11 @@
+module ModuleSync
+  module Constants
+    MODULE_FILES_DIR     = 'moduleroot/'
+    CONF_FILE            = 'config_defaults.yml'
+    MODULE_CONF_FILE     = '.sync.yml'
+    MODULESYNC_CONF_FILE = 'modulesync.yml'
+    PROJ_ROOT            = './modules'
+    ENDPOINT             = 'git@github.com'
+    HOOK_FILE            = '.git/hooks/pre-push'
+  end
+end

data/lib/modulesync/git.rb

@@ -0,0 +1,81 @@
+require 'git'
+
+module ModuleSync
+  module Git
+    include Constants
+
+    def self.pull(org, name)
+      if ! Dir.exists?(PROJ_ROOT)
+        Dir.mkdir(PROJ_ROOT)
+      end
+
+      # Repo needs to be cloned in the cwd
+      if ! Dir.exists?("#{PROJ_ROOT}/#{name}") || ! Dir.exists?("#{PROJ_ROOT}/#{name}/.git")
+        puts "Cloning repository fresh"
+        remote = "#{ENDPOINT}:#{org}/#{name}.git"
+        local = "#{PROJ_ROOT}/#{name}"
+        repo = ::Git.clone(remote, local)
+
+      # Repo already cloned, check out master and override local changes
+      else
+        puts "Overriding any local changes to repositories in #{PROJ_ROOT}"
+        repo = ::Git.open("#{PROJ_ROOT}/#{name}")
+        repo.branch('master').checkout
+        repo.reset_hard
+        repo.pull
+      end
+    end
+
+    # Git add/rm, git commit, git push
+    def self.update(name, files, message, branch)
+      repo = ::Git.open("#{PROJ_ROOT}/#{name}")
+      repo.branch(branch).checkout
+      files.each do |file|
+        if repo.status.deleted.include?(file)
+          repo.remove(file)
+        else
+          repo.add(file)
+        end
+      end
+      begin
+        repo.commit(message)
+        #repo.push
+      rescue ::Git::GitExecuteError => git_error
+        if git_error.message.include? "nothing to commit, working directory clean"
+          puts "There were no files to update in #{name}. Not committing."
+        else
+          puts git_error
+          exit
+        end
+      end
+    end
+
+    # Needed because of a bug in the git gem that lists ignored files as
+    # untracked under some circumstances
+    # https://github.com/schacon/ruby-git/issues/130
+    def self.untracked_unignored_files(repo)
+      ignored = File.open("#{repo.dir.path}/.gitignore").read.split
+      repo.status.untracked.keep_if{|f,_| !ignored.any?{|i| f.match(/#{i}/)}}
+    end
+
+    def self.update_noop(name, branch)
+      puts "Using no-op. Files in #{name} may be changed but will not be committed."
+
+      repo = ::Git.open("#{PROJ_ROOT}/#{name}")
+      repo.branch(branch).checkout
+
+      puts "Files changed: "
+      repo.diff('HEAD', '--').each do |diff|
+        puts diff.patch
+      end
+
+      puts "Files added: "
+      untracked_unignored_files(repo).each do |file,_|
+        puts file
+      end
+
+      puts "\n\n"
+      puts '--------------------------------'
+    end
+  end
+end

data/lib/modulesync/hook.rb

@@ -0,0 +1,36 @@
+module ModuleSync
+  module Hook
+    include Constants
+
+    def self.activate(args)
+      repo = args[:configs]
+      hook_args = ''
+      hook_args <<= " -n #{args[:namespace]}" if args[:namespace]
+      hook_args <<= " -b #{args[:branch]}" if args[:branch]
+      hook = <<EOF
+#!/usr/bin/env bash
+
+current_branch=\`git symbolic-ref HEAD | sed -e 's,.*/\(.*\),\1,'\`
+git_dir=\`git rev-parse --show-toplevel\`
+message=\`git log -1 --format=%B\`
+msync -m "\$message"#{hook_args}
+EOF
+      File.open("#{repo}/#{HOOK_FILE}", 'w') do |file|
+        file.write(hook)
+      end
+    end
+
+    def self.deactivate(repo)
+      hook_path = "#{repo}/#{HOOK_FILE}"
+      File.delete(hook_path) if File.exists?(hook_path)
+    end
+
+    def self.hook(command, args)
+      if (command == 'activate')
+        activate(args)
+      else
+        deactivate(args[:configs])
+      end
+    end
+  end
+end

data/lib/modulesync/renderer.rb

@@ -0,0 +1,41 @@
+require 'erb'
+require 'find'
+
+module ModuleSync
+  module Renderer
+
+    class ForgeModuleFile
+      def initialize(configs= {})
+        @configs = configs
+      end
+    end
+
+    def self.build(from_erb_template)
+      erb_obj = ERB.new(File.read(from_erb_template), nil, '-')
+      erb_obj.filename = from_erb_template.chomp('.erb')
+      erb_obj.def_method(ForgeModuleFile, 'render()')
+      erb_obj
+    end
+
+    def self.remove(file)
+      if File.exists?(file)
+        File.delete(file)
+      end
+    end
+
+    def self.render(template, configs = {})
+      ForgeModuleFile.new(configs).render()
+    end
+
+    def self.sync(template, to_file)
+      path = to_file.rpartition('/').first
+      if(! path.empty?)
+        FileUtils.mkdir_p(path)
+      end
+      File.open(to_file, 'w') do |file|
+        file.write(template)
+      end
+    end
+
+  end
+end

data/lib/modulesync/util.rb

@@ -0,0 +1,23 @@
+require 'yaml'
+
+module ModuleSync
+  module Util
+
+    def self.parse_config(config_file)
+      if File.exist?(config_file)
+        YAML.load_file(config_file) || {}
+      else
+        {}
+      end
+    end
+  end
+end
+
+class Hash
+  #take keys of hash and transform those to a symbols
+  def self.transform_keys_to_symbols(value)
+    return value if not value.is_a?(Hash)
+    hash = value.inject({}){|memo,(k,v)| memo[k.to_sym] = Hash.transform_keys_to_symbols(v); memo}
+    return hash
+  end
+end

data/lib/modulesync.rb

@@ -0,0 +1,86 @@
+require 'fileutils'
+require 'modulesync/cli'
+require 'modulesync/constants'
+require 'modulesync/git'
+require 'modulesync/hook'
+require 'modulesync/renderer'
+require 'modulesync/util'
+
+module ModuleSync
+  include Constants
+
+  def self.local_file(config_path, file)
+    "#{config_path}/#{MODULE_FILES_DIR}/#{file}"
+  end
+
+  def self.module_file(puppet_module, file)
+    "#{PROJ_ROOT}/#{puppet_module}/#{file}"
+  end
+
+  def self.local_files(path)
+    if File.exists?(path)
+      local_files = Find.find(path).collect { |file| file if !File.directory?(file) }.compact
+    else
+      puts "#{path} does not exist. Check that you are working in your module configs directory or that you have passed in the correct directory with -c."
+      exit
+    end
+  end
+
+  def self.module_files(local_files, path)
+    local_files.map { |file| file.sub(/#{path}/, '') }
+  end
+
+  def self.managed_modules(path)
+    managed_modules = Util.parse_config(path)
+    if managed_modules.empty?
+      puts "No modules found at #{path}. Check that you specified the right configs directory containing managed_modules.yml."
+      exit
+    end
+    managed_modules
+  end
+
+  def self.run(args)
+    cli = CLI.new
+    cli.parse_opts(args)
+    options  = cli.options
+    if options[:command] == 'update'
+      defaults = Util.parse_config("#{options[:configs]}/#{CONF_FILE}")
+
+      path = "#{options[:configs]}/#{MODULE_FILES_DIR}"
+      local_files = self.local_files(path)
+      module_files = self.module_files(local_files, path)
+
+      managed_modules = self.managed_modules("#{options[:configs]}/managed_modules.yml")
+
+      managed_modules.each do |puppet_module|
+        puts "Syncing #{puppet_module}"
+        Git.pull(options[:namespace], puppet_module)
+        module_configs = Util.parse_config("#{PROJ_ROOT}/#{puppet_module}/#{MODULE_CONF_FILE}")
+        files_to_manage = module_files | defaults.keys | module_configs.keys
+        files_to_delete = []
+        files_to_manage.each do |file|
+          file_configs = (defaults[file] || {}).merge(module_configs[file] || {})
+          if file_configs['unmanaged']
+            puts "Not managing #{file} in #{puppet_module}"
+            files_to_delete << file
+          elsif file_configs['delete']
+            Renderer.remove(module_file(puppet_module, file))
+          else
+            erb = Renderer.build(local_file(options[:configs], file))
+            template = Renderer.render(erb, file_configs)
+            Renderer.sync(template, "#{PROJ_ROOT}/#{puppet_module}/#{file}")
+          end
+        end
+        files_to_manage -= files_to_delete
+        if options[:noop]
+          Git.update_noop(puppet_module, options[:branch])
+        else
+          Git.update(puppet_module, files_to_manage, options[:message], options[:branch])
+        end
+      end
+    elsif options[:command] == 'hook'
+      Hook.hook(args[1], options)
+    end
+  end
+
+end

data/modulesync.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = 'modulesync'
+  spec.version       = '0.0.1'
+  spec.authors       = ['Colleen Murphy']
+  spec.email         = ['colleen@puppetlabs.com']
+  spec.summary       = %q{Puppet Module Synchronizer}
+  spec.description   = %q{Utility to synchronize common files across puppet modules in Github.}
+  spec.homepage      = "http://github.com/puppetlabs/modulesync"
+  spec.license       = "Apache 2"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+
+  spec.add_runtime_dependency 'git', '~>1.2'
+end

metadata

@@ -0,0 +1,88 @@
+--- !ruby/object:Gem::Specification
+name: modulesync
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Colleen Murphy
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-09-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: git
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.2'
+description: Utility to synchronize common files across puppet modules in Github.
+email:
+- colleen@puppetlabs.com
+executables:
+- msync
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- bin/msync
+- lib/modulesync.rb
+- lib/modulesync/cli.rb
+- lib/modulesync/constants.rb
+- lib/modulesync/git.rb
+- lib/modulesync/hook.rb
+- lib/modulesync/renderer.rb
+- lib/modulesync/util.rb
+- modulesync.gemspec
+homepage: http://github.com/puppetlabs/modulesync
+licenses:
+- Apache 2
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Puppet Module Synchronizer
+test_files: []
+has_rdoc: