puppet-armature 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 20b1e802ab0134205a8fac195ba528a3664fd16f
-  data.tar.gz: c1a48279c939c21b4ae482ced27e44243c69c4fc
+  metadata.gz: 39ee81e7821523113eb5b9868a0dca398ef3a6b0
+  data.tar.gz: 31da92d00fb15e345c8fdb28fce24e6e07fb1ec0
 SHA512:
-  metadata.gz: ee52d4da30d62132d98b0e0246c4ce6aeb5c99126e7ce9538fe7de28e72c7926f6d33edc377d403bb048d9b2e917ad1f4685b61703b32bf2d2435f85f019b1dc
-  data.tar.gz: 0da4f27f3df184cc48c1ae5d8383d7a62a874573a15fca64a3aff0cb54323144ef65da983a7c10c85d81c3977ce7e307abf1116ea5353fd8b7b32c28ce371a19
+  metadata.gz: a2c928a9daa31005a7d80b757080c6f41cd7b3785703c05bb61b10a51df848675b7907da9b63381d15a48b745ce6dfe3de1c19d50cbb7dbea2f83dfc69a8f141
+  data.tar.gz: d174e15de2dd26fbe228fed75383ffc6377afd166e79388a690e689f6d8f27329ebc4603e40bdd65e111cb91b94a2e295ce9f7b7fecf14d1f707bba29b87bba9

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    puppet-armature (0.2.2)
+    puppet-armature (0.4.0)
       gli (= 2.14.0)
       logging (~> 2)
 
@@ -9,19 +9,20 @@ GEM
   remote: https://rubygems.org/
   specs:
     gli (2.14.0)
-    little-plugger (1.1.3)
-    logging (2.0.0)
+    little-plugger (1.1.4)
+    logging (2.2.2)
       little-plugger (~> 1.1)
       multi_json (~> 1.10)
     minitest (5.10.1)
-    multi_json (1.12.1)
+    multi_json (1.13.1)
 
 PLATFORMS
   ruby
+  x86_64-darwin-16
 
 DEPENDENCIES
   minitest (~> 5.9)
   puppet-armature!
 
 BUNDLED WITH
-   1.14.6
+   1.17.1

data/TODO.md

@@ -1,13 +1,5 @@
 # Future development
 
-### Update module branches when updating an environment
-
-Rather than waiting for a periodic `update-branches` job to run, update module
-branches when their environment is deployed.
-
-This will require some caching so that work is not duplicated when updating all
-environments.
-
 ### Webhook endpoints
 
 I'm unsure if I want this. Webhooks are definitely useful, but I can get that

data/bin/armature

@@ -20,6 +20,9 @@ switch [:v,:verbose]
 desc 'Show DEBUG level output'
 switch [:d,:debug]
 
+desc 'Add timestamp for each line in log'
+switch [:"log-time"]
+
 if Process.uid == 0
   environments_default = '/etc/puppetlabs/code/environments'
   cache_default = '/srv/armature-cache'
@@ -47,12 +50,12 @@ command "deploy-ref" do |c|
     # This should fail as early as possible (e.g. if the path isn't correct)
     environments = Armature::Environments.new(@environments_path, @cache)
 
-    repo = @cache.get_repo(arguments.shift)
+    repo = Armature::Repo::Git.from_url(@cache, arguments.shift)
     ref = arguments.shift
     name = arguments.shift
 
     begin
-      environments.checkout_ref(repo, ref, name)
+      environments.check_out_ref(repo, ref, name)
     rescue
       @logger.error("Error deploying ref '#{ref}' as '#{name}'")
       raise
@@ -68,11 +71,14 @@ arg 'BRANCH', :multiple=>true
 command "deploy-branches" do |c|
   c.desc "Delete environments that aren't being deployed"
   c.switch [:d,"delete-other"]
+  c.desc "Fix environment names instead of skipping them"
+  c.switch [:x,"fix-environment-names"]
   c.action do |global_options, options, arguments|
     # This should fail as early as possible (e.g. if the path isn't correct)
     environments = Armature::Environments.new(@environments_path, @cache)
+    environments.fix_environment_names = options["fix-environment-names"]
 
-    repo = @cache.get_repo(arguments.shift)
+    repo = Armature::Repo::Git.from_url(@cache, arguments.shift)
     all_branches = repo.get_branches()
     branches = Set.new()
 
@@ -96,7 +102,7 @@ command "deploy-branches" do |c|
 
     branches.each do |branch|
       begin
-        environments.checkout_ref(repo, branch)
+        environments.check_out_ref(repo, branch)
       rescue => e
         @logger.error("Error deploying branch '#{branch}' (skipping): #{e}")
       end
@@ -111,11 +117,20 @@ end
 
 desc 'Update all branches in the cache'
 command :update do |c|
-  c.action { @cache.update_branches() }
+  c.action { @cache.update_mutable_refs() }
 end
 
 pre do |global_options, command, options, arguments|
-  Logging.logger.root.add_appenders Logging.appenders.stdout
+  appender = Logging.appenders.stdout
+
+  if global_options[:"log-time"]
+    appender.layout = Logging.layouts.pattern.new(
+      :pattern =>  "%d %-5l [%c] %m\n",
+      :date_pattern =>  "%Y-%m-%d %H:%M:%S",
+    )
+  end
+
+  Logging.logger.root.add_appenders(appender)
 
   Logging.logger.root.level = :info if global_options[:verbose]
   Logging.logger.root.level = :debug if global_options[:debug]

data/lib/armature.rb

@@ -1,10 +1,18 @@
+module Armature
+end
+
+require "armature/errors.rb"
+
 require "armature/cache.rb"
 require "armature/environments.rb"
-require "armature/gitrepo.rb"
 require "armature/puppetfile.rb"
+require "armature/ref/base.rb"
+require "armature/ref/identity.rb"
+require "armature/ref/immutable.rb"
+require "armature/ref/mutable.rb"
+require "armature/repo.rb"
+require "armature/repo/git.rb"
+require "armature/repo/forge.rb"
 require "armature/run.rb"
 require "armature/util.rb"
 require "armature/version.rb"
-
-module Armature
-end

data/lib/armature/cache.rb

@@ -19,73 +19,48 @@ module Armature
       end
     end
 
-    def flush_memory!
-      @logger.debug("Flushing in-memory caches for all repos")
-      @repos.each do |name, repo|
-        repo.freshen!
+    # Get the path to a repo on disk.
+    #
+    # If the repo doesn't exist locally, then it creates the path, locks it,
+    # and runs the passed block to create the repo.
+    def open_repo(type, url)
+      repo_id = _fs_repo_id(type, url)
+      repo_path = "#{@path}/repo/#{repo_id}"
+
+      if Dir.exist? repo_path
+        return repo_path
       end
-    end
 
-    # Get GitRepo object for a local clone of a remote repo at a URL
-    #
-    # This will clone the repo if it doesn't already exist.
-    def get_repo(url)
-      safe_url = fs_sanitize(url)
-
-      repo_path = "#{@path}/repo/#{safe_url}"
-      if ! Dir.exist? repo_path
-        @logger.info("Cloning '#{url}' for the first time")
-        Armature::Util::lock repo_path, File::LOCK_EX, "clone" do
-          if Dir.exist? repo_path
-            @logger.info("Another process cloned '#{url}' while we were blocked")
-          else
-            # Mirror copies *all* refs, not just branches. Ignore output.
-            Armature::Run.command(
-              "git", "clone", "--quiet", "--mirror", url, repo_path)
-            @logger.debug("Done cloning '#{url}'")
-          end
+      @logger.debug("Creating repo for #{type} '#{url}'")
+      Armature::Util::lock(repo_path, File::LOCK_EX, "create") do
+        if Dir.exist? repo_path
+          @logger.debug("Another process created repo for #{type} '#{url}' while we were waiting")
+          return repo_path
         end
-      end
 
-      get_repo_by_name(safe_url)
-    end
+        temp_path = new_temp_path()
+        Dir.mkdir(temp_path)
 
-    # Get a GitRepo object for an existing local repo by its santized URL
-    def get_repo_by_name(safe_url)
-      @repos[safe_url] ||= GitRepo.new("#{@path}/repo/#{safe_url}", safe_url)
-    end
+        yield temp_path
 
-    # Check out a ref from a repo and return the path
-    def checkout(repo, ref, refresh=false, options={})
-      if refresh
-        # Don't check the cache; refresh it from source.
-        repo.freshen()
-        ref = repo.canonical_ref(ref)
-      else
-        # This will raise a RefError if the ref doesn't exist
-        begin
-          ref = repo.canonical_ref(ref)
-        rescue RefError
-          repo.freshen()
-          ref = repo.canonical_ref(ref)
-        end
+        File.rename(temp_path, repo_path)
       end
 
-      type = repo.ref_type(ref)
-      safe_ref = fs_sanitize(ref)
-      repo_path = "#{@path}/ref/#{type}/#{repo.name}"
-      ref_path = "#{repo_path}/#{safe_ref}"
+      return repo_path
+    end
 
-      if ! refresh
-        # Check cache first
-        if Dir.exist? ref_path
-          return ref_path
-        end
-      end
+    def open_ref(ref)
+      safe_ref = fs_sanitize(ref.canonical_name)
+      repo_path = "#{@path}/ref/#{ref.type}/#{fs_repo_id(ref.repo)}"
+      ref_path = "#{repo_path}/#{safe_ref}"
 
       FileUtils.mkdir_p(repo_path)
 
-      identity_path = checkout_identity(repo, repo.ref_identity(ref))
+      @logger.debug("Checking for #{ref} in #{ref.repo}")
+      identity_path = open_identity(ref.repo, ref.identity) do |object_path|
+        yield object_path
+      end
+
       if identity_path != ref_path
         atomic_symlink(identity_path, ref_path)
       end
@@ -93,6 +68,33 @@ module Armature
       ref_path
     end
 
+    def register_repo(repo)
+      @repos[fs_repo_id(repo)] = repo
+    end
+
+    # Takes a string like "git:https://github.com/a/b.git" and returns Repo::Git
+    def repo_klass(repo_id)
+      type = repo_id.split(":", 2).first
+      Repo.const_get(type.capitalize())
+    end
+
+    # Get a repo object for an existing local repo by its santized URL
+    def repo_by_id(repo_id)
+      @repos[repo_id] ||= repo_klass(repo_id).from_path(self, "#{@path}/repo/#{repo_id}")
+    end
+
+    def get_repo(type, url)
+      @repos[_fs_repo_id(type, url)]
+    end
+
+    # This is used in the testing code
+    def flush_memory!
+      @logger.debug("Flushing in-memory caches for all repos")
+      @repos.each do |name, repo|
+        repo.flush_memory!
+      end
+    end
+
     # Creates a symlink atomically
     #
     # That is, if a symlink or file exists at new_path, then this will replace
@@ -102,8 +104,6 @@ module Armature
     def atomic_symlink(target_path, new_path)
       new_path.chomp!("/")
 
-      @logger.debug("#{new_path} -> #{target_path}")
-
       begin
         if File.readlink(new_path) == target_path
           return
@@ -111,6 +111,8 @@ module Armature
       rescue Errno::ENOENT
       end
 
+      @logger.debug("Updating symlink #{new_path} -> #{target_path}")
+
       temp_path = new_temp_path()
       File.symlink(target_path, temp_path)
       File.rename(temp_path, new_path)
@@ -119,15 +121,15 @@ module Armature
       raise
     end
 
-    def update_branches()
+    def update_mutable_refs()
       Dir.glob("#{@path}/ref/mutable/*/*") do |path|
-        ref = fs_unsanitize(File.basename(path))
-        repo = get_repo_by_name(File.basename(File.dirname(path)))
-        @logger.info("Updating #{ref} ref from #{repo.url}")
+        repo = repo_by_id(File.basename(File.dirname(path)))
+        ref = repo.mutable_fs_ref(fs_unsanitize(File.basename(path)))
+        @logger.info("Updating #{ref} ref from #{repo}")
 
         begin
-          checkout(repo, ref, :refresh=>true)
-        rescue RefError
+          ref.check_out()
+        rescue Armature::RefError
           # The ref no longer exists, so we can't update it. Leave the old
           # checkout in place for safety; garbage collection will remove it if
           # it's no longer used.
@@ -167,6 +169,18 @@ module Armature
       @lock_file = nil
     end
 
+    # Open a temp directory, chdir into it, yield, then delete the directory
+    def open_temp
+      path = new_temp_path()
+
+      Dir.mkdir(path)
+      Dir.chdir(path) do
+        yield
+      end
+
+      FileUtils.remove_entry(path)
+    end
+
   private
 
     def new_temp_path
@@ -183,20 +197,29 @@ module Armature
       "#{@path}/object/#{@process_prefix}.#{@sequence}#{name}"
     end
 
-    def fs_sanitize(ref)
+    def fs_repo_id(repo)
+      _fs_repo_id(repo.type, repo.url)
+    end
+
+    def _fs_repo_id(type, url)
+      fs_sanitize("#{type}:#{url}")
+    end
+
+    ### FIXME write tests for this
+    def fs_sanitize(str)
       # Escape | and replace / with |. Also escape a leading .
-      ref.sub(/\A\./, "\\.").gsub(/[\\|]/, '\\\0').gsub(/\//, '|')
+      str.sub(/\A\./, "\\.").gsub(/[\\|]/, '\\\0').gsub(%r{/}, '|')
     end
 
-    def fs_unsanitize(name)
-      name.gsub(/\|/, '/').gsub(/\\([\\|])/, '\0').sub(/^\\\./, '.')
+    ### FIXME write tests for this
+    def fs_unsanitize(str)
+      str.gsub(/\\([\\|])/, '\0').gsub(/\|/, '/').sub(/^\A\\\./, '.')
     end
 
-    # Assumes identity exists. Use checkout() if it might not.
-    def checkout_identity(repo, identity)
+    def open_identity(repo, identity)
       safe_identity = fs_sanitize(identity)
 
-      repo_path = "#{@path}/identity/#{repo.name}"
+      repo_path = "#{@path}/ref/identity/#{fs_repo_id(repo)}"
       identity_path = "#{repo_path}/#{safe_identity}"
       if Dir.exist? identity_path
         return identity_path
@@ -204,7 +227,7 @@ module Armature
 
       FileUtils.mkdir_p(repo_path)
 
-      Armature::Util::lock identity_path, File::LOCK_EX, "checkout" do
+      Armature::Util::lock(identity_path, File::LOCK_EX, "check out") do
         # Another process may have created the object before we got the lock
         if Dir.exist? identity_path
           return identity_path
@@ -214,11 +237,13 @@ module Armature
         FileUtils.mkdir_p object_path
 
         @logger.debug(
-          "Checking out '#{identity}' from '#{repo.url}' into '#{object_path}'")
-        repo.git "reset", "--hard", identity, :work_dir=>object_path
+          "Checking out '#{identity}' from '#{repo}' into '#{object_path}'")
+        yield object_path
         atomic_symlink(object_path, identity_path)
       end
 
+      @logger.debug("Finished checking out \"#{identity}\" from \"#{repo}\"")
+
       identity_path
     end
 

data/lib/armature/environments.rb

@@ -1,11 +1,37 @@
 module Armature
   class Environments
+    class InvalidNameError < Armature::Error
+    end
+
+    # The documentation claims that uppercase letters are invalid, but in
+    # practice they seem to be fine.
+    #
+    # https://docs.puppet.com/puppet/latest/reference/lang_reserved.html#environments
+    def self.valid_environment_name?(name)
+      name =~ /\A[A-Za-z0-9_]+\Z/
+    end
+
+    def self.assert_valid_environment_name(name)
+      if ! valid_environment_name?(name)
+        raise InvalidNameError, "Invalid environment name '#{name}'"
+      end
+    end
+
+    # Same rules as for a class
+    def self.assert_valid_module_name(name)
+      if name !~ /\A[a-z][a-z0-9_]*\Z/
+        raise InvalidNameError, "Invalid module name '#{name}'"
+      end
+    end
+
     attr_reader :path
+    attr_accessor :fix_environment_names
 
     # path is the path to the directory containing all the environments
     def initialize(path, cache)
       @cache = cache
       @logger = Logging.logger[self]
+      @fix_environment_names = false
 
       if not File.directory? path
         abort "Puppet environments path does not exist: '#{path}'"
@@ -25,23 +51,32 @@ module Armature
       @logger.debug "Environment '#{name}' does not exist"
     end
 
-    def checkout_ref(repo, ref, name=ref)
-      # This will add and update a modules dir in any repo, even if the repo is
-      # used in a Puppetfile. (Perhaps the cache is used for multiple repos?)
-
-      # https://docs.puppet.com/puppet/latest/reference/lang_reserved.html#environments
-      if name !~ /\A[a-z0-9_]+\Z/
-        raise "Invalid environment name '#{name}'"
+    def normalize_environment_name(name)
+      if @fix_environment_names && ! self.class.valid_environment_name?(name)
+        old = name
+        name = old.gsub(/[^A-Za-z0-9_]/, "_")
+        @logger.info("Changing invalid environment name \"#{old}\" to \"#{name}\"")
       end
 
+      self.class.assert_valid_environment_name(name)
+      name
+    end
+
+    # Create an environment from a ref
+    #
+    # This will add and update a modules dir in any repo, even if the repo is
+    # used in a Puppetfile. (Perhaps the cache is used for multiple repos?)
+    def check_out_ref(repo, ref_str, name=ref_str)
+      name = normalize_environment_name(name)
+
       @cache.lock File::LOCK_SH do
-        @logger.info "Deploying ref '#{ref}' from '#{repo.url}' as" \
+        @logger.info "Deploying ref '#{ref_str}' from '#{repo}' as" \
           " environment '#{name}'"
 
         begin
-          ref_path = @cache.checkout(repo, ref, true)
-        rescue RefError
-          @logger.info "Ref '#{ref}' does not exist; ensuring environment" \
+          ref_path = repo.general_ref(ref_str).check_out()
+        rescue Armature::RefError
+          @logger.info "Ref '#{ref_str}' does not exist; ensuring environment" \
             " '#{name}' is gone"
           remove(name)
           return
@@ -50,7 +85,7 @@ module Armature
         puppetfile_path = "#{ref_path}/Puppetfile"
         if File.exist?(puppetfile_path)
           @logger.debug "Found Puppetfile in environment '#{name}'"
-          module_refs = Armature::Puppetfile.new().include(puppetfile_path)
+          module_refs = Armature::Puppetfile.new(@cache).include(puppetfile_path)
           @logger.debug "Loaded Puppetfile in environment '#{name}' with" \
             " #{module_refs.length} modules"
         else
@@ -60,8 +95,9 @@ module Armature
 
         update_modules(ref_path, module_refs)
 
+        # Make the change live
         @cache.atomic_symlink(ref_path, "#{@path}/#{name}")
-        @logger.debug "Done deploying ref '#{ref}' from '#{repo.url}' as" \
+        @logger.debug "Done deploying ref '#{ref_str}' from '#{repo}' as" \
           " environment '#{name}'"
       end
     end
@@ -69,6 +105,8 @@ module Armature
   private
 
     # Apply the results of the Puppetfile to a ref (e.g. an environment)
+    #
+    ### FIXME This could update modules in an existing check out.
     def update_modules(target_path, module_refs)
       modules_path = "#{target_path}/modules"
       if ! Dir.exist? modules_path
@@ -76,14 +114,9 @@ module Armature
       end
 
       module_refs.each do |name, info|
-        if name =~ /\A\./
-          raise "Module name may not start with period: '#{name}'"
-        elsif name =~ /\//
-          raise "Module name may not contain /: '#{name}'"
-        end
-
-        ref_path = @cache.checkout(@cache.get_repo(info[:git]), info[:ref])
+        self.class.assert_valid_module_name(name)
 
+        ref_path = info[:ref].check_out()
         @cache.atomic_symlink(ref_path, "#{modules_path}/#{name}")
       end