puppet-armature 0.3.0 → 0.4.0

data/lib/armature/errors.rb

@@ -0,0 +1,7 @@
+module Armature
+  class Error < StandardError
+  end
+
+  class RefError < Armature::Error
+  end
+end

data/lib/armature/puppetfile.rb

@@ -2,26 +2,52 @@ module Armature
   class Puppetfile
     attr_reader :results
 
-    def initialize()
+    def initialize(cache)
+      @cache = cache
       @results = {}
+      @logger = Logging.logger[self]
+      @forge_url = "https://forge.puppet.com"
     end
 
+    ### FIXME this will have access to @cache and @results
     def include(path)
       instance_eval(IO.read(path), path)
       @results
     end
 
-    def mod(name, options={})
-      if name =~ /\A\./
-        raise "Module name may not start with period: '#{name}'"
-      elsif name =~ /\//
-        raise "Module name may not contain /: '#{name}'"
-      end
+    def forge(url)
+      @forge_url = url.chomp("/")
+    end
+
+    def mod(full_name, options={})
+      name = full_name.split("-", 2).last()
+      Armature::Environments.assert_valid_module_name(name)
 
       if @results[name]
         raise "Module #{name} declared twice"
       end
 
+      if options.is_a?(String) || options == {} || options == nil
+        _mod_forge(full_name, name, options)
+      elsif options[:git]
+        _mod_git(name, options)
+      else
+        raise "Invalid mod call: #{full_name} #{options}"
+      end
+    end
+
+    def _mod_forge(full_name, name, version="latest")
+      if version == nil || version == {}
+        version = "latest"
+      end
+
+      repo = Repo::Forge.from_url(@cache, @forge_url, full_name)
+      ref = repo.general_ref(version)
+      @logger.debug("mod #{name}: #{ref}")
+      @results[name] = { :name => name, :ref => ref }
+    end
+
+    def _mod_git(name, options={})
       options = Armature::Util.process_options(options, {
         :commit => nil,
         :tag => nil,
@@ -31,45 +57,43 @@ module Armature
         :git => nil,
       })
 
+      repo = Repo::Git.from_url(@cache, options[:git])
       ref = nil
 
       if options[:commit]
         if ref
           raise "Module #{name} has more than one of :commit, :tag, :branch, or :ref"
         end
-        ref = options[:commit]
+        ref = repo.identity_ref(options[:commit])
       end
 
       if options[:tag]
         if ref
           raise "Module #{name} has more than one of :commit, :tag, :branch, or :ref"
         end
-        ref = "refs/tags/#{options[:tag]}"
+        ref = repo.tag_ref(options[:tag])
       end
 
       if options[:branch]
         if ref
           raise "Module #{name} has more than one of :commit, :tag, :branch, or :ref"
         end
-        ref = "refs/heads/#{options[:branch]}"
+        ref = repo.branch_ref(options[:branch])
       end
 
       if options[:ref]
         if ref
           raise "Module #{name} has more than one of :commit, :tag, :branch, or :ref"
         end
-        ref = options[:ref]
+        ref = repo.general_ref(options[:ref])
       end
 
       if ! ref
-        ref = "refs/heads/master"
+        ref = repo.branch_ref("master")
       end
 
-      @results[name] = { :name => name, :ref => ref, :git => options[:git] }
-    end
-
-    def forge(*arguments)
-      ### error?
+      @logger.debug("mod #{name}: #{ref}")
+      @results[name] = { :name => name, :ref => ref }
     end
   end
 end

data/lib/armature/ref/base.rb

@@ -0,0 +1,21 @@
+module Armature::Ref
+  class Base
+    attr_reader :repo, :canonical_name, :identity, :human_type, :human_name
+
+    def initialize(repo, canonical_name, identity, human_type, human_name)
+      @repo = repo
+      @canonical_name = canonical_name
+      @identity = identity
+      @human_type = human_type
+      @human_name = human_name
+    end
+
+    def check_out
+      @repo.check_out(self)
+    end
+
+    def to_s
+      "#{@human_type} \"#{@human_name}\""
+    end
+  end
+end

data/lib/armature/ref/identity.rb

@@ -0,0 +1,5 @@
+class Armature::Ref::Identity < Armature::Ref::Base
+  def type
+    :identity
+  end
+end

data/lib/armature/ref/immutable.rb

@@ -0,0 +1,5 @@
+class Armature::Ref::Immutable < Armature::Ref::Base
+  def type
+    :immutable
+  end
+end

data/lib/armature/ref/mutable.rb

@@ -0,0 +1,5 @@
+class Armature::Ref::Mutable < Armature::Ref::Base
+  def type
+    :mutable
+  end
+end

data/lib/armature/repo.rb

@@ -0,0 +1,56 @@
+class Armature::Repo
+  def self.from_path(cache, path)
+    # Called from the cache; don't check for an existing repo
+    self.new(cache, path)
+  end
+
+  def initialize(cache, repo_dir)
+    @cache = cache
+    @repo_dir = repo_dir
+    @logger = Logging.logger[self]
+    @url = nil
+    flush_memory!
+
+    @cache.register_repo(self)
+  end
+
+  # You may wish to override these methods
+
+  def url
+    @url
+  end
+
+  def check_out(ref)
+  end
+
+  def freshen!
+    flush_memory!
+  end
+
+  def flush_memory!
+    @fresh = false
+  end
+
+  # Generally these don't need to be overridden
+
+  def self.type
+    self.name.split('::').last().downcase()
+  end
+
+  def type
+    self.class.type()
+  end
+
+  def to_s
+    url()
+  end
+
+  def freshen
+    if ! @fresh
+      freshen!
+      true
+    else
+      false
+    end
+  end
+end

data/lib/armature/repo/forge.rb

@@ -0,0 +1,154 @@
+# Get a module from the Forge
+class Armature::Repo::Forge < Armature::Repo
+  CANONICAL_FORGE_URL = "https://forge.puppet.com"
+  FORGE_URLS = [
+    "https://forge.puppetlabs.com",
+    "https://forgeapi.puppetlabs.com",
+    "https://forgeapi.puppet.com",
+    "http://forge.puppetlabs.com",
+    "http://forgeapi.puppetlabs.com",
+    "http://forge.puppet.com",
+    "http://forgeapi.puppet.com",
+  ]
+
+  def self.normalize_forge_url(url)
+    url.chomp!("/")
+    if FORGE_URLS.include? url
+      CANONICAL_FORGE_URL
+    else
+      url
+    end
+  end
+
+  # Gets a repo object for a given Forge module. This just contains metadata.
+  def self.from_url(cache, forge_url, full_name)
+    forge_url = self.normalize_forge_url(forge_url)
+
+    url = "#{forge_url}/#{full_name}"
+    repo = cache.get_repo("forge", url)
+    if repo
+      return repo
+    end
+
+    repo_dir = cache.open_repo("forge", url) do |temp_path|
+      File.write("#{temp_path}/url", "#{url}\n")
+      File.write("#{temp_path}/forge_url", "#{forge_url}\n")
+      File.write("#{temp_path}/full_name", "#{full_name}\n")
+
+      Logging.logger[self].debug("Created stub repo for '#{url}'")
+    end
+
+    return self.new(cache, repo_dir)
+  end
+
+  def url
+    @url || File.read("#{@repo_dir}/url").chomp()
+  end
+
+  def forge_url
+    @forge_url || File.read("#{@repo_dir}/forge_url").chomp()
+  end
+
+  def full_name
+    @full_name || File.read("#{@repo_dir}/full_name").chomp()
+  end
+
+  # Ensure we've got the correct version and return its path
+  def check_out(ref)
+    @cache.open_ref(ref) do |object_path|
+      @cache.open_temp() do
+        @logger.debug("Downloading #{ref} from #{url}")
+        url = forge_url() + release_metadata(ref.identity, "file_uri")
+
+        tarball = Armature::Util::http_get(url)
+
+        # This extracts into a temp directory
+        Armature::Run::pipe_command(tarball, "tar", "xzf" , "-")
+
+        # Move everything into the object directory
+        extract_dir = Dir["*"].first()
+        Dir.entries(extract_dir).each do |child|
+          if ! [".", ".."].include?(child)
+            File.rename("#{extract_dir}/#{child}", "#{object_path}/#{child}")
+          end
+        end
+      end
+    end
+  end
+
+  def freshen!
+    flush_memory!
+    download_metadata()
+  end
+
+  def flush_memory!
+    @fresh = false
+    @metadata = nil
+  end
+
+  # Get ref object
+  def general_ref(version)
+    if version.nil? || version == "latest"
+      latest_ref()
+    else
+      version_ref(version)
+    end
+  end
+
+  def version_ref(version)
+    Armature::Ref::Immutable.new(self, version, version, "version", version)
+  end
+
+  def latest_ref
+    freshen()
+    version = metadata("current_release.version")
+
+    Armature::Ref::Mutable.new(self, "latest", version, "version", "latest")
+  end
+
+private
+
+  def metadata(path=nil)
+    ### FIXME cache metadata in repo
+    @metadata ||= download_metadata()
+    return @metadata if path == nil
+
+    begin
+      dig(@metadata, path.split("."))
+    rescue => e
+      raise "Got invalid metadata from #{metadata_url()}: #{e}"
+    end
+  end
+
+  def release_metadata(version, path=nil)
+    metadata("releases").each do |release|
+      begin
+        if release["version"] == version
+          return dig(release, path.split("."))
+        end
+      rescue => e
+        raise "Got invalid metadata from #{metadata_url()}: #{e}"
+      end
+    end
+
+    raise "Release #{version} not found"
+  end
+
+  def metadata_url
+    [forge_url(), "v3", "modules", full_name()].join("/")
+  end
+
+  def download_metadata
+    @fresh = true
+    @metadata = Armature::Util::http_get_json(metadata_url())
+  end
+
+  def dig(parent, path)
+    key = path.shift()
+    if path.length() == 0
+      parent[key]
+    else
+      dig(parent[key], path)
+    end
+  end
+end

data/lib/armature/repo/git.rb

@@ -0,0 +1,228 @@
+class Armature::Repo::Git < Armature::Repo
+  # Gets a repo object for a given URL. Mirrors the repo in the local cache
+  # if it doesn't already exist.
+  def self.from_url(cache, url)
+    repo = cache.get_repo("git", url)
+    if repo
+      return repo
+    end
+
+    fresh = false
+    repo_dir = cache.open_repo("git", url) do |temp_path|
+      Logging.logger[self].debug("Cloning '#{url}' for the first time")
+
+      # Mirror copies *all* refs, not just branches. Ignore output.
+      Armature::Run.clean_git("clone", "--quiet", "--mirror", url, temp_path)
+      fresh = true
+
+      Logging.logger[self].debug("Done cloning '#{url}'")
+    end
+
+    return self.new(cache, repo_dir, fresh)
+  end
+
+  def initialize(cache, repo_dir, is_fresh=false)
+    super(cache, repo_dir)
+    @fresh = is_fresh
+  end
+
+  def url
+    @url ||= git("config", "--get", "remote.origin.url").chomp()
+  end
+
+  # Check out a ref from a repo and return the path
+  def check_out(ref)
+    @cache.open_ref(ref) do |object_path|
+      git "reset", "--hard", ref.identity, :work_dir=>object_path
+    end
+  end
+
+  def freshen!
+    flush_memory!
+
+    @logger.info("Fetching from #{url}")
+    Armature::Util::lock(@repo_dir, File::LOCK_EX, "fetch") do
+      ### FIXME Only flush memory if this makes changes
+      git "remote", "update", "--prune"
+    end
+    @fresh = true
+  end
+
+  def flush_memory!
+    @fresh = false
+    @ref_cache = {}
+    @rev_cache = {}
+  end
+
+  # Get ref object
+  def general_ref(ref_str)
+    return @ref_cache[ref_str] if @ref_cache[ref_str]
+
+    if ref_str.start_with? "refs/heads/"
+      return branch_ref(ref_str.sub("refs/heads/", ""))
+    end
+
+    if ref_str.start_with? "refs/tags/"
+      return tag_ref(ref_str.sub("refs/tags/", ""))
+    end
+
+    retry_fresh do
+      if ref_str.start_with? "refs/"
+        freshen()
+        return make_ref(Armature::Ref::Mutable, ref_str, rev_parse!(ref_str),
+          "ref", ref_str)
+      end
+
+      if sha = rev_parse("refs/heads/#{ref_str}")
+        return branch_ref(ref_str)
+      end
+
+      if sha = rev_parse("refs/tags/#{ref_str}")
+        return tag_ref(ref_str)
+      end
+
+      # This could trigger a retry
+      sha = rev_parse!(ref_str)
+      if sha_match? sha, ref_str
+        return identity_ref(sha)
+      end
+
+      # It exists, but it's outside of refs/. Treat it as mutable.
+      make_ref(Armature::Ref::Mutable, ref_str, sha, "ref", ref_str)
+    end
+  end
+
+  # Get ref object for some sort of mutable ref we found in the FS cache
+  def mutable_fs_ref(ref_str)
+    freshen()
+
+    return @ref_cache[ref_str] if @ref_cache[ref_str]
+
+    if ref_str.start_with? "refs/heads/"
+      return branch_ref(ref_str.sub("refs/heads/", ""))
+    end
+
+    if ref_str.start_with? "refs/tags/"
+      return tag_ref(ref_str.sub("refs/tags/", ""))
+    end
+
+    make_ref(Armature::Ref::Mutable, ref_str, rev_parse!(ref_str), "ref", ref_str)
+  end
+
+  # The identity of an object itself, i.e. a SHA
+  def identity_ref(sha)
+    return @ref_cache[sha] if @ref_cache[sha]
+
+    real_sha = nil # needs to be bound to this scope
+    retry_fresh do
+      # real_sha will always be a full SHA, but sha might be partial
+      real_sha = rev_parse!(sha)
+      if ! sha_match? real_sha, sha
+        raise Armature::RefError, "'#{sha}' is not a Git SHA"
+      end
+    end
+
+    make_ref(Armature::Ref::Identity, real_sha, real_sha, "revision", real_sha)
+
+    # If sha is partial, register it in the cache too.
+    @ref_cache[sha] = @ref_cache[real_sha]
+  end
+
+  def branch_ref(name)
+    ref_str = "refs/heads/#{name}"
+    freshen()
+
+    return @ref_cache[ref_str] if @ref_cache[ref_str]
+
+    sha = rev_parse!(ref_str)
+    _branch_ref(ref_str, name, sha)
+  end
+
+  def _branch_ref(ref_str, name, sha)
+    make_ref(Armature::Ref::Mutable, ref_str, sha, "branch", name)
+    @ref_cache[name] = @ref_cache[ref_str]
+  end
+
+  def tag_ref(name)
+    ref_str = "refs/tags/#{name}"
+    return @ref_cache[ref_str] if @ref_cache[ref_str]
+
+    sha = retry_fresh { rev_parse!(ref_str) }
+    make_ref(Armature::Ref::Immutable, ref_str, sha, "tag", name)
+    @ref_cache[name] = @ref_cache[ref_str]
+  end
+
+  def git(*arguments)
+    # This accepts a hash of options as the last argument
+    options = if arguments.last.is_a? Hash then arguments.pop else {} end
+    options = Armature::Util.process_options(options, { :work_dir => nil }, {})
+
+    if options[:work_dir]
+      work_dir_arguments = [ "--work-tree=" + options[:work_dir] ]
+    else
+      work_dir_arguments = []
+    end
+
+    command = [ "--git-dir=#{@repo_dir}" ] \
+      + work_dir_arguments \
+      + arguments
+
+    Armature::Run.clean_git(*command)
+  end
+
+  def get_branches()
+    freshen()
+    data = git("for-each-ref",
+      "--format", "%(objectname) %(refname)",
+      "refs/heads")
+    lines = data.split(/[\r\n]/).reject { |line| line == "" }
+
+    lines.map do |line|
+      sha, ref_str = line.split(' ', 2)
+      name = ref_str.sub("refs/heads/", "")
+      _branch_ref(ref_str, name, sha)
+      name
+    end
+  end
+
+private
+
+  def retry_fresh
+    yield
+  rescue Armature::RefError
+    if @fresh
+      raise
+    end
+
+    @logger.debug("Got ref error; fetching to see if that helps.")
+    freshen()
+    yield
+  end
+
+  def make_ref(klass, ref_str, sha, type, name)
+    @ref_cache[ref_str] = klass.new(self, ref_str, sha, type, name)
+  end
+
+  # Get the SHA for a ref, or nil if it doesn't exist
+  def rev_parse(ref_str)
+    rev_parse!(ref_str)
+  rescue Armature::RefError
+    nil
+  end
+
+  # Get the SHA for a ref, or raise if it doesn't exist
+  def rev_parse!(ref_str)
+    if @rev_cache[ref_str]
+      @rev_cache[ref_str]
+    end
+
+    @rev_cache[ref_str] = git("rev-parse", "--verify", "#{ref_str}^{commit}").chomp
+  rescue Armature::Run::CommandFailureError
+    raise Armature::RefError, "no such ref '#{ref_str}' in repo '#{self}'"
+  end
+
+  # Check if a real (full) SHA matches a (possibly partial) candidate SHA
+  def sha_match? real, candidate
+    real.start_with? candidate
+  end
+end