simp-metadata 0.4.3

data/lib/simp/media/type/control-repo.rb

@@ -0,0 +1,211 @@
+module Simp
+  module Media
+    module Type
+      class Control_repo < Simp::Media::Type::Base
+        attr_accessor :options
+        def initialize(options, engine)
+          super(options, engine)
+          @cleanup = []
+          if (options["output"] == nil)
+            raise "output must be specified for control-repo output"
+          end
+          @origtempdir = Dir.mktmpdir("cachedir")
+          @repopath = "#{@origtempdir}/control-repo"
+          FileUtils.mkdir_p(@repopath)
+          @cleanup << @origtempdir
+          exit_code = run("git clone #{options["output"]} #{@repopath}")
+          unless (exit_code.success?)
+            uri = URI(options["output"])
+            if (uri.scheme == "file")
+              FileUtils.mkdir_p(uri.path)
+              Dir.chdir(uri.path) do |path|
+                if (uri.path =~ /.*\.git$/)
+                  run("git init --bare")
+                else
+                  run("git init")
+                end
+
+                run("git clone #{options["output"]} #{@repopath}")
+              end
+            else
+              raise "output is not a valid control-repo"
+            end
+          end
+          Dir.chdir(@repopath) do |path|
+            exit_code = run("git checkout #{options["branch"]}")
+            unless exit_code.success?
+              exit_code = run("git checkout -b #{options["branch"]}")
+              unless (exit_code.success?)
+                raise "error, unable to checkout #{options["branch"]} in git repo #{uri}"
+              end
+            end
+            @branch = options["branch"]
+            if (options["destination_branch"] != nil)
+              @branch = options["destination_branch"]
+              exit_code = run("git checkout -b #{options["destination_branch"]}")
+              unless exit_code.success?
+                  raise "error, unable to create branch #{options["destination_branch"]} in git repo #{uri}"
+              end
+            end
+            run("rm -rf SIMP/modules")
+            run("rm -rf SIMP/assets")
+          end
+        end
+
+        def add_component(component, fetch_return_value)
+          if (options["embed"] == true)
+            # XXX ToDo: Copy components to control-repo if embed == true
+            case component.component_type
+              when "documentation"
+              when "simp-metadata"
+                subdirectory = "SIMP/metadata"
+                outputpath = "#{@repopath}/#{subdirectory}/#{component.name}"
+                FileUtils.mkdir_p(outputpath)
+                if (Dir.exists?("#{fetch_return_value['path']}/.git"))
+                  exit_code = run("cd #{fetch_return_value['path']} && git --work-tree=\"#{outputpath}\" checkout #{component.version} .")
+                else
+                  exit_code = run("cd #{fetch_return_value['path']} && tar -cf - . | tar -xvpf - -C \"#{outputpath}\"")
+                end
+
+                unless exit_code.success?
+                  error "unable to copy #{component.name} to #{outputpath}: error code #{exit_code.exitstatus}"
+                end
+              when "puppet-module"
+                subdirectory = "SIMP/modules"
+                outputpath = "#{@repopath}/#{subdirectory}/#{component.module_name}"
+                debug2("Copying #{component.module_name} to #{outputpath}")
+                FileUtils.mkdir_p(outputpath)
+                exit_code = run("cd #{fetch_return_value['path']} && git --work-tree=\"#{outputpath}\" checkout #{component.version} .")
+                unless exit_code.success?
+                  error "unable to copy #{component.module_name} to #{outputpath}: error code #{exit_code.exitstatus}"
+                end
+              else
+                subdirectory = "SIMP/assets/#{component.name}"
+                case component.output_type
+                  when :file
+                    FileUtils.mkdir_p("#{@repopath}/#{subdirectory}")
+                    FileUtils.cp(fetch_return_value["path"], "#{@repopath}/#{subdirectory}/#{component.output_filename}")
+                end
+            end
+          else
+            # XXX ToDo: Add necessary references to generate the puppetfile during finalize
+            raise "not yet implemented"
+          end
+
+        end
+
+        def finalize(manifest)
+          # XXX ToDo: Generate Puppetfile (if options["embed"] == false)
+          #     Otherwise copy to control-repo
+          #     XXX ToDo: Munge Puppetfile
+          environmentconf = "#{@repopath}/environment.conf"
+          hierayaml = "#{@repopath}/hiera.yaml"
+          # XXX ToDo: Munge hiera.yaml
+          self.munge_hierayaml(hierayaml)
+          if (options["embed"] == true)
+            self.munge_environmentconf(environmentconf)
+          end
+          run("cd #{@repopath} && git add -A")
+          run("cd #{@repopath} && git commit -m \"simp-install: upgrade to #{options["version"]}\"")
+          run("cd #{@repopath} && git push origin #{@branch}")
+        end
+
+        def munge_environmentconf(environmentconf)
+          # Munge environment.conf to add SIMP/modules to modulepath
+          if (File.exists?(environmentconf))
+            data = File.read(environmentconf).split("\n")
+            data.each_with_index do |line, fileline|
+              if (/^modulepath = (?<capture>.*)$/ =~ line)
+                paths = capture.split(":")
+                found = false
+                module_index = nil
+                paths.each_with_index do |path, index|
+                  if (path =~ /modules/)
+                    module_index = index
+                  end
+                  if (path =~ /simp\/modules/)
+                    found = true
+                  end
+                end
+                if (found == false)
+                  newarray = []
+                  paths.each do |path, index|
+                    newarray << path
+                    if (index == module_index)
+                      newarray << "SIMP/modules"
+                    end
+                  end
+                  data[fileline] = "modulepath = #{newarray.join(":")}"
+                  File.open(environmentconf, "w") { |f| f.write(data.join("\n")) }
+                end
+              end
+            end
+          else
+            File.open(environmentconf, "w") { |f| f.write("modulepath = modules:SIMP/modules:$basemodulepath\n") }
+          end
+        end
+
+        def munge_hierayaml(hierayaml)
+          data = {}
+          if (File.exists?(hierayaml))
+            data = YAML.load(File.read(hierayaml))
+            version = data["version"]
+            case version
+              when 4
+                # XXX ToDo: Add version 4 hiera.yaml support
+                raise "currently version 4 hiera.yaml's are not supported"
+              when 5
+                found = false
+                data["hierarchy"].each_with_index do |hash|
+                  if (hash["lookup_key"] == "compliance_markup::enforcement")
+                    found = true
+                  end
+                end
+                if (found == false)
+                  hash = {"name" => "SIMP Compliance Engine", "lookup_key" => "compliance_markup::enforcement"}
+                  data["hierarchy"] << hash
+                end
+              when nil
+                # XXX ToDo: Add version 3 hiera.yaml support
+                raise "currently version 3 hiera.yaml's are not supported"
+                return 0
+            end
+          else
+            raw_yaml = <<-EOF
+---
+version: 5
+defaults:
+  datadir: "data"
+  data_hash: "yaml_data"
+hierarchy:
+  - name: 'SIMP Defaults'
+    paths:
+      - 'hosts/%{trusted.certname}'
+      - 'domains/%{facts.domain}'
+      - '%{facts.os.family}'
+      - '%{facts.os.name}/%{facts.os.release.full}'
+      - '%{facts.os.name}/%{facts.os.release.major}'
+      - '%{facts.os.name}'
+      - 'hostgroups/%{::hostgroup}'
+      - 'hosttypes/%{::hosttype}'
+      - 'users'
+      - 'groups'
+      - 'type/%{::nodetype}/common.yaml'
+      - 'default'
+      - 'simp_config_settings'
+  - name: 'SIMP Compliance Engine'
+    lookup_key: 'compliance_markup::enforcement'
+            EOF
+            data = YAML.load(raw_yaml)
+          end
+          File.open(hierayaml, "w") { |f| f.write(data.to_yaml)}
+        end
+        def cleanup()
+          @cleanup.each do |path|
+            FileUtils.rmtree(path)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/simp/media/type/internet.rb

@@ -0,0 +1,38 @@
+
+require 'open3'
+
+module Simp
+  module Media
+    module Type
+      class Internet < Simp::Media::Type::Base
+        attr_accessor :options
+        def initialize(options, engine)
+          @cleanup = []
+          super(options, engine)
+        end
+
+        def input_directory=(directory)
+            @input_directory = directory
+        end
+        def input_directory
+          if (@input_directory == nil)
+            target = Dir.mktmpdir("cachedir")
+            @cleanup << target
+            @input_directory = target
+          else
+            @input_directory
+          end
+        end
+        def fetch_component(component, options)
+          Simp::Metadata.download_component(component, options.merge({"target" => self.input_directory}))
+        end
+
+        def cleanup()
+          @cleanup.each do |path|
+            FileUtils.rmtree(path)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/simp/media/type/iso.rb

@@ -0,0 +1,9 @@
+module Simp
+  module Media
+    module Type
+      class Iso < Simp::Media::Type::Base
+
+      end
+    end
+  end
+end

data/lib/simp/media/type/local.rb

@@ -0,0 +1,36 @@
+require 'fileutils'
+module Simp
+  module Media
+    module Type
+      class Local < Simp::Media::Type::Base
+        def input_directory=(directory)
+          @input_directory = directory
+        end
+        def input_directory
+          @options["input"]
+        end
+        def fetch_component(component, options)
+          retval = {}
+
+          case component.class.to_s
+            when "String"
+              retval["path"] = "#{options["input"]}/simp/metadata/#{component.name}"
+            when "Simp::Metadata::Component"
+              # XXX ToDo: Add manifest.yaml support so we don't need this logic at all
+              case component.component_type
+                when "documentation"
+                  subdirectory = "simp/docs"
+                when "puppet-module"
+                  subdirectory = "simp/modules"
+                else
+                  subdirectory = "simp/assets"
+              end
+
+              retval["path"] = "#{options["input"]}/#{subdirectory}/#{component.name}"
+          end
+          return retval
+        end
+      end
+    end
+  end
+end

data/lib/simp/media/type/tar.rb

@@ -0,0 +1,71 @@
+require 'fileutils'
+module Simp
+  module Media
+    module Type
+      class Tar < Simp::Media::Type::Base
+
+        def initialize(options, engine)
+          @cleanup = []
+
+          @origtempdir = Dir.mktmpdir("cachedir")
+          @tempdir = @origtempdir + "/" + File.basename(options["output"], ".*")
+          @cleanup << @origtempdir
+          super(options, engine)
+        end
+
+        def add_component(component, fetch_return_value)
+          case component.component_type
+            when "documentation"
+              subdirectory = "SIMP/docs"
+            when "simp-metadata"
+              subdirectory = "SIMP/metadata"
+            when "puppet-module"
+              subdirectory = "SIMP/modules"
+            else
+              subdirectory = "SIMP/assets/#{component.name}"
+          end
+          case component.output_type
+            when :directory
+              if (Dir.exists?(fetch_return_value["path"]))
+                unless (Dir.exists?(@tempdir + "/#{subdirectory}/#{component.name}"))
+                  FileUtils.mkdir_p(@tempdir + "/#{subdirectory}/#{component.name}")
+                end
+                FileUtils.cp_r(fetch_return_value["path"] + "/.", @tempdir + "/#{subdirectory}/#{component.output_filename}")
+              else
+                raise "Unable to find component #{component.name} in input source: path=#{fetch_return_value["path"]}"
+              end
+            when :file
+              if (File.exists?(fetch_return_value["path"]))
+                FileUtils.mkdir_p(@tempdir + "/#{subdirectory}")
+                FileUtils.cp_r(fetch_return_value["path"], @tempdir + "/#{subdirectory}/#{component.output_filename}")
+              else
+                raise "Unable to find component #{component.name} in input source: path=#{fetch_return_value["path"]}"
+              end
+          end
+
+        end
+
+        def finalize(manifest)
+          if @options.key?("local_directory")
+            if Dir.exists?(@options["local_directory"])
+              FileUtils.cp_r(Dir.glob(@options["local_directory"] + "/*"), @tempdir)
+            end
+          end
+          Dir.chdir(@origtempdir) do
+            `tar -cf - * | gzip -9nc >#{@options["output"]}`
+          end
+        end
+
+        def output(options, directory, version, metadata, output)
+          metadata.releases[version].components.each do |component|
+          end
+        end
+        def cleanup()
+          @cleanup.each do |path|
+            FileUtils.rmtree(path)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/simp/metadata.rb

@@ -0,0 +1,416 @@
+# vim: set expandtab ts=2 sw=2:
+require 'open3'
+require 'tempfile'
+require 'tmpdir'
+require 'net/http'
+require 'uri'
+require 'openssl'
+require 'json'
+require 'simp/metadata/engine'
+require 'simp/metadata/fake_uri'
+
+require 'simp/metadata/source'
+require 'simp/metadata/bootstrap_source'
+
+require 'simp/metadata/releases'
+require 'simp/metadata/release'
+
+require 'simp/metadata/components'
+require 'simp/metadata/component'
+require 'simp/metadata/buildinfo'
+
+require 'simp/metadata/locations'
+require 'simp/metadata/location'
+
+module Simp
+  module Metadata
+
+    def self.directory_name(component, options)
+      if (options["target"] != nil)
+        basedir = options["target"]
+      else
+        raise "Must specify 'target'"
+      end
+      case component.class.to_s
+        when "String"
+          "#{basedir}/#{component}"
+        when "Simp::Metadata::Component"
+          "#{basedir}/#{component.output_filename}"
+      end
+    end
+
+    # XXX: ToDo this entire logic stream is crappy.
+    #      We need to replace this with a much more simplified version.
+    def self.download_component(component, options)
+      directory_name = self.directory_name(component, options)
+      retval = {}
+      case component.class.to_s
+        when "String"
+          retval["path"] = self.directory_name(component, options)
+          # XXX: ToDo We can bootstrap this with a hard coded source in the simp engine
+          bootstrapped_components = {
+              "simp-metadata" => {
+                  "url" => "https://github.com/simp/simp-metadata",
+                  "method" => "git"
+              },
+              "enterprise-metadata" => {
+                  "url" => "simp-enterprise:///enterprise-metadata?version=master&filetype=tgz",
+                  "method" => "file",
+                  "extract" => true,
+              },
+          }
+          # All this should be removed and be based on component.file_type
+          componentspec = bootstrapped_components[component]
+          if (componentspec["extract"] == true)
+            tarball = "#{directory_name}.tgz"
+            fetch_from_url(componentspec, tarball, nil, options)
+            unless Dir.exists?(retval["path"])
+              Dir.mkdir(retval["path"])
+            end
+            `tar -xvpf #{tarball} -C #{retval["path"]}`
+          else
+            fetch_from_url(componentspec, retval["path"], nil, options)
+          end
+        when "Simp::Metadata::Component"
+          retval["path"] = directory_name
+          if (options["url"])
+            location = component.primary
+            location.url = options["url"]
+            urlspec = location
+            location.method = "git"
+          else
+            urlspec = component.primary
+          end
+          fetch_from_url(urlspec, retval["path"], component, options)
+        else
+          raise "component.class=#{component.class.to_s}, #{component.class.to_s} is not in ['String', 'Simp::Metadata::Component']"
+      end
+      return retval
+    end
+
+    def self.uri(url)
+      case url
+        when /git@/
+          uri = Simp::Metadata::FakeURI.new(uri)
+          uri.scheme = "ssh"
+          uri
+        else
+          URI(url)
+      end
+    end
+
+    def self.fetch_from_url(urlspec, target, component = nil, options)
+      case urlspec.class.to_s
+        when "Simp::Metadata::Location"
+          url = urlspec.url
+          uri = uri(url)
+          method = urlspec.method
+        when "Hash"
+          url = urlspec["url"]
+          uri = uri(urlspec["url"])
+          if (urlspec.key?("method"))
+            method = urlspec["method"]
+          else
+            # XXX ToDo remove once the upstream simp-metadata has been updated so type != method
+            if (urlspec.key?("type"))
+              if (urlspec["type"] == "git")
+                method = "git"
+              else
+                method = "file"
+              end
+            else
+              method = "file"
+            end
+          end
+        when "String"
+          url = urlspec
+          uri = uri(urlspec)
+          method = "file"
+      end
+      case method
+        when "git"
+          case uri.scheme
+            when "simp"
+              fetch_simp_enterprise(url, target, component, urlspec, options)
+            when "simp-enterprise"
+              fetch_simp_enterprise(url, target, component, urlspec, options)
+            else
+              unless (Dir.exists?(target))
+                info("Cloning from #{url}")
+                run("git clone #{url} #{target}")
+              else
+                Dir.chdir(target) do
+                  info("Updating from #{url}")
+                  run("git pull origin")
+                end
+              end
+          end
+        when "file"
+          case uri.scheme
+            when "simp"
+              fetch_simp_enterprise(url, target, component, urlspec, options)
+            when "simp-enterprise"
+              fetch_simp_enterprise(url, target, component, urlspec, options)
+            when "http"
+              fetch_simp_enterprise(url, target, component, urlspec, options)
+            when "https"
+              fetch_simp_enterprise(url, target, component, urlspec, options)
+            else
+              raise "unsupported url type #{uri.scheme}"
+          end
+      end
+    end
+
+    def self.get_license_data(filename)
+      ret_filename = nil
+      ret_data = ""
+      license_data = ENV.fetch('SIMP_LICENSE_KEY', nil)
+      if (license_data != nil)
+        # Environment data trumps all
+        ret_data = license_data
+        if ($simp_license_temp == nil)
+          $simp_license_temp = Tempfile.new('license_data')
+          $simp_license_temp.write(license_data)
+        end
+        ret_filename = $simp_license_temp.path
+      else
+        if (filename.class.to_s == "String")
+          # Attempt to load from the filename passed
+          ret_filename = filename
+        else
+          # Try to load from /etc/simp/license.key file
+          ret_filename = "/etc/simp/license.key"
+        end
+        if (File.exists?(ret_filename))
+          ret_data = File.read(ret_filename)
+        else
+          if ($simp_license_temp == nil)
+            $simp_license_temp = Tempfile.new('license_data')
+            $simp_license_temp.write("")
+          end
+          ret_filename = $simp_license_temp.path
+        end
+      end
+      return ret_filename, ret_data
+    end
+
+    def self.fetch_simp_enterprise(url, destination, component, location = nil, options)
+      if (location.class.to_s == "Simp::Metadata::Location")
+        extract = location.extract
+      else
+        extract = false
+      end
+      uri = uri(url)
+      case uri.scheme
+        when "simp-enterprise"
+          scheme = "https"
+          host = 'enterprise-download.simp-project.com'
+          filetype = 'tgz'
+          if (component != nil)
+            if (component.extension != "")
+              filetype = component.extension
+            end
+          end
+          version = 'latest'
+          if (component != nil)
+            if (component.version != "")
+              version = component.version
+            end
+          end
+          if (uri.query != nil)
+            uri.query.split("&").each do |element|
+              if (element.class.to_s == "String")
+                elements = element.split("=")
+                if (elements.size > 1)
+                  case elements[0]
+                    when "version"
+                      version = elements[1]
+                    when "filetype"
+                      filetype = elements[1]
+                  end
+                end
+              end
+            end
+          end
+
+          if (component != nil)
+            name = "/#{component.name}/#{component.binaryname}"
+          else
+            name = "#{uri.path}#{uri.path}#{name}-#{version}.#{filetype}"
+          end
+          path = "/products/simp-enterprise#{name}"
+        when "simp"
+          scheme = "https"
+          host = 'download.simp-project.com'
+          filetype = 'tgz'
+          if (component != nil)
+            if (component.extension != "")
+              filetype = component.extension
+            end
+          end
+          version = 'latest'
+          if (component != nil)
+            if (component.version != "")
+              version = component.version
+            end
+          end
+          if (uri.query != nil)
+            uri.query.split("&").each do |element|
+              if (element.class.to_s == "String")
+                elements = element.split("=")
+                if (elements.size > 1)
+                  case elements[0]
+                    when "version"
+                      version = elements[1]
+                    when "filetype"
+                      filetype = elements[1]
+                  end
+                end
+              end
+            end
+          end
+          if (component != nil)
+            name = "/#{component.name}/#{component.binaryname}"
+          else
+            name = "#{uri.path}#{uri.path}#{name}-#{version}.#{filetype}"
+          end
+          path = "/SIMP/assets#{name}"
+        else
+          scheme = uri.scheme
+          host = uri.host
+          path = uri.path
+      end
+      port = uri.port ? uri.port : 443
+      http = Net::HTTP.new(host, port)
+      case scheme
+        when "https"
+          http.use_ssl = true
+          case uri.scheme
+            when "simp-enterprise"
+              filename, data = self.get_license_data(options["license"])
+              unless (filename == nil)
+                http.ca_file = filename
+              end
+              unless (data == nil)
+                http.cert = OpenSSL::X509::Certificate.new(data)
+                http.key = OpenSSL::PKey::RSA.new(data)
+                http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+              end
+
+              debug2("using the following certificate (#{filename}) for client certificate auth: #{http.cert.subject}")
+          end
+      end
+      info("Fetching from #{scheme}://#{host}:#{port}#{path}")
+      req = Net::HTTP::Get.new(path)
+      response = http.request(req)
+      case response.code
+        when '200'
+          if (extract == true)
+            File.open("#{destination}.tgz", 'w') do |f|
+              f.write response.body
+            end
+            FileUtils.mkdir_p(destination)
+            run("tar -xvpf #{destination}.tgz -C #{destination}")
+          else
+            File.open(destination, 'w') do |f|
+              f.write response.body
+            end
+          end
+        when '302'
+          fetch_simp_enterprise(response['location'], destination, component, location)
+        when '301'
+          fetch_simp_enterprise(response['location'], destination, component, location)
+        else
+          $errorcode = response.code.to_i
+          raise "HTTP Error Code: #{response.code}"
+      end
+    end
+
+    def self.run(command)
+      exitcode = nil
+      Open3.popen3(command) do |stdin, stdout, stderr, thread|
+        pid = thread.pid
+        Simp::Metadata.debug2(stdout.read.chomp)
+        Simp::Metadata.debug1(stderr.read.chomp)
+        exitcode = thread.value
+      end
+      exitcode
+    end
+
+    def self.level?(level)
+      setlevel = Simp::Metadata.convert_level($simp_metadata_debug_level)
+      checklevel = Simp::Metadata.convert_level(level)
+      if (checklevel <= setlevel)
+        true
+      else
+        false
+      end
+    end
+
+    def self.convert_level(level)
+      case level
+        when 'disabled'
+          0
+        when 'critical'
+          1
+        when 'error'
+          2
+        when 'warning'
+          3
+        when 'info'
+          4
+        when 'debug1'
+          5
+        when 'debug2'
+          6
+        else
+          3
+      end
+    end
+
+    def self.print_message(prefix, message)
+      message.split("\n").each do |line|
+        output = "#{prefix}: #{line}"
+        unless ($simp_metadata_debug_output_disabled == true)
+          STDERR.puts output
+        end
+      end
+    end
+
+    def self.debug1(message)
+      if Simp::Metadata.level?('debug1')
+        Simp::Metadata.print_message("DEBUG1", message)
+      end
+    end
+
+    def self.debug2(message)
+      if Simp::Metadata.level?('debug2')
+        Simp::Metadata.print_message("DEBUG2", message)
+      end
+    end
+
+    def self.info(message)
+      if Simp::Metadata.level?('info')
+        Simp::Metadata.print_message("INFO", message)
+      end
+    end
+
+    def self.warning(message)
+      if Simp::Metadata.level?('warning')
+        Simp::Metadata.print_message("WARN", message)
+      end
+    end
+
+    def self.error(message)
+      if Simp::Metadata.level?('error')
+        Simp::Metadata.print_message("ERROR", message)
+      end
+    end
+
+    def self.critical(message)
+      if Simp::Metadata.level?('critical')
+        Simp::Metadata.print_message("CRITICAL", message)
+      end
+    end
+  end
+end