choria-mcorpc-support 0.0.1

data/lib/mcollective/pluginmanager.rb

@@ -0,0 +1,180 @@
+module MCollective
+  # A simple plugin manager, it stores one plugin each of a specific type
+  # the idea is that we can only have one security provider, one connector etc.
+  module PluginManager
+    @plugins = {}
+
+    # Adds a plugin to the list of plugins, we expect a hash like:
+    #
+    #    {:type => "base",
+    #     :class => foo.new}
+    #
+    # or like:
+    #    {:type => "base",
+    #     :class => "Foo::Bar"}
+    #
+    # In the event that we already have a class with the given type
+    # an exception will be raised.
+    #
+    # If the :class passed is a String then we will delay instantiation
+    # till the first time someone asks for the plugin, this is because most likely
+    # the registration gets done by inherited() hooks, at which point the plugin class is not final.
+    #
+    # If we were to do a .new here the Class initialize method would get called and not
+    # the plugins, we there for only initialize the classes when they get requested via []
+    #
+    # By default all plugin instances are cached and returned later so there's
+    # always a single instance.  You can pass :single_instance => false when
+    # calling this to instruct it to always return a new instance when a copy
+    # is requested.  This only works with sending a String for :class.
+    def self.<<(plugin)
+      plugin[:single_instance] = true unless plugin.include?(:single_instance)
+
+      type = plugin[:type]
+      klass = plugin[:class]
+      single = plugin[:single_instance]
+
+      raise("Plugin #{type} already loaded") if @plugins.include?(type)
+
+
+      # If we get a string then store 'nil' as the instance, signalling that we'll
+      # create the class later on demand.
+      if klass.is_a?(String)
+        @plugins[type] = {:loadtime => Time.now, :class => klass, :instance => nil, :single => single}
+        Log.debug("Registering plugin #{type} with class #{klass} single_instance: #{single}")
+      else
+        @plugins[type] = {:loadtime => Time.now, :class => klass.class, :instance => klass, :single => true}
+        Log.debug("Registering plugin #{type} with class #{klass.class} single_instance: true")
+      end
+    end
+
+    # Removes a plugim the list
+    def self.delete(plugin)
+      @plugins.delete(plugin) if @plugins.include?(plugin)
+    end
+
+    # Finds out if we have a plugin with the given name
+    def self.include?(plugin)
+      @plugins.include?(plugin)
+    end
+
+    # Provides a list of plugins we know about
+    def self.pluginlist
+      @plugins.keys.sort
+    end
+
+    # deletes all registered plugins
+    def self.clear
+      @plugins.clear
+    end
+
+    # Gets a plugin by type
+    def self.[](plugin)
+      raise("No plugin #{plugin} defined") unless @plugins.include?(plugin)
+
+      klass = @plugins[plugin][:class]
+
+      if @plugins[plugin][:single]
+        # Create an instance of the class if one hasn't been done before
+        if @plugins[plugin][:instance] == nil
+          Log.debug("Returning new plugin #{plugin} with class #{klass}")
+          @plugins[plugin][:instance] = create_instance(klass)
+        else
+          Log.debug("Returning cached plugin #{plugin} with class #{klass}")
+        end
+
+        @plugins[plugin][:instance]
+      else
+        Log.debug("Returning new plugin #{plugin} with class #{klass}")
+        create_instance(klass)
+      end
+    end
+
+    # use eval to create an instance of a class
+    def self.create_instance(klass)
+      begin
+        eval("#{klass}.new")
+      rescue Exception => e
+        raise("Could not create instance of plugin #{klass}: #{e}")
+      end
+    end
+
+    # Finds plugins in all configured libdirs
+    #
+    #   find("agent")
+    #
+    # will return an array of just agent names, for example:
+    #
+    #   ["puppetd", "package"]
+    #
+    # Can also be used to find files of other extensions:
+    #
+    #   find("agent", "ddl")
+    #
+    # Will return the same list but only of files with extension .ddl
+    # in the agent subdirectory
+    def self.find(type, extension="rb")
+      extension = ".#{extension}" unless extension.match(/^\./)
+
+      plugins = []
+
+      Config.instance.libdir.each do |libdir|
+        plugdir = File.join([libdir, "mcollective", type.to_s])
+        next unless File.directory?(plugdir)
+
+        Dir.new(plugdir).grep(/#{extension}$/).map do |plugin|
+          plugins << File.basename(plugin, extension)
+        end
+      end
+
+      plugins.sort.uniq
+    end
+
+    # Finds and loads from disk all plugins from all libdirs that match
+    # certain criteria.
+    #
+    #    find_and_load("pluginpackager")
+    #
+    # Will find all .rb files in the libdir/mcollective/pluginpackager/
+    # directory in all libdirs and load them from disk.
+    #
+    # You can influence what plugins get loaded using a block notation:
+    #
+    #    find_and_load("pluginpackager") do |plugin|
+    #       plugin.match(/puppet/)
+    #    end
+    #
+    # This will load only plugins matching /puppet/
+    def self.find_and_load(type, extension="rb")
+      extension = ".#{extension}" unless extension.match(/^\./)
+
+      klasses = find(type, extension).map do |plugin|
+        if block_given?
+          next unless yield(plugin)
+        end
+
+        "%s::%s::%s" % [ "MCollective", type.capitalize, plugin.capitalize ]
+      end.compact
+
+      klasses.sort.uniq.each {|klass| loadclass(klass, true)}
+    end
+
+    # Loads a class from file by doing some simple search/replace
+    # on class names and then doing a require.
+    def self.loadclass(klass, squash_failures=false)
+      fname = klass.gsub("::", "/").downcase + ".rb"
+
+      Log.debug("Loading #{klass} from #{fname}")
+
+      load fname
+    rescue Exception => e
+      Log.error("Failed to load #{klass}: #{e}")
+      raise unless squash_failures
+    end
+
+    # Grep's over the plugin list and returns the list found
+    def self.grep(regex)
+      @plugins.keys.grep(regex)
+    end
+  end
+end

data/lib/mcollective/pluginpackager.rb

@@ -0,0 +1,98 @@
+module MCollective
+  module PluginPackager
+    # Plugin definition classes
+    require "mcollective/pluginpackager/agent_definition"
+    require "mcollective/pluginpackager/standard_definition"
+
+    # Package implementation plugins
+    def self.load_packagers
+      PluginManager.find_and_load("pluginpackager")
+    end
+
+    def self.[](klass)
+      const_get("#{klass}")
+    end
+
+    # Fetch and return metadata from plugin DDL
+    def self.get_metadata(path, type)
+      ddl = DDL.new("package", type.to_sym, false)
+
+      begin
+        ddl_file = File.read(Dir.glob(File.join(path, type, "*.ddl")).first)
+      rescue Exception
+        raise "failed to load ddl file in plugin directory : #{File.join(path, type)}"
+      end
+      ddl.instance_eval ddl_file
+
+      return ddl.meta, ddl.requirements[:mcollective]
+    end
+
+    # Checks if a directory is present and not empty
+    def self.check_dir_present(path)
+      (File.directory?(path) && !Dir.glob(File.join(path, "*")).empty?)
+    end
+
+    # Quietly calls a block if verbose parameter is false
+    def self.execute_verbosely(verbose, &block)
+      unless verbose
+        old_stdout = $stdout.clone
+        $stdout.reopen(File.new("/dev/null", "w"))
+        begin
+          block.call
+        rescue Exception => e
+          $stdout.reopen old_stdout
+          raise e
+        ensure
+          $stdout.reopen old_stdout
+        end
+      else
+        block.call
+      end
+    end
+
+    # Checks if a build tool is present on the system
+    def self.command_available?(build_tool)
+      ENV["PATH"].split(File::PATH_SEPARATOR).each do |path|
+        builder = File.join(path, build_tool)
+        if File.exists?(builder)
+          return true
+        end
+      end
+      false
+    end
+
+    def self.safe_system(*args)
+      raise(RuntimeError, "Failed: #{args.join(' ')}") unless system *args
+    end
+
+    # Filter out platform specific dependencies
+    # Given a list of dependencies named -
+    # debian::foo
+    # redhat::bar
+    # PluginPackager.filter_dependencies('debian', dependencies)
+    # will return foo.
+    def self.filter_dependencies(prefix, dependencies)
+      dependencies.map do |dependency|
+        if dependency[:name] =~ /^(\w+)::(\w+)/
+          if prefix == $1
+            dependency[:name] = $2
+            dependency
+          else
+            nil
+          end
+        else
+          dependency
+        end
+      end.reject{ |dependency| dependency == nil }
+    end
+
+    # Return the path to a plugin's core directories
+    def self.get_plugin_path(target)
+      if (File.exists?(File.join(target, "lib", "mcollective")))
+        return File.join(target, "lib", "mcollective")
+      end
+
+      return target
+    end
+  end
+end

data/lib/mcollective/pluginpackager/agent_definition.rb

@@ -0,0 +1,94 @@
+module MCollective
+  module PluginPackager
+    # MCollective Agent Plugin package
+    class AgentDefinition
+      attr_accessor :path, :packagedata, :metadata, :target_path, :vendor, :revision, :preinstall
+      attr_accessor :plugintype, :dependencies, :postinstall, :mcname, :mcversion
+
+      def initialize(configuration, mcdependency, plugintype)
+        @plugintype = plugintype
+        @path = PluginPackager.get_plugin_path(configuration[:target])
+        @packagedata = {}
+        @revision = configuration[:revision] || 1
+        @preinstall = configuration[:preinstall]
+        @postinstall = configuration[:postinstall]
+        @vendor = configuration[:vendor] || "Puppet Labs"
+        @dependencies = configuration[:dependency] || []
+        @target_path = File.expand_path(@path)
+        @metadata, mcversion = PluginPackager.get_metadata(@path, "agent")
+        @mcname = mcdependency[:mcname] ||  "mcollective"
+        @mcversion = mcdependency[:mcversion] || mcversion
+        @metadata[:version] = (configuration[:version] || @metadata[:version])
+        @dependencies << {:name => "#{@mcname}-common", :version => @mcversion}
+        @metadata[:name] = (configuration[:pluginname] || @metadata[:name]).downcase.gsub(/\s+|_/, "-")
+        identify_packages
+      end
+
+      # Identify present packages and populate packagedata hash.
+      def identify_packages
+        common_package = common
+        @packagedata[:common] = common_package if common_package
+        agent_package = agent
+        @packagedata[:agent] = agent_package if agent_package
+        client_package = client
+        @packagedata[:client] = client_package if client_package
+      end
+
+      # Obtain Agent package files and dependencies.
+      def agent
+        agent = {:files => [],
+                 :dependencies => @dependencies.clone,
+                 :description => "Agent plugin for #{@metadata[:name]}"}
+
+        agentdir = File.join(@path, "agent")
+
+        if (PluginPackager.check_dir_present(agentdir))
+          ddls = Dir.glob(File.join(agentdir, "*.ddl"))
+          agent[:files] = (Dir.glob(File.join(agentdir, "**", "**")) - ddls)
+        else
+          return nil
+        end
+        agent[:plugindependency] = {:name => "#{@mcname}-#{@metadata[:name]}-common", :version => @metadata[:version], :revision => @revision}
+        agent
+      end
+
+      # Obtain client package files and dependencies.
+      def client
+        client = {:files => [],
+                  :dependencies => @dependencies.clone,
+                  :description => "Client plugin for #{@metadata[:name]}"}
+
+        clientdir = File.join(@path, "application")
+        aggregatedir = File.join(@path, "aggregate")
+
+        client[:files] += Dir.glob(File.join(clientdir, "*")) if PluginPackager.check_dir_present clientdir
+        client[:files] += Dir.glob(File.join(aggregatedir, "*")) if PluginPackager.check_dir_present aggregatedir
+        client[:plugindependency] = {:name => "#{@mcname}-#{@metadata[:name]}-common", :version => @metadata[:version], :revision => @revision}
+        client[:files].empty? ? nil : client
+      end
+
+      # Obtain common package files and dependencies.
+      def common
+        common = {:files =>[],
+                  :dependencies => @dependencies.clone,
+                  :description => "Common libraries for #{@metadata[:name]}"}
+
+        datadir = File.join(@path, "data", "**")
+        utildir = File.join(@path, "util", "**", "**")
+        ddldir = File.join(@path, "agent", "*.ddl")
+        validatordir = File.join(@path, "validator", "**")
+
+        [datadir, utildir, validatordir, ddldir].each do |directory|
+          common[:files] += Dir.glob(directory)
+        end
+
+        # We fail if there is no ddl file present
+        if common[:files].grep(/^.*\.ddl$/).empty?
+          raise "cannot create package - No ddl file found in #{File.join(@path, "agent")}"
+        end
+
+        common[:files].empty? ? nil : common
+      end
+    end
+  end
+end

data/lib/mcollective/pluginpackager/debpackage_packager.rb

@@ -0,0 +1,237 @@
+module MCollective
+  module PluginPackager
+    class DebpackagePackager
+      require 'erb'
+
+      def initialize(plugin, pluginpath = nil, signature = nil, verbose = false, keep_artifacts = nil, module_template = nil)
+        if PluginPackager.command_available?('debuild')
+          @plugin = plugin
+          @verbose = verbose
+          @libdir = pluginpath || '/usr/share/mcollective/plugins/mcollective/'
+          @signature = signature
+          @package_name = "#{@plugin.mcname}-#{@plugin.metadata[:name]}"
+          @keep_artifacts = keep_artifacts
+        else
+          raise("Cannot build package. 'debuild' is not present on the system.")
+        end
+      end
+
+      # Build process :
+      # - create buildroot
+      # - craete buildroot/debian
+      # - create the relative directories with package contents
+      # - create install files for each of the plugins that are going to be built
+      # - create debian build files
+      # - create tarball
+      # - create pre and post install files
+      # - run the build script
+      # - move packages to cwd
+      # - clean up
+      def create_packages
+        begin
+          puts "Building packages for #{@package_name} plugin."
+
+          @tmpdir = Dir.mktmpdir('mcollective_packager')
+          @build_dir = File.join(@tmpdir, "#{@package_name}_#{@plugin.metadata[:version]}")
+          Dir.mkdir(@build_dir)
+
+          create_debian_dir
+          @plugin.packagedata.each do |type, data|
+            prepare_tmpdirs(data)
+            create_install_file(type, data)
+            create_pre_and_post_install(type)
+          end
+          create_debian_files
+          create_tar
+          run_build
+          move_packages
+
+          puts "Completed building all packages for #{@package_name} plugin."
+        ensure
+          if @keep_artifacts
+            puts 'Keeping build artifacts.'
+            puts "Build artifacts saved - #{@tmpdir}"
+          else
+            puts 'Removing build artifacts.'
+            cleanup_tmpdirs
+          end
+        end
+      end
+
+      private
+
+      def create_debian_files
+        ['control', 'Makefile', 'compat', 'rules', 'copyright', 'changelog'].each do |f|
+          create_file(f)
+        end
+      end
+
+      def run_build
+        FileUtils.cd(@build_dir) do
+          PluginPackager.execute_verbosely(@verbose) do
+            if @signature
+              if @signature.is_a?(String)
+                PluginPackager.safe_system("debuild --no-lintian -i -k#{@signature}")
+              else
+                PluginPackager.safe_system("debuild --no-lintian -i")
+              end
+            else
+              PluginPackager.safe_system("debuild --no-lintian -i -us -uc")
+            end
+          end
+        end
+      end
+
+      # Creates a string used by the control file to specify dependencies
+      # Dependencies can be formatted as :
+      # foo (>= x.x-x)
+      # foo (>= x.x)
+      # foo
+      def build_dependency_string(data)
+        dependencies = []
+        PluginPackager.filter_dependencies('debian', data[:dependencies]).each do |dep|
+          if dep[:version] && dep[:revision]
+            dependencies << "#{dep[:name]} (>=#{dep[:version]}-#{dep[:revision]}) | puppet-agent"
+          elsif dep[:version]
+            dependencies << "#{dep[:name]} (>=#{dep[:version]}) | puppet-agent"
+          else
+            dependencies << "#{dep[:name]} | puppet-agent"
+          end
+        end
+
+        if data[:plugindependency]
+          dependencies << "#{data[:plugindependency][:name]} (= ${binary:Version})"
+        end
+
+        dependencies.join(', ')
+      end
+
+      # Creates an install file for each of the packages that are going to be created
+      # for the plugin
+      def create_install_file(type, data)
+        install_file = "#{@package_name}-#{type}"
+        begin
+          install_file = File.join(@build_dir, 'debian', "#{install_file}.install")
+          File.open(install_file, 'w') do |file|
+            data[:files].each do |f|
+              extended_filename = File.join(@libdir, File.expand_path(f).gsub(/^#{@plugin.target_path}/, ''))
+              file.puts "#{extended_filename} #{File.dirname(extended_filename)}"
+            end
+          end
+        rescue Errno::EACCES => e
+          puts "Could not create install file '#{install_file}'. Permission denied"
+          raise e
+        rescue => e
+          puts "Could not create install file '#{install_file}'."
+          raise e
+        end
+      end
+
+      # Move source package and debs to cwd
+      def move_packages
+        begin
+          files_to_copy = Dir.glob(File.join(@tmpdir, '*.{deb,dsc,diff.gz,orig.tar.gz,changes}'))
+          FileUtils.cp(files_to_copy, '.')
+        rescue => e
+          puts 'Could not copy packages to working directory.'
+          raise e
+        end
+      end
+
+      # Create pre and post install files in $buildroot/debian
+      # from supplied scripts.
+      # Note that all packages built for the plugin will invoke
+      # the same pre and post install scripts.
+      def create_pre_and_post_install(type)
+        if @plugin.preinstall
+          if !File.exists?(@plugin.preinstall)
+            puts "pre-install script '#{@plugin.preinstall}' not found."
+            raise(Errno::ENOENT, @plugin.preinstall)
+          else
+            FileUtils.cp(@plugin.preinstall, File.join(@build_dir, 'debian', "#{@package_name}-#{type}.preinst"))
+          end
+        end
+
+        if @plugin.postinstall
+          if !File.exists?(@plugin.postinstall)
+            puts "post-install script '#{@plugin.postinstall}' not found."
+            raise(Errno::ENOENT, @plugin.postinstall)
+          else
+            FileUtils.cp(@plugin.postinstall, File.join(@build_dir, 'debian', "#{@package_name}-#{type}.postinst"))
+          end
+        end
+      end
+
+      # Tar up source
+      # Expects directory : $mcollective-$agent_$version
+      # Creates file : $buildroot/$mcollective-$agent_$version.orig.tar.gz
+      def create_tar
+        name_and_version = "#{@package_name}_#{@plugin.metadata[:version]}"
+        tarfile = "#{name_and_version}.orig.tar.gz"
+        begin
+          PluginPackager.execute_verbosely(@verbose) do
+            Dir.chdir(@tmpdir) do
+              PluginPackager.safe_system("tar -Pcvzf #{File.join(@tmpdir, tarfile)} #{name_and_version}")
+            end
+          end
+        rescue Exception => e
+          puts "Could not create tarball - #{tarfile}"
+          raise e
+        end
+      end
+
+      def create_file(filename)
+        begin
+          file = ERB.new(File.read(File.join(File.dirname(__FILE__), 'templates', 'debian', "#{filename}.erb")), nil, '-')
+          File.open(File.join(@build_dir, 'debian', filename), 'w') do |f|
+            f.puts file.result(binding)
+          end
+        rescue => e
+          puts "Could not create file - '#{filename}'"
+          raise e
+        end
+      end
+
+      # Move files contained in the plugin to the correct directory
+      # relative to the build root.
+      def prepare_tmpdirs(data)
+        data[:files].each do |file|
+          begin
+            targetdir = File.join(@build_dir, @libdir, File.dirname(File.expand_path(file)).gsub(/^#{@plugin.target_path}/, ""))
+            FileUtils.mkdir_p(targetdir) unless File.directory?(targetdir)
+            FileUtils.cp_r(file, targetdir)
+          rescue Errno::EACCES => e
+            puts "Could not create directory '#{targetdir}'. Permission denied"
+            raise e
+          rescue Errno::ENOENT => e
+            puts "Could not copy file '#{file}' to '#{targetdir}'. File does not exist"
+            raise e
+          rescue => e
+            puts 'Could not prepare build directory'
+            raise e
+          end
+        end
+      end
+
+      # Create the $buildroot/debian directory
+      def create_debian_dir
+        deb_dir = File.join(@build_dir, 'debian')
+        begin
+          FileUtils.mkdir_p(deb_dir)
+        rescue => e
+          puts "Could not create directory '#{deb_dir}'"
+          raise e
+        end
+      end
+
+      def cleanup_tmpdirs
+        begin
+          FileUtils.rm_r(@tmpdir) if File.directory?(@tmpdir)
+        rescue => e
+          puts "Could not remove temporary build directory - '#{@tmpdir}'"
+          raise e
+        end
+      end
+    end
+  end
+end