octofacts-updater 0.5.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 05efcd40e8410c43eb2fd197165de57b754d0e28
+  data.tar.gz: ad65e6ce8d288593e0b2b03075fcda6f52fbd4cb
+SHA512:
+  metadata.gz: 507c0eecd6dd8c7c281e6734f1dbdde0731ee2e8cfcedd2bc5f9d7fc68fc7805207d5032d3ffc1d20df0e75154afb3786e0ff6f93d03af9e271592d9d45996ec
+  data.tar.gz: e017a38aa6681a768d06cc085f4dc4480e4ef736d6782fd6f05a9d2b24b391bc5911b9e0d47791f080ea3321c96b3117f44cd864e60ba2108f95c22c675e60f7

data/.version

@@ -0,0 +1 @@
+0.5.0

data/bin/octofacts-updater

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "octofacts_updater"
+cli = OctofactsUpdater::CLI.new(ARGV)
+cli.run

data/lib/octofacts_updater.rb

@@ -0,0 +1,19 @@
+require "octofacts_updater/cli"
+require "octofacts_updater/fact"
+require "octofacts_updater/fact_index"
+require "octofacts_updater/fixture"
+require "octofacts_updater/plugin"
+require "octofacts_updater/plugins/ip"
+require "octofacts_updater/plugins/ssh"
+require "octofacts_updater/plugins/static"
+require "octofacts_updater/service/base"
+require "octofacts_updater/service/enc"
+require "octofacts_updater/service/github"
+require "octofacts_updater/service/local_file"
+require "octofacts_updater/service/puppetdb"
+require "octofacts_updater/service/ssh"
+require "octofacts_updater/version"
+
+module OctofactsUpdater
+  #
+end

data/lib/octofacts_updater/cli.rb

@@ -0,0 +1,239 @@
+# :nocov:
+require "optparse"
+
+module OctofactsUpdater
+  class CLI
+    # Constructor.
+    #
+    # argv - The Array with command line arguments.
+    def initialize(argv)
+      @opts = {}
+      OptionParser.new(argv) do |opts|
+        opts.banner = "Usage: octofacts-updater [options]"
+
+        opts.on("-a", "--action <action>", String, "Action to take") do |a|
+          @opts[:action] = a
+        end
+
+        opts.on("-c", "--config <config_file>", String, "Path to configuration file") do |f|
+          raise "Invalid configuration file" unless File.file?(f)
+          @opts[:config] = f
+        end
+
+        opts.on("-H", "--hostname <hostname>", String, "FQDN of the host whose facts are to be gathered") do |h|
+          @opts[:hostname] = h
+        end
+
+        opts.on("-o", "--output-file <filename>", String, "Path to output file to write") do |i|
+          @opts[:output_file] = i
+        end
+
+        opts.on("-l", "--list <host1,host2,...>", Array, "List of hosts to update or index") do |l|
+          @opts[:host_list] = l
+        end
+
+        opts.on("--[no-]quick", "Quick indexing: Use existing YAML fact fixtures when available") do |q|
+          @opts[:quick] = q
+        end
+
+        opts.on("-p", "--path <directory>", "Path where to read/write host fixtures when working in bulk") do |path|
+          @opts[:path] = path
+        end
+
+        opts.on("--github", "Push any changes to a branch on GitHub (requires --action=bulk)") do
+          @opts[:github] ||= {}
+          @opts[:github][:enabled] = true
+        end
+
+        opts.on("--datasource <datasource>", "Specify the data source to use when retrieving facts (localfile, puppetdb, ssh)") do |ds|
+          unless %w{localfile puppetdb ssh}.include?(ds)
+            raise ArgumentError, "Invalid datasource #{ds.inspect}. Acceptable values: localfile, puppetdb, ssh."
+          end
+          @opts[:datasource] = ds.to_sym
+        end
+
+        opts.on("--config-override <section:key=value>", Array, "Override a portion of the configuration") do |co_array|
+          co_array.each do |co|
+            if co =~ /\A(\w+):(\S+?)=(.+?)\z/
+              @opts[Regexp.last_match(1).to_sym] ||= {}
+              @opts[Regexp.last_match(1).to_sym][Regexp.last_match(2).to_sym] = Regexp.last_match(3)
+            else
+              raise ArgumentError, "Malformed argument: --config-override must be in the format section:key=value"
+            end
+          end
+        end
+      end.parse!
+      validate_cli
+    end
+
+    def usage
+      puts "Usage: octofacts-updater --action <action> [--config-file /path/to/config.yaml] [other options]"
+      puts ""
+      puts "Available actions:"
+      puts "  bulk:  Update fixtures and index in bulk"
+      puts "  facts: Obtain facts for one node (requires --hostname <hostname>)"
+      puts ""
+    end
+
+    # Run method. Call this to run the octofacts updater with the object that was
+    # previously construcuted.
+    def run
+      unless opts[:action]
+        usage
+        exit 255
+      end
+
+      @config = {}
+
+      if opts[:config]
+        @config = YAML.load_file(opts[:config])
+        substitute_relative_paths!(@config, File.dirname(opts[:config]))
+        load_plugins(@config["plugins"]) if @config.key?("plugins")
+      end
+
+      @config[:options] = {}
+      opts.each do |k, v|
+        if v.is_a?(Hash)
+          @config[k.to_s] ||= {}
+          v.each do |v_key, v_val|
+            @config[k.to_s][v_key.to_s] = v_val
+            @config[k.to_s].delete(v_key.to_s) if v_val.nil?
+          end
+        else
+          @config[:options][k] = v
+        end
+      end
+
+      return handle_action_bulk if opts[:action] == "bulk"
+      return handle_action_facts if opts[:action] == "facts"
+
+      usage
+      exit 255
+    end
+
+    def substitute_relative_paths!(object_in, basedir)
+      if object_in.is_a?(Hash)
+        object_in.each { |k, v| object_in[k] = substitute_relative_paths!(v, basedir) }
+      elsif object_in.is_a?(Array)
+        object_in.map! { |v| substitute_relative_paths!(v, basedir) }
+      elsif object_in.is_a?(String)
+        if object_in =~ %r{^\.\.?(/|\z)}
+          object_in = File.expand_path(object_in, basedir)
+        end
+        object_in
+      else
+        object_in
+      end
+    end
+
+    def handle_action_bulk
+      facts_to_index = @config.fetch("index", {})["indexed_facts"]
+      unless facts_to_index.is_a?(Array)
+        raise ArgumentError, "Must declare index:indexed_facts in configuration to use bulk update"
+      end
+
+      nodes = if opts[:host_list]
+        opts[:host_list]
+      elsif opts[:hostname]
+        [opts[:hostname]]
+      else
+        OctofactsUpdater::FactIndex.load_file(index_file).nodes(true)
+      end
+      if nodes.empty?
+        raise ArgumentError, "Cannot run bulk update with no nodes to check"
+      end
+
+      path = opts[:path] || @config.fetch("index", {})["node_path"]
+      paths = []
+
+      fixtures = nodes.map do |hostname|
+        if opts[:quick] && path && File.file?(File.join(path, "#{hostname}.yaml"))
+          OctofactsUpdater::Fixture.load_file(hostname, File.join(path, "#{hostname}.yaml"))
+        else
+          fixture = OctofactsUpdater::Fixture.make(hostname, @config)
+          if path && File.directory?(path)
+            fixture.write_file(File.join(path, "#{hostname}.yaml"))
+            paths << File.join(path, "#{hostname}.yaml")
+          end
+          fixture
+        end
+      end
+
+      index = OctofactsUpdater::FactIndex.load_file(index_file)
+      index.reindex(facts_to_index, fixtures)
+      index.write_file
+      paths << index_file
+
+      if opts[:github] && opts[:github][:enabled]
+        OctofactsUpdater::Service::GitHub.run(config["github"]["base_directory"], paths, @config)
+      end
+    end
+
+    def handle_action_facts
+      unless opts[:hostname]
+        raise ArgumentError, "--hostname <hostname> must be specified to use --action facts"
+      end
+
+      facts_for_one_node
+    end
+
+    private
+
+    attr_reader :config, :opts
+
+    # Determine the facts for one node and print to the console or write to the specified file.
+    def facts_for_one_node
+      fixture = OctofactsUpdater::Fixture.make(opts[:hostname], @config)
+      print_or_write(fixture.to_yaml)
+    end
+
+    # Get the index file from the options or configuration file. Raise error if it does not exist or
+    # was not specified.
+    def index_file
+      @index_file ||= begin
+        if config.fetch("index", {})["file"]
+          return config["index"]["file"] if File.file?(config["index"]["file"])
+          raise Errno::ENOENT, "Index file (#{config['index']['file'].inspect}) does not exist"
+        end
+        raise ArgumentError, "No index file specified on command line (--index-file) or in configuration file"
+      end
+    end
+
+    # Load plugins as per configuration file. Note: all plugins embedded in this gem are automatically
+    # loaded. This is just for user-specified plugins.
+    #
+    # plugins - An Array of file names to load
+    def load_plugins(plugins)
+      unless plugins.is_a?(Array)
+        raise ArgumentError, "load_plugins expects an array, got #{plugins.inspect}"
+      end
+
+      plugins.each do |plugin|
+        plugin_file = plugin.start_with?("/") ? plugin : File.expand_path("../../#{plugin}", File.dirname(__FILE__))
+        unless File.file?(plugin_file)
+          raise Errno::ENOENT, "Failed to find plugin #{plugin.inspect} at #{plugin_file}"
+        end
+        require plugin_file
+      end
+    end
+
+    # Print or write to file depending on whether or not the output file was set.
+    #
+    # data - Data to print or write.
+    def print_or_write(data)
+      if opts[:output_file]
+        File.open(opts[:output_file], "w") { |f| f.write(data) }
+      else
+        puts data
+      end
+    end
+
+    # Validate command line options. Kick out invalid combinations of options immediately.
+    def validate_cli
+      if opts[:path] && !File.directory?(opts[:path])
+        raise Errno::ENOENT, "An existing directory must be specified with -p/--path"
+      end
+    end
+  end
+end
+# :nocov:

data/lib/octofacts_updater/fact.rb

@@ -0,0 +1,145 @@
+# This class represents a fact, either structured or unstructured.
+# The fact has a name and a value. The name is a string, and the value
+# can either be a string/integer/boolean (unstructured) or a hash (structured).
+# This class also has methods used to deal with structured facts (in particular, allowing
+# representation of a structure delimited with ::).
+
+module OctofactsUpdater
+  class Fact
+    attr_reader :name
+
+    # Constructor.
+    #
+    # name  - The String naming the fact.
+    # value - The arbitrary object with the value of the fact.
+    def initialize(name, value)
+      @name = name
+      @value = value
+    end
+
+    # Get the value of the fact. If the name is specified, this will dig into a structured fact to pull
+    # out the value within the structure.
+    #
+    # name_in - An optional String to dig into the structure (formatted with :: indicating hash delimiters)
+    #
+    # Returns the value of the fact.
+    def value(name_in = nil)
+      # Just a normal lookup -- return the value
+      return @value if name_in.nil?
+
+      # Structured lookup returns nil unless the fact is actually structured.
+      return unless @value.is_a?(Hash)
+
+      # Dig into the hash to pull out the desired value.
+      pointer = @value
+      parts = name_in.split("::")
+      last_part = parts.pop
+
+      parts.each do |part|
+        return unless pointer[part].is_a?(Hash)
+        pointer = pointer[part]
+      end
+
+      pointer[last_part]
+    end
+
+    # Set the value of the fact.
+    #
+    # new_value - An object with the new value for the fact
+    def value=(new_value)
+      set_value(new_value)
+    end
+
+    # Set the value of the fact. If the name is specified, this will dig into a structured fact to set
+    # the value within the structure.
+    #
+    # new_value - An object with the new value for the fact
+    # name_in   - An optional String to dig into the structure (formatted with :: indicating hash delimiters)
+    def set_value(new_value, name_in = nil)
+      if name_in.nil?
+        if new_value.is_a?(Proc)
+          return @value = new_value.call(@value)
+        end
+
+        return @value = new_value
+      end
+
+      parts = if name_in.is_a?(String)
+        name_in.split("::")
+      elsif name_in.is_a?(Array)
+        name_in.map do |item|
+          if item.is_a?(String)
+            item
+          elsif item.is_a?(Hash) && item.key?("regexp")
+            Regexp.new(item["regexp"])
+          else
+            raise ArgumentError, "Unable to interpret structure item: #{item.inspect}"
+          end
+        end
+      else
+        raise ArgumentError, "Unable to interpret structure: #{name_in.inspect}"
+      end
+
+      set_structured_value(@value, parts, new_value)
+    end
+
+    private
+
+    # Set a value in the data structure of a structured fact. This is intended to be
+    # called recursively.
+    #
+    # subhash - The Hash, part of the fact, being operated upon
+    # parts   - The Array to dig in to the hash
+    # value   - The value to set the ultimate last part to
+    #
+    # Does not return anything, but modifies 'subhash'
+    def set_structured_value(subhash, parts, value)
+      return if subhash.nil?
+      raise ArgumentError, "Cannot set structured value at #{parts.first.inspect}" unless subhash.is_a?(Hash)
+      raise ArgumentError, "parts must be an Array, got #{parts.inspect}" unless parts.is_a?(Array)
+
+      # At the top level, find all keys that match the first item in the parts.
+      matching_keys = subhash.keys.select do |key|
+        if parts.first.is_a?(String)
+          key == parts.first
+        elsif parts.first.is_a?(Regexp)
+          parts.first.match(key)
+        else
+          # :nocov:
+          # This is a bug - this code should be unreachable because of the checking in `set_value`
+          raise ArgumentError, "part must be a string or regexp, got #{parts.first.inspect}"
+          # :nocov:
+        end
+      end
+
+      # Auto-create a new hash if there is a value, the part is a string, and the key doesn't exist.
+      if parts.first.is_a?(String) && !value.nil? && !subhash.key?(parts.first)
+        subhash[parts.first] = {}
+        matching_keys << parts.first
+      end
+      return unless matching_keys.any?
+
+      # If we are at the end, set the value or delete the key.
+      if parts.size == 1
+        if value.nil?
+          matching_keys.each { |k| subhash.delete(k) }
+        elsif value.is_a?(Proc)
+          matching_keys.each do |k|
+            new_value = value.call(subhash[k])
+            if new_value.nil?
+              subhash.delete(k)
+            else
+              subhash[k] = new_value
+            end
+          end
+        else
+          matching_keys.each { |k| subhash[k] = value }
+        end
+        return
+      end
+
+      # We are not at the end. Recurse down to the next level.
+      matching_keys.each { |k| set_structured_value(subhash[k], parts[1..-1], value) }
+    end
+  end
+end

data/lib/octofacts_updater/fact_index.rb

@@ -0,0 +1,164 @@
+# This class represents a fact index, which is ultimately represented by a YAML file of
+# each index fact, the values seen, and the node(s) containing each value.
+#
+# fact_one:
+#   value_one:
+#     - node-1.example.net
+#     - node-2.example.net
+#   value_three:
+#     - node-3.example.net
+# fact_two:
+#   value_abc:
+#     - node-1.example.net
+#   value_def:
+#     - node-2.example.net
+#     - node-3.example.net
+
+require "set"
+require "yaml"
+
+module OctofactsUpdater
+  class FactIndex
+    # We will create a pseudo-fact that simply lists all of the nodes that were considered
+    # in the index. Define the name of that pseudo-fact here.
+    TOP_LEVEL_NODES_KEY = "_nodes".freeze
+
+    attr_reader :index_data
+
+    # Load an index from the YAML file.
+    #
+    # filename - A String with the file to be loaded.
+    #
+    # Returns a OctofactsUpdater::FactIndex object.
+    def self.load_file(filename)
+      unless File.file?(filename)
+        raise Errno::ENOENT, "load_index cannot load #{filename.inspect}"
+      end
+
+      data = YAML.safe_load(File.read(filename))
+      new(data, filename: filename)
+    end
+
+    # Constructor.
+    #
+    # data     - A Hash of existing index data.
+    # filename - Optionally, a String with a file name to write the index to
+    def initialize(data = {}, filename: nil)
+      @index_data = data
+      @filename = filename
+    end
+
+    # Add a fact to the index. If the fact already exists in the index, this will overwrite it.
+    #
+    # fact_name - A String with the name of the fact
+    # fixtures  - An Array with fact fixtures (must respond to .facts and .hostname)
+    def add(fact_name, fixtures)
+      @index_data[fact_name] ||= {}
+      fixtures.each do |fixture|
+        fact_value = get_fact(fixture, fact_name)
+        next if fact_value.nil?
+        @index_data[fact_name][fact_value] ||= []
+        @index_data[fact_name][fact_value] << fixture.hostname
+      end
+    end
+
+    # Get a list of all of the nodes in the index. This supports a quick mode (default) where the
+    # TOP_LEVEL_NODES_KEY key is used, and a more detailed mode where this digs through each indexed
+    # fact and value to build a list of nodes.
+    #
+    # quick_mode - Boolean whether to use quick mode (default=true)
+    #
+    # Returns an Array of nodes whose facts are indexed.
+    def nodes(quick_mode = true)
+      if quick_mode && @index_data.key?(TOP_LEVEL_NODES_KEY)
+        return @index_data[TOP_LEVEL_NODES_KEY]
+      end
+
+      seen_hosts = Set.new
+      @index_data.each do |fact_name, fact_values|
+        next if fact_name == TOP_LEVEL_NODES_KEY
+        fact_values.each do |_fact_value, nodes|
+          seen_hosts.merge(nodes)
+        end
+      end
+      seen_hosts.to_a.sort
+    end
+
+    # Rebuild an index with a specified list of facts. This will remove any indexed facts that
+    # are not on the list of facts to use.
+    #
+    # facts_to_index - An Array of Strings with facts to index
+    # fixtures       - An Array with fact fixtures (must respond to .facts and .hostname)
+    def reindex(facts_to_index, fixtures)
+      @index_data = {}
+      facts_to_index.each { |fact| add(fact, fixtures) }
+      set_top_level_nodes_fact(fixtures)
+    end
+
+    # Create the top level nodes pseudo-fact.
+    #
+    # fixtures - An Array with fact fixtures (must respond to .hostname)
+    def set_top_level_nodes_fact(fixtures)
+      @index_data[TOP_LEVEL_NODES_KEY] = fixtures.map { |f| f.hostname }.sort
+    end
+
+    # Get YAML representation of the index.
+    # This sorts the hash and any arrays without modifying the object.
+    def to_yaml
+      YAML.dump(recursive_sort(index_data))
+    end
+
+    def recursive_sort(object_in)
+      if object_in.is_a?(Hash)
+        object_out = {}
+        object_in.keys.sort.each { |k| object_out[k] = recursive_sort(object_in[k]) }
+        object_out
+      elsif object_in.is_a?(Array)
+        object_in.sort.map { |v| recursive_sort(v) }
+      else
+        object_in
+      end
+    end
+
+    # Write the fact index out to a YAML file.
+    #
+    # filename - A String with the file to write (defaults to filename from constructor if available)
+    def write_file(filename = nil)
+      filename ||= @filename
+      unless filename.is_a?(String)
+        raise ArgumentError, "Called write_file() for fact_index without a filename"
+      end
+      File.open(filename, "w") { |f| f.write(to_yaml) }
+    end
+
+    private
+
+    # Extract a (possibly) structured fact.
+    #
+    # fixture   - Fact fixture, must respond to .facts
+    # fact_name - A String with the name of the fact
+    #
+    # Returns the value of the fact, or nil if fact or structure does not exist.
+    def get_fact(fixture, fact_name)
+      pointer = fixture.facts
+
+      # Get the fact of interest from the fixture, whether structured or not.
+      components = fact_name.split(".")
+      first_component = components.shift
+      return unless pointer.key?(first_component)
+
+      # For simple non-structured facts, just return the value.
+      return pointer[first_component].value if components.empty?
+
+      # Structured facts: dig into the structure.
+      pointer = pointer[first_component].value
+      last_component = components.pop
+      components.each do |part|
+        return unless pointer.key?(part)
+        return unless pointer[part].is_a?(Hash)
+        pointer = pointer[part]
+      end
+      pointer[last_component]
+    end
+  end
+end