octofacts-updater 0.5.0

data/lib/octofacts_updater/fixture.rb

@@ -0,0 +1,136 @@
+# This class represents a fact fixture, which is a set of facts along with a node name.
+# Facts are OctofactsUpdater::Fact objects, and internally are stored as a hash table
+# with the key being the fact name and the value being the OctofactsUpdater::Fact object.
+
+require "yaml"
+
+module OctofactsUpdater
+  class Fixture
+    attr_reader :facts, :hostname
+
+    # Make a fact fixture for the specified host name by consulting data sources
+    # specified in the configuration.
+    #
+    # hostname - A String with the FQDN of the host.
+    # config   - A Hash with configuration data.
+    #
+    # Returns the OctofactsUpdater::Fixture object.
+    def self.make(hostname, config)
+      fact_hash = facts_from_configured_datasource(hostname, config)
+
+      if config.key?("enc")
+        enc_data = OctofactsUpdater::Service::ENC.run_enc(hostname, config)
+        if enc_data.key?("parameters")
+          fact_hash.merge! enc_data["parameters"]
+        end
+      end
+
+      obj = new(hostname, config, fact_hash)
+      obj.execute_plugins!
+    end
+
+    # Get fact hash from the first configured and working data source.
+    #
+    # hostname - A String with the FQDN of the host.
+    # config   - A Hash with configuration data.
+    #
+    # Returns a Hash with the facts for the specified node; raises exception if this was not possible.
+    def self.facts_from_configured_datasource(hostname, config)
+      last_exception = nil
+      data_sources = %w(LocalFile PuppetDB SSH)
+      data_sources.each do |ds|
+        next if config.fetch(:options, {})[:datasource] && config[:options][:datasource] != ds.downcase.to_sym
+        next unless config.key?(ds.downcase)
+        clazz = Kernel.const_get("OctofactsUpdater::Service::#{ds}")
+        begin
+          result = clazz.send(:facts, hostname, config)
+          return result["values"] if result["values"].is_a?(Hash)
+          return result
+        rescue => e
+          last_exception = e
+        end
+      end
+
+      raise last_exception if last_exception
+      raise ArgumentError, "No fact data sources were configured"
+    end
+
+    # Load a fact fixture from a file. This helps create an index without the more expensive operation
+    # of actually looking up the facts from the data source.
+    #
+    # hostname - A String with the FQDN of the host.
+    # filename - A String with the filename of the existing host.
+    #
+    # Returns the OctofactsUpdater::Fixture object.
+    def self.load_file(hostname, filename)
+      unless File.file?(filename)
+        raise Errno::ENOENT, "Could not load facts from #{filename} because it does not exist"
+      end
+
+      data = YAML.safe_load(File.read(filename))
+      new(hostname, {}, data)
+    end
+
+    # Constructor.
+    #
+    # hostname  - A String with the FQDN of the host.
+    # config    - A Hash with configuration data.
+    # fact_hash - A Hash with the facts (key = fact name, value = fact value).
+    def initialize(hostname, config, fact_hash = {})
+      @hostname = hostname
+      @config = config
+      @facts = Hash[fact_hash.collect { |k, v| [k, OctofactsUpdater::Fact.new(k, v)] }]
+    end
+
+    # Execute plugins to clean up facts as per configuration. This modifies the value of the facts
+    # stored in this object. Any facts with a value of nil are removed.
+    #
+    # Returns a copy of this object.
+    def execute_plugins!
+      return self unless @config["facts"].is_a?(Hash)
+
+      @config["facts"].each do |fact_tag, args|
+        fact_names(fact_tag, args).each do |fact_name|
+          @facts[fact_name] ||= OctofactsUpdater::Fact.new(fact_name, nil)
+          plugin_name = args.fetch("plugin", "noop")
+          OctofactsUpdater::Plugin.execute(plugin_name, @facts[fact_name], args, @facts)
+          @facts.delete(fact_name) if @facts[fact_name].value.nil?
+        end
+      end
+
+      self
+    end
+
+    # Get fact names associated with a particular data structure. Implements:
+    # - Default behavior, where YAML key = fact name
+    # - Regexp behavior, where YAML "regexp" key is used to match against all facts
+    # - Override behavior, where YAML "fact" key overrides whatever is in the tag
+    #
+    # fact_tag - A String with the YAML key
+    # args     - A Hash with the arguments
+    #
+    # Returns an Array of Strings with all fact names matched.
+    def fact_names(fact_tag, args = {})
+      return [args["fact"]] if args.key?("fact")
+      return [fact_tag] unless args.key?("regexp")
+      rexp = Regexp.new(args["regexp"])
+      @facts.keys.select { |k| rexp.match(k) }
+    end
+
+    # Write this fixture to a file.
+    #
+    # filename - A String with the filename to write.
+    def write_file(filename)
+      File.open(filename, "w") { |f| f.write(to_yaml) }
+    end
+
+    # YAML representation of the fact fixture.
+    #
+    # Returns a String containing the YAML representation of the fact fixture.
+    def to_yaml
+      sorted_facts = @facts.sort.to_h
+      facts_hash_with_expanded_values = Hash[sorted_facts.collect { |k, v| [k, v.value] }]
+      YAML.dump(facts_hash_with_expanded_values)
+    end
+  end
+end

data/lib/octofacts_updater/plugin.rb

@@ -0,0 +1,70 @@
+# This class provides the base methods for fact manipulation plugins.
+
+require "digest"
+
+module OctofactsUpdater
+  class Plugin
+    # Register a plugin.
+    #
+    # plugin_name - A Symbol which is the name of the plugin.
+    # block       - A block of code that constitutes the plugin. See sample plugins for expected format.
+    def self.register(plugin_name, &block)
+      @plugins ||= {}
+      if @plugins.key?(plugin_name.to_sym)
+        raise ArgumentError, "A plugin named #{plugin_name} is already registered."
+      end
+      @plugins[plugin_name.to_sym] = block
+    end
+
+    # Execute a plugin
+    #
+    # plugin_name - A Symbol which is the name of the plugin.
+    # fact        - An OctofactsUpdater::Fact object
+    # args        - An optional Hash of additional configuration arguments
+    # all_facts   - A Hash of all of the facts
+    #
+    # Returns nothing, but may adjust the "fact"
+    def self.execute(plugin_name, fact, args = {}, all_facts = {})
+      unless @plugins.key?(plugin_name.to_sym)
+        raise NoMethodError, "A plugin named #{plugin_name} could not be found."
+      end
+
+      begin
+        @plugins[plugin_name.to_sym].call(fact, args, all_facts)
+      rescue => e
+        warn "#{e.class} occurred executing #{plugin_name} on #{fact.name} with value #{fact.value.inspect}"
+        raise e
+      end
+    end
+
+    # Clear out a plugin definition. (Useful for testing.)
+    #
+    # plugin_name - The name of the plugin to clear.
+    def self.clear!(plugin_name)
+      @plugins ||= {}
+      @plugins.delete(plugin_name.to_sym)
+    end
+
+    # Get the plugins hash.
+    def self.plugins
+      @plugins
+    end
+
+    # ---------------------------
+    # Below this point are shared methods intended to be called by plugins.
+    # ---------------------------
+
+    # Randomize a long string. This method accepts a string (consisting of, for example, a SSH key)
+    # and returns a string of the same length, but with randomized characters.
+    #
+    # string_in - A String with the original fact value.
+    #
+    # Returns a String with the same length as string_in.
+    def self.randomize_long_string(string_in)
+      seed = Digest::MD5.hexdigest(string_in).to_i(36)
+      prng = Random.new(seed)
+      chars = [("a".."z"), ("A".."Z"), ("0".."9")].flat_map(&:to_a)
+      (1..(string_in.length)).map { chars[prng.rand(chars.length)] }.join
+    end
+  end
+end

data/lib/octofacts_updater/plugins/ip.rb

@@ -0,0 +1,38 @@
+# This file is part of the octofacts updater fact manipulation plugins. This plugin provides
+# methods to update facts that are IP addresses in order to anonymize or randomize them.
+
+require "ipaddr"
+
+# ipv4_anonymize. This method modifies an IP (version 4) address and
+# sets it to a randomized (yet consistent) address in the given
+# network.
+#
+# Supported parameters in args:
+# - subnet: (Required) The network prefix in CIDR notation
+OctofactsUpdater::Plugin.register(:ipv4_anonymize) do |fact, args = {}, facts|
+  raise ArgumentError, "ipv4_anonymize requires a subnet" if args["subnet"].nil?
+
+  subnet_range = IPAddr.new(args["subnet"], Socket::AF_INET).to_range
+  # Convert the original IP to an integer representation that we can use as seed
+  seed = IPAddr.new(fact.value(args["structure"]), Socket::AF_INET).to_i
+  srand seed
+  random_ip = IPAddr.new(rand(subnet_range.first.to_i..subnet_range.last.to_i), Socket::AF_INET)
+  fact.set_value(random_ip.to_s, args["structure"])
+end
+
+# ipv6_anonymize. This method modifies an IP (version 6) address and
+# sets it to a randomized (yet consistent) address in the given
+# network.
+#
+# Supported parameters in args:
+# - subnet: (Required) The network prefix in CIDR notation
+OctofactsUpdater::Plugin.register(:ipv6_anonymize) do |fact, args = {}, facts|
+  raise ArgumentError, "ipv6_anonymize requires a subnet" if args["subnet"].nil?
+
+  subnet_range = IPAddr.new(args["subnet"], Socket::AF_INET6).to_range
+  # Convert the hostname to an integer representation that we can use as seed
+  seed = IPAddr.new(fact.value(args["structure"]), Socket::AF_INET6).to_i
+  srand seed
+  random_ip = IPAddr.new(rand(subnet_range.first.to_i..subnet_range.last.to_i), Socket::AF_INET6)
+  fact.set_value(random_ip.to_s, args["structure"])
+end

data/lib/octofacts_updater/plugins/ssh.rb

@@ -0,0 +1,23 @@
+# This file is part of the octofacts updater fact manipulation plugins. This plugin provides
+# methods to update facts that are SSH keys, since we do not desire to commit SSH keys from
+# actual hosts into the source code repository.
+
+# sshfp. This method randomizes the secret key for sshfp formatted keys. Each key is replaced
+# by a randomized (yet consistent) string the same length as the input key.
+# The input looks like this:
+#  sshfp_ecdsa: |-
+#    SSHFP 3 1 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+#    SSHFP 3 2 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+OctofactsUpdater::Plugin.register(:sshfp_randomize) do |fact, args = {}|
+  blk = Proc.new do |val|
+    lines = val.split("\n").map(&:strip)
+    result = lines.map do |line|
+      unless line =~ /\ASSHFP (\d+) (\d+) (\w+)/
+        raise "Unparseable pattern: #{line}"
+      end
+      "SSHFP #{Regexp.last_match(1)} #{Regexp.last_match(2)} #{OctofactsUpdater::Plugin.randomize_long_string(Regexp.last_match(3))}"
+    end
+    result.join("\n")
+  end
+  fact.set_value(blk, args["structure"])
+end

data/lib/octofacts_updater/plugins/static.rb

@@ -0,0 +1,53 @@
+# This file is part of the octofacts updater fact manipulation plugins. This plugin provides
+# methods to do static operations on facts -- delete, add, or set to a known value.
+
+# Delete. This method deletes the fact or the identified portion. Setting the value to nil
+# causes the tooling to remove any such portions of the value.
+#
+# Supported parameters in args:
+# - structure: A String or Array of a structure within a structured fact
+OctofactsUpdater::Plugin.register(:delete) do |fact, args = {}, _all_facts = {}|
+  fact.set_value(nil, args["structure"])
+end
+
+# Set. This method sets the fact or the identified portion to a static value.
+#
+# Supported parameters in args:
+# - structure: A String or Array of a structure within a structured fact
+# - value: The new value to set the fact to
+OctofactsUpdater::Plugin.register(:set) do |fact, args = {}, _all_facts = {}|
+  fact.set_value(args["value"], args["structure"])
+end
+
+# Remove matching objects from a delimited string. Requires that the delimiter
+# and regular expression be set. This is useful, for example, to transform a
+# string like `foo,bar,baz,fizz` into `foo,fizz` (by removing /^ba/).
+#
+# Supported parameters in args:
+# - delimiter: (Required) Character that is the delimiter.
+# - regexp: (Required) String used to construct a regular expression of items to remove
+OctofactsUpdater::Plugin.register(:remove_from_delimited_string) do |fact, args = {}, _all_facts = {}|
+  unless fact.value.nil?
+    unless args["delimiter"]
+      raise ArgumentError, "remove_from_delimited_string requires a delimiter, got #{args.inspect}"
+    end
+    unless args["regexp"]
+      raise ArgumentError, "remove_from_delimited_string requires a regexp, got #{args.inspect}"
+    end
+    parts = fact.value.split(args["delimiter"])
+    regexp = Regexp.new(args["regexp"])
+    parts.delete_if { |part| regexp.match(part) }
+    fact.set_value(parts.join(args["delimiter"]))
+  end
+end
+
+# No-op. Do nothing at all.
+OctofactsUpdater::Plugin.register(:noop) do |_fact, _args = {}, _all_facts = {}|
+  #
+end
+
+# Randomize long string. This is just a wrapper around OctofactsUpdater::Plugin.randomize_long_string
+OctofactsUpdater::Plugin.register(:randomize_long_string) do |fact, args = {}, _all_facts = {}|
+  blk = Proc.new { |val| OctofactsUpdater::Plugin.randomize_long_string(val) }
+  fact.set_value(blk, args["structure"])
+end

data/lib/octofacts_updater/service/base.rb

@@ -0,0 +1,35 @@
+# This contains handy utility methods that might be used in any of the other classes.
+
+require "yaml"
+
+module OctofactsUpdater
+  module Service
+    class Base
+      # Parse a YAML fact file from PuppetServer. This removes the header (e.g. "--- !ruby/object:Puppet::Node::Facts")
+      # so that it's not necessary to bring in all of Puppet.
+      #
+      # yaml_string - A String with YAML to parse.
+      #
+      # Returns a Hash with the facts.
+      def self.parse_yaml(yaml_string)
+        # Convert first "---" after any comments and blank lines.
+        yaml_array = yaml_string.to_s.split("\n")
+        yaml_array.each_with_index do |line, index|
+          next if line =~ /\A\s*#/
+          next if line.strip == ""
+          if line.start_with?("---")
+            yaml_array[index] = "---"
+          end
+          break
+        end
+
+        # Parse the YAML file
+        result = YAML.safe_load(yaml_array.join("\n"))
+
+        # Pull out "values" if this is in a name-values format. Otherwise just return the hash.
+        return result["values"] if result["values"].is_a?(Hash)
+        result
+      end
+    end
+  end
+end

data/lib/octofacts_updater/service/enc.rb

@@ -0,0 +1,41 @@
+# This class contains methods to interact with an external node classifier.
+
+require "open3"
+require "shellwords"
+require "yaml"
+
+module OctofactsUpdater
+  module Service
+    class ENC
+      # Execute the external node classifier script. This expects the value of "path" to be
+      # set in the configuration.
+      #
+      # hostname - A String with the FQDN of the host.
+      # config   - A Hash with configuration data.
+      #
+      # Returns a Hash consisting of the parsed output of the ENC.
+      def self.run_enc(hostname, config)
+        unless config["enc"].is_a?(Hash)
+          raise ArgumentError, "The ENC configuration must be defined"
+        end
+
+        unless config["enc"]["path"].is_a?(String)
+          raise ArgumentError, "The ENC path must be defined"
+        end
+
+        unless File.file?(config["enc"]["path"])
+          raise Errno::ENOENT, "The ENC script could not be found at #{config['enc']['path'].inspect}"
+        end
+
+        command = [config["enc"]["path"], hostname].map { |x| Shellwords.escape(x) }.join(" ")
+        stdout, stderr, exitstatus = Open3.capture3(command)
+        unless exitstatus.exitstatus == 0
+          output = { "stdout" => stdout, "stderr" => stderr, "exitstatus" => exitstatus.exitstatus }
+          raise "Error executing #{command.inspect}: #{output.to_yaml}"
+        end
+
+        YAML.load(stdout)
+      end
+    end
+  end
+end

data/lib/octofacts_updater/service/github.rb

@@ -0,0 +1,230 @@
+# This class contains methods to interact with the GitHub API.
+# frozen_string_literal: true
+
+require "octokit"
+require "pathname"
+
+module OctofactsUpdater
+  module Service
+    class GitHub
+      attr_reader :options
+
+      # Callable external method: Push all changes to the indicated paths to GitHub.
+      #
+      # root    - A String with the root directory, to which paths are relative.
+      # paths   - An Array of Strings, which are relative to the repository root.
+      # options - A Hash with configuration options.
+      #
+      # Returns true if there were any changes made, false otherwise.
+      def self.run(root, paths, options = {})
+        root ||= options.fetch("github", {})["base_directory"]
+        unless root && File.directory?(root)
+          raise ArgumentError, "Base directory must be specified"
+        end
+        github = new(options)
+        project_root = Pathname.new(root)
+        paths.each do |path|
+          absolute_path = Pathname.new(path)
+          stripped_path = absolute_path.relative_path_from(project_root)
+          github.commit_data(stripped_path.to_s, File.read(path))
+        end
+        github.finalize_commit
+      end
+
+      # Constructor.
+      #
+      # options - Hash with options
+      def initialize(options = {})
+        @options = options
+        @verbose = github_options.fetch("verbose", false)
+        @changes = []
+      end
+
+      # Commit a file to a location in the repository with the provided message. This will return true if there was
+      # an actual change, and false otherwise. This method does not actually do the commit, but rather it batches up
+      # all of the changes which must be realized later.
+      #
+      # path        - A String with the path at which to commit the file
+      # new_content - A String with the new contents
+      #
+      # Returns true (and updates @changes) if there was actually a change, false otherwise.
+      def commit_data(path, new_content)
+        ensure_branch_exists
+
+        old_content = nil
+        begin
+          contents = octokit.contents(repository, path: path, ref: branch)
+          old_content = Base64.decode64(contents.content)
+        rescue Octokit::NotFound
+          verbose("No old content found in #{repository.inspect} at #{path.inspect} in #{branch.inspect}")
+          # Fine, we will add below.
+        end
+
+        if new_content == old_content
+          verbose("Content of #{path} matches, no commit needed")
+          return false
+        else
+          verbose("Content of #{path} does not match. A commit is needed.")
+          verbose(Diffy::Diff.new(old_content, new_content))
+        end
+
+        @changes << Hash(
+          path: path,
+          mode: "100644",
+          type: "blob",
+          sha: octokit.create_blob(repository, new_content)
+        )
+
+        verbose("Batched update of #{path}")
+        true
+      end
+
+      # Finalize the GitHub commit by actually pushing any of the changes. This will not do anything if there
+      # are not any changes batched via the `commit_data` method.
+      #
+      # message - A String with a commit message, defaults to the overall configured commit message.
+      def finalize_commit(message = commit_message)
+        return unless @changes.any?
+
+        ensure_branch_exists
+        branch_ref = octokit.branch(repository, branch)
+        commit = octokit.git_commit(repository, branch_ref[:commit][:sha])
+        tree = commit["tree"]
+        new_tree = octokit.create_tree(repository, @changes, base_tree: tree["sha"])
+        new_commit = octokit.create_commit(repository, message, new_tree["sha"], commit["sha"])
+        octokit.update_ref(repository, "heads/#{branch}", new_commit["sha"])
+        verbose("Committed #{@changes.size} change(s) to GitHub")
+        find_or_create_pull_request
+      end
+
+      # Delete a file from the repository. Because of the way the GitHub API works, this will generate an
+      # immediate commit and push. It will NOT be batched for later application.
+      #
+      # path    - A String with the path at which to commit the file
+      # message - A String with a commit message, defaults to the overall configured commit message.
+      #
+      # Returns true if the file existed before and was deleted. Returns false if the file didn't exist anyway.
+      def delete_file(path, message = commit_message)
+        ensure_branch_exists
+        contents = octokit.contents(repository, path: path, ref: branch)
+        blob_sha = contents.sha
+        octokit.delete_contents(repository, path, message, blob_sha, branch: branch)
+        verbose("Deleted #{path}")
+        find_or_create_pull_request
+        true
+      rescue Octokit::NotFound
+        verbose("Deleted #{path} (already gone)")
+        false
+      end
+
+      private
+
+      # Private: Build an octokit object from the provided options.
+      #
+      # Returns an octokit object.
+      def octokit
+        @octokit ||= begin
+          token = options.fetch("github", {})["token"] || ENV["OCTOKIT_TOKEN"]
+          if token
+            Octokit::Client.new(access_token: token)
+          else
+            raise ArgumentError, "Access token must be provided in config file or OCTOKIT_TOKEN environment variable."
+          end
+        end
+      end
+
+      # Private: Get the default branch from the repository. Unless default_branch is specified in the options, then use
+      # that instead.
+      #
+      # Returns a String with the name of the default branch.
+      def default_branch
+        github_options["default_branch"] || octokit.repo(repository)[:default_branch]
+      end
+
+      # Private: Ensure branch exists. This will use octokit to create the branch on GitHub if the branch
+      # does not already exist.
+      def ensure_branch_exists
+        @ensure_branch_exists ||= begin
+          created = false
+          begin
+            if octokit.branch(repository, branch)
+              verbose("Branch #{branch} already exists in #{repository}.")
+              created = true
+            end
+          rescue Octokit::NotFound
+            # Fine, we'll create it
+          end
+
+          unless created
+            base_sha = octokit.branch(repository, default_branch)[:commit][:sha]
+            octokit.create_ref(repository, "heads/#{branch}", base_sha)
+            verbose("Created branch #{branch} based on #{default_branch} #{base_sha}.")
+          end
+
+          true
+        end
+      end
+
+      # Private: Find an existing pull request for the branch, and commit a new pull request if
+      # there was not an existing one open.
+      #
+      # Returns the pull request object that was created.
+      def find_or_create_pull_request
+        @find_or_create_pull_request ||= begin
+          prs = octokit.pull_requests(repository, head: "github:#{branch}", state: "open")
+          if prs && !prs.empty?
+            verbose("Found existing PR #{prs.first.html_url}")
+            prs.first
+          else
+            new_pr = octokit.create_pull_request(
+              repository,
+              default_branch,
+              branch,
+              pr_subject,
+              pr_body
+            )
+            verbose("Created a new PR #{new_pr.html_url}")
+            new_pr
+          end
+        end
+      end
+
+      # Simple methods not covered by unit tests explicitly.
+      # :nocov:
+
+      # Log a verbose message.
+      #
+      # message - A String with the message to print.
+      def verbose(message)
+        return unless @verbose
+        puts "*** #{Time.now}: #{message}"
+      end
+
+      def github_options
+        return {} unless options.is_a?(Hash)
+        options.fetch("github", {})
+      end
+
+      def repository
+        github_options.fetch("repository")
+      end
+
+      def branch
+        github_options.fetch("branch")
+      end
+
+      def commit_message
+        github_options.fetch("commit_message")
+      end
+
+      def pr_subject
+        github_options.fetch("pr_subject")
+      end
+
+      def pr_body
+        github_options.fetch("pr_body")
+      end
+      # :nocov:
+    end
+  end
+end