onceover 3.22.0 → 5.0.0

data/lib/onceover/beaker/spec_helper.rb

@@ -35,17 +35,17 @@ RSpec.configure do |c|
 
   #default option values
   defaults = {
-    :nodeset     => 'onceover-nodes',
+    :nodeset => 'onceover-nodes',
   }
   #read env vars
   env_vars = {
-    :color       => ENV['BEAKER_color'] || ENV.fetch('RS_COLOR', nil),
-    :nodeset     => ENV['BEAKER_set'] || ENV.fetch('RS_SET', nil),
+    :color => ENV['BEAKER_color'] || ENV.fetch('RS_COLOR', nil),
+    :nodeset => ENV['BEAKER_set'] || ENV.fetch('RS_SET', nil),
     :nodesetfile => ENV['BEAKER_setfile'] || ENV.fetch('RS_SETFILE', nil),
-    :provision   => ENV['BEAKER_provision'] || ENV.fetch('RS_PROVISION', nil),
-    :keyfile     => ENV['BEAKER_keyfile'] || ENV.fetch('RS_KEYFILE', nil),
-    :debug       => ENV['BEAKER_debug'] || ENV.fetch('RS_DEBUG', nil),
-    :destroy     => ENV['BEAKER_destroy'] || ENV.fetch('RS_DESTROY', nil),
+    :provision => ENV['BEAKER_provision'] || ENV.fetch('RS_PROVISION', nil),
+    :keyfile => ENV['BEAKER_keyfile'] || ENV.fetch('RS_KEYFILE', nil),
+    :debug => ENV['BEAKER_debug'] || ENV.fetch('RS_DEBUG', nil),
+    :destroy => ENV['BEAKER_destroy'] || ENV.fetch('RS_DESTROY', nil),
   }.compact
   #combine defaults and env_vars to determine overall options
   options = defaults.merge(env_vars)

data/lib/onceover/beaker.rb

@@ -11,9 +11,8 @@ class Onceover
       # se or an array
 
       if facts.is_a?(Array)
-        returnval = []
-        facts.each do |fact|
-          returnval << self.facts_to_vagrant_box(fact)
+        returnval = facts.map do |fact|
+          self.facts_to_vagrant_box(fact)
         end
         return returnval
       end
@@ -64,9 +63,8 @@ class Onceover
       warn "[DEPRECATION] #{__method__} is deprecated due to the removal of Beaker"
 
       if facts.is_a?(Array)
-        returnval = []
-        facts.each do |fact|
-          returnval << self.facts_to_platform(fact)
+        returnval = facts.map do |fact|
+          self.facts_to_platform(fact)
         end
         return returnval
       end
@@ -131,9 +129,9 @@ class Onceover
 
       # Do an r10k deploy
       r10k_deploy(host, {
-        :puppetfile => true,
-        :configfile => '/tmp/r10k.yaml',
-        })
+                    :puppetfile => true,
+                    :configfile => '/tmp/r10k.yaml',
+                  })
     end
 
     # This actually provisions a node and checks that puppet will be able to run and
@@ -153,7 +151,6 @@ class Onceover
       opts = {:check_idempotency => true}.merge(opts)
       opts = {:deploy_controlrepo => true}.merge(opts)
 
-
       raise "Hosts must be a single host object, not an array" if host.is_a?(Array)
       raise "Class must be a single Class [String], not an array" unless puppet_class.is_a?(String)
 
@@ -216,8 +213,8 @@ class Onceover
       # to refer to each key as either a string or an object
       current_opts.default_proc = proc do |h, k|
         case k
-          when String then sym = k.to_sym; h[sym] if h.key?(sym)
-          when Symbol then str = k.to_s; h[str] if h.key?(str)
+        when String then sym = k.to_sym; h[sym] if h.key?(sym)
+        when Symbol then str = k.to_s; h[str] if h.key?(str)
         end
       end
 

data/lib/onceover/cli/run.rb

@@ -44,6 +44,7 @@ This includes deploying using r10k and running all custom tests.
             optional nil, :format, 'Which RSpec formatter to use, valid options are: documentation, progress, FailureCollector, OnceoverFormatter. You also specify this multiple times', multiple: true, default: :defaults
             optional nil, :no_workarounds, 'Disables workarounds that have been added for convenience to get around common RSPec issues such as https://github.com/rodjek/rspec-puppet/issues/665'
             optional :ff, :fail_fast, 'Abort the run after the first failure'
+            optional nil, :auto_vendored, 'Attempt to resolve vendored puppet modules. Ex: puppetlabs/cron_core', default: false
 
             run do |opts, args, cmd|
               repo = Onceover::Controlrepo.new(opts)

data/lib/onceover/controlrepo.rb

@@ -119,7 +119,7 @@ class Onceover
       @spec_dir         = opts[:spec_dir]         || File.expand_path('./spec', @root)
       @facts_dir        = opts[:facts_dir]        || File.expand_path('factsets', @spec_dir)
       _facts_dirs       = [@facts_dir, File.expand_path('../../factsets', __dir__)]
-      _facts_files      = opts[:facts_files]      || _facts_dirs.map{|d| File.join(d, '*.json')}
+      _facts_files      = opts[:facts_files] || _facts_dirs.map{|d| File.join(d, '*.json')}
       @facts_files      = _facts_files.map{|_path| Dir[_path]}.flatten
 
       @nodeset_file     = opts[:nodeset_file]     || File.expand_path('./spec/acceptance/nodesets/onceover-nodes.yml', @root)
@@ -139,7 +139,6 @@ class Onceover
       end
     end
 
-
     def to_s
       require 'colored'
 
@@ -182,9 +181,8 @@ class Onceover
       end
 
       # Get all the classes from all of the manifests
-      classes = []
-      code_dirs.each do |dir|
-        classes << get_classes(dir)
+      classes = code_dirs.map do |dir|
+        get_classes(dir)
       end
       classes.flatten
     end
@@ -208,9 +206,8 @@ class Onceover
         raise "Filter param must be a hash" unless filter.is_a?(Hash)
 
         all_facts.keep_if do |hash|
-          matches = []
-          filter.each do |filter_fact,value|
-            matches << keypair_is_in_hash(hash,filter_fact,value)
+          matches = filter.map do |filter_fact,value|
+            keypair_is_in_hash(hash,filter_fact,value)
           end
           !matches.include? false
         end
@@ -231,10 +228,9 @@ class Onceover
       puppetfile.load!
 
       output_array = []
-      threads      = []
       error_array  = []
-      puppetfile.modules.each do |mod|
-        threads << Thread.new do
+      threads = puppetfile.modules.map do |mod|
+        Thread.new do
           begin
             row = []
             logger.debug "Loading data for #{mod.full_name}"
@@ -268,7 +264,7 @@ class Onceover
                         "PatchLevel_minor".green
                       else
                         "No".green
-                      end
+                     end
 
               row << mod.v3_module.endorsement
               superseded_by = mod.v3_module.superseded_by
@@ -290,8 +286,8 @@ class Onceover
             error_array << error
             logger.debug "Error loading module #{mod.full_name} - #{e.inspect}"
           end
-          end
         end
+      end
 
       threads.map(&:join)
 
@@ -313,13 +309,12 @@ class Onceover
       # TODO: Make sure we can deal with :latest
 
       # Create threading resources
-      threads = []
-      queue   = Queue.new
+      queue = Queue.new
       queue.push(puppetfile_string)
 
       puppetfile.modules.keep_if {|m| m.is_a?(R10K::Module::Forge)}
-      puppetfile.modules.each do |mod|
-        threads << Thread.new do
+      threads = puppetfile.modules.map do |mod|
+        Thread.new do
           logger.debug "Getting latest version of #{mod.full_name}"
           latest_version = mod.v3_module.current_release.version
 
@@ -360,14 +355,14 @@ class Onceover
             # Set it up as a symlink, because we are using local files in the Puppetfile
             symlinks << {
               'name' => mod.name,
-              'dir'  => mod.expected_version[:path]
+              'dir' => mod.expected_version[:path]
             }
           elsif mod.expected_version.is_a?(String)
             # Set it up as a normal forge module
             forge_modules << {
               'name' => mod.name,
               'repo' => mod.title,
-              'ref'  => mod.expected_version
+              'ref' => mod.expected_version
             }
           end
         elsif mod.is_a? R10K::Module::Git
@@ -377,7 +372,7 @@ class Onceover
               # I know I shouldn't be doing this, but trust me, there are no methods
               # anywhere that expose this value, I looked.
               'repo' => mod.instance_variable_get(:@remote),
-              'ref'  => mod.version
+              'ref' => mod.version
             }
         end
       end
@@ -392,7 +387,7 @@ class Onceover
         Dir["#{dir}/*"].each do |mod|
           symlinks << {
             'name' => File.basename(mod),
-            'dir'  => Pathname.new(File.expand_path(mod)).relative_path_from(Pathname.new(@root))#File.expand_path(mod)
+            'dir' => Pathname.new(File.expand_path(mod)).relative_path_from(Pathname.new(@root))#File.expand_path(mod)
           }
         end
       end
@@ -574,9 +569,9 @@ class Onceover
         # Add the resulting info to the hosts hash. This is what the
         # template will output
         hosts_hash[node_name] = {
-          :platform    => platform,
-          :boxname     => boxname,
-          :url         => url,
+          :platform => platform,
+          :boxname => boxname,
+          :url => url,
           :comment_out => comment_out
         }
       end

data/lib/onceover/deploy.rb

@@ -13,6 +13,10 @@ class Onceover
       skip_r10k_default = !(File.file?(repo.puppetfile))
       skip_r10k = opts[:skip_r10k] || skip_r10k_default
       force     = opts[:force] || false
+      # Only attempt to resolve vendored modules if configured to do so
+      auto_vendored = opts[:auto_vendored] || false
+
+      require 'onceover/vendored_modules' if auto_vendored
 
       if repo.tempdir == nil
         repo.tempdir = Dir.mktmpdir('r10k')
@@ -63,7 +67,11 @@ class Onceover
           # add to list of files copied to cache by onceover
           onceover_manifest << relative_source
 
-          if File.directory? source
+          if File.symlink?(source)
+            # Handle symlinks
+            link_target = File.readlink(source) # Get the target of the symlink
+            FileUtils.ln_s link_target, target, force: true # Create symlink at target
+          elsif File.directory? source
             Find.prune if excluded_dirs.include? source
             FileUtils.mkdir target
           else
@@ -90,7 +98,26 @@ class Onceover
         if /:control_branch/.match(puppetfile_contents)
           logger.debug "replacing :control_branch mentions in the Puppetfile with #{git_branch}"
           new_puppetfile_contents = puppetfile_contents.gsub(":control_branch", "'#{git_branch}'")
-          File.write("#{temp_controlrepo}/Puppetfile", new_puppetfile_contents)  
+          File.write("#{temp_controlrepo}/Puppetfile", new_puppetfile_contents)
+        end
+
+        if auto_vendored
+          tmp_puppetfile = File.join(temp_controlrepo, 'Puppetfile')
+          tmp_puppetfile_contents = File.read(tmp_puppetfile)
+          vm = Onceover::VendoredModules.new({ repo: repo })
+          puppetfile = R10K::ModuleLoader::Puppetfile.new(basedir: temp_controlrepo)
+          vm.puppetfile_missing_vendored(puppetfile)
+          unless vm.missing_vendored.empty?
+            missing_slugs = vm.missing_vendored.map do |missing_mod|
+              missing_mod.keys[0]
+            end
+            logger.debug "Adding #{missing_slugs} to #{tmp_puppetfile}"
+            modlines = vm.missing_vendored.map do |missing_mod|
+              mod_slug = missing_mod.keys[0]
+              "mod '#{mod_slug}',\n  git: '#{missing_mod[mod_slug][:git]}',\n  ref: '#{missing_mod[mod_slug][:ref]}'"
+            end.join("\n")
+            File.write(tmp_puppetfile, "#{tmp_puppetfile_contents}\n# Onceover Managed Vendored Modules\n#{modlines}")
+          end
         end
       end
 

data/lib/onceover/group.rb

@@ -28,10 +28,8 @@ class Onceover
         @members = []
       else
         # Turn it into a full list
-        member_objects = []
-
         # This should also handle lists that include groups
-        members.each { |member| member_objects << Onceover::TestConfig.find_list(member) }
+        member_objects = members.map { |member| Onceover::TestConfig.find_list(member) }
         member_objects.flatten!
 
         # Check that they are all the same type

data/lib/onceover/logger.rb

@@ -8,8 +8,8 @@ module Onceover::Logger
         'bright',
         :levels => {
           :debug => :cyan,
-          :info  => :green,
-          :warn  => :yellow,
+          :info => :green,
+          :warn => :yellow,
           :error => :red,
           :fatal => [:white, :on_red]
         }
@@ -18,7 +18,7 @@ module Onceover::Logger
       Logging.appenders.stdout(
         'stdout',
         :layout => Logging.layouts.pattern(
-          :pattern      => '%l\t -> %m\n',
+          :pattern => '%l\t -> %m\n',
           :color_scheme => 'bright'
         )
       )

data/lib/onceover/node.rb

@@ -4,7 +4,6 @@ class Onceover
   class Node
     @@all = []
 
-
     attr_accessor :name
     attr_accessor :beaker_node
     attr_accessor :fact_set
@@ -44,7 +43,6 @@ class Onceover
       end
 
       @@all << self
-
     end
 
     def self.find(node_name)

data/lib/onceover/rake_tasks.rb

@@ -1,10 +1,10 @@
 require 'onceover/controlrepo'
 require 'pathname'
+require 'voxpupuli/test/rake'
 
 @repo   = nil
 @config = nil
 
-
 desc 'Writes a `fixtures.yml` file based on the Puppetfile'
 task :generate_fixtures do
   repo = Onceover::Controlrepo.new
@@ -15,7 +15,6 @@ task :generate_fixtures do
   File.write(File.expand_path('./.fixtures.yml', repo.root), repo.fixtures)
 end
 
-
 desc "Modifies your `hiera.yaml` to point at the hieradata relative to its position."
 task :hiera_setup do
   repo = Onceover::Controlrepo.new
@@ -45,7 +44,6 @@ task :generate_onceover_yaml do
   puts ERB.new(onceover_yaml_template, nil, '-').result(binding)
 end
 
-
 task :generate_nodesets do
   warn "[DEPRECATION] #{__method__} is deprecated due to the removal of Beaker"
 
@@ -75,7 +73,7 @@ task :generate_nodesets do
       comment_out = false
       box_info = MultiJson.load(response.body)
       box_info['current_version']['providers'].each do |provider|
-        if  provider['name'] == 'virtualbox'
+        if provider['name'] == 'virtualbox'
           url = provider['original_url']
         end
       end
@@ -86,5 +84,13 @@ task :generate_nodesets do
     fixtures_template = File.read(File.expand_path('./nodeset.yaml.erb', template_dir))
     puts ERB.new(fixtures_template, nil, '-').result(binding)
   end
+end
+
+desc 'Create cache for vendored modules'
+task :generate_vendor_cache do
+  require 'onceover/controlrepo'
+  require 'onceover/vendored_modules'
 
+  repo = Onceover::Controlrepo.new(debug: true)
+  Onceover::VendoredModules.new({ repo: repo, cachedir: File.join(repo.spec_dir, 'vendored_modules'), force_update: true })
 end

data/lib/onceover/rspec/formatters.rb

@@ -1,5 +1,7 @@
 require 'rspec'
+require 'rspec-puppet'
 require 'pathname'
+require 'English'
 
 class OnceoverFormatter
   RSpec::Core::Formatters.register(
@@ -127,9 +129,9 @@ class OnceoverFormatter
 
         scanned_errors.map do |error_matches|
           {
-            text:   error_matches[0],
-            file:   calculate_relative_source(error_matches[1]),
-            line:   error_matches[2],
+            text: error_matches[0],
+            file: calculate_relative_source(error_matches[1]),
+            line: error_matches[2],
             column: error_matches[3],
           }
         end
@@ -223,7 +225,6 @@ class OnceoverFormatter
   def longest_group
     RSpec.configuration.world.example_groups.max { |a,b| a.description.length <=> b.description.length}.description.length
   end
-
 end
 
 class OnceoverFormatterParallel < OnceoverFormatter
@@ -280,7 +281,7 @@ class OnceoverFormatterParallel < OnceoverFormatter
       # Read all files and merge them
       errs.merge(YAML.load_file(file)) {|key, oldval, newval| [oldval, newval].flatten }
     end
-  
+
     # Delete files from the disk
     files.each { |f| File.delete(f) }
 

data/lib/onceover/runner.rb

@@ -92,8 +92,8 @@ class Onceover
           result = run_command(@command_prefix.strip.split, 'rake', 'parallel_spec')
         else
           require 'io/console'
-          logger.debug "Running #{@command_prefix}rake spec_standalone from #{@repo.tempdir}"
-          result = run_command(@command_prefix.strip.split, 'rake', 'spec_standalone')
+          logger.debug "Running #{@command_prefix}rake spec:standalone from #{@repo.tempdir}"
+          result = run_command(@command_prefix.strip.split, 'rake', 'spec:standalone')
         end
 
         # Reset env to previous state if we modified it
@@ -118,7 +118,7 @@ class Onceover
 
       Dir.chdir(@repo.tempdir) do
         #`bundle install --binstubs`
-        #`bin/rake spec_standalone`
+        #`bin/rake spec:standalone`
         logger.debug "Running #{@command_prefix}rake acceptance from #{@repo.tempdir}"
         result = run_command(@command_prefix.strip.split, 'rake', 'acceptance')
       end

data/lib/onceover/test.rb

@@ -12,9 +12,8 @@ class Onceover
     # it will then detect them and expand them out into their respective objects so that
     # we just end up with a list of nodes and classes
     def initialize(on_this, test_this, test_config)
-
       @default_test_config = {
-        'check_idempotency'       => true,
+        'check_idempotency' => true,
         'runs_before_idempotency' => 1
       }
 

data/lib/onceover/testconfig.rb

@@ -282,9 +282,9 @@ class Onceover
     def run_filters(tests)
       # All of this needs to be applied AFTER deduplication but BEFORE writing
       filters = {
-        'tags'    => @filter_tags,
+        'tags' => @filter_tags,
         'classes' => @filter_classes,
-        'nodes'   => @filter_nodes
+        'nodes' => @filter_nodes
       }
       filters.each do |method, filter_list|
         if filter_list

data/lib/onceover/vendored_modules.rb

@@ -0,0 +1,186 @@
+require 'puppet/version'
+require 'net/http'
+require 'uri'
+require 'multi_json'
+require 'r10k/module_loader/puppetfile'
+require 'onceover/logger'
+
+### operations
+#
+# 1. resolve all the component json files in the puppet-agent repo for vendored modules
+# 2. parse each json file and determine vendored modules repo + ref
+#
+###
+
+## Example
+#
+# vm = Onceover::VendoredModules.new
+# puts vm.vendored_references
+# puppetfile = R10K::ModuleLoader::Puppetfile.new(basedir: '.')
+# vm.puppetfile_missing_vendored(puppetfile)
+# puts vm.missing_vendored.inspect
+
+class Onceover
+  class VendoredModules
+    attr_reader :vendored_references, :missing_vendored
+
+    def initialize(opts = {})
+      @repo = opts[:repo] || Onceover::Controlrepo.new
+      @cachedir = opts[:cachedir] || File.join(@repo.tempdir, 'vendored_modules')
+      @puppet_version = Gem::Version.new(Puppet.version)
+      @puppet_major_version = Gem::Version.new(@puppet_version.segments[0])
+      @force_update = opts[:force_update] || false
+
+      @missing_vendored = []
+
+      # This only applies to puppet >= 6 so bail early
+      raise 'Auto resolving vendored modules only applies to puppet versions >= 6' unless @puppet_major_version >= Gem::Version.new('6')
+
+      # Create cachedir
+      unless File.directory?(@cachedir)
+        logger.debug "Creating #{@cachedir}"
+        FileUtils.mkdir_p(@cachedir)
+      end
+
+      # Location of user provided caches:
+      #   control-repo/spec/vendored_modules/<component>-puppet_agent-<agent version>.json
+      @manual_vendored_dir = File.join(@repo.spec_dir, 'vendored_modules')
+
+      # Get the entire file tree of the puppetlabs/puppet-agent repository
+      # https://docs.github.com/en/rest/git/trees?apiVersion=2022-11-28#get-a-tree
+      puppet_agent_tree = query_or_cache(
+        "https://api.github.com/repos/OpenVoxProject/openvox-agent/git/trees/#{@puppet_version}",
+        { recursive: true },
+        component_cache('repo_tree'),
+      )
+      # Get only the module-puppetlabs-<something>_core.json component files
+      vendored_components =  puppet_agent_tree['tree'].select { |file| %r{configs/components/module-puppetlabs-\w+\.json}.match(file['path']) }
+      # Get the contents of each component file
+      # https://docs.github.com/en/rest/git/blobs?apiVersion=2022-11-28#get-a-blob
+      @vendored_references = vendored_components.map do |component|
+        mod_slug = component['path'].match(/.*(puppetlabs-\w+).json$/)[1]
+        mod_name = mod_slug.match(/puppetlabs-(\w+)/)[1]
+        query_or_cache(
+          component['url'],
+          nil,
+          component_cache(mod_name),
+        )
+      end
+    end
+
+    def component_cache(component)
+      # Ideally we want a cache for the version of the puppet agent used in tests
+      desired_name = "#{component}-puppet_agent-#{@puppet_version}.json"
+      # By default look for any caches created during previous runs
+      cache_file = File.join(@cachedir, desired_name)
+
+      # If the user provides their own cache
+      if !@force_update && File.directory?(@manual_vendored_dir)
+                # Check for any '<component>-puppet_agent-<puppet version>.json' files
+                dg = Dir.glob(File.join(@manual_vendored_dir, "#{component}-puppet_agent*"))
+                # Check if there are multiple versions of the component cache
+                if dg.size > 1
+                  # If there is the same version supplied as whats being tested against use that
+                  if dg.any? { |s| s[desired_name] }
+                    cache_file = File.join(@manual_vendored_dir, desired_name)
+                  # If there are any with the same major version, use the latest supplied
+                  elsif dg.any? { |s| s["#{component}-puppet_agent-#{@puppet_major_version}"] }
+                    maj_match = dg.select { |f| /#{component}-puppet_agent-#{@puppet_major_version}.\d+\.\d+\.json/.match(f) }
+                    maj_match.each do |f|
+                      next unless (version_from_file(cache_file) == version_from_file(desired_name)) || (version_from_file(f) >= version_from_file(cache_file))
+
+                      # if the current cache version matches the desired version, use the first matching major version in user cache
+                      # if there are multiple major version matches in user cache, use the latest
+                      cache_file = f
+                    end
+                  # Otherwise just use the latest supplied
+                  else
+                    dg.each { |f| cache_file = f if version_from_file(f) >= version_from_file(cache_file) }
+                  end
+                # If there is only one use that
+                elsif dg.size == 1
+                  cache_file = dg[0]
+                end
+      end
+
+      # Warn the user if cached version does not match whats being used to test
+      cache_version = version_from_file(cache_file)
+      logger.warn "Cache for #{component} is for puppet_agent #{cache_version}, while you are testing against puppet_agent #{@puppet_version}. Consider updating your cache to ensure consistent behavior in your tests" if cache_version != @puppet_version
+
+      cache_file
+    end
+
+    def version_from_file(cache_file)
+      version_regex = /.*-puppet_agent-(\d+\.\d+\.\d+)\.json/
+      Gem::Version.new(version_regex.match(cache_file)[1])
+    end
+
+    # Currently expects to be passed a R10K::Puppetfile object.
+    # ex: R10K::ModuleLoader::Puppetfile.new(basedir: '.')
+    def puppetfile_missing_vendored(puppetfile)
+      puppetfile.load
+      @vendored_references.each do |mod|
+        # Extract name and slug from url
+        mod_slug = mod['url'].match(/.*(puppetlabs-\w+)\.git/)[1]
+        mod_name = mod_slug.match(/^puppetlabs-(\w+)$/)[1]
+        # Array of modules whos names match
+        existing = puppetfile.modules.select { |e_mod| e_mod.name == mod_name }
+        if existing.empty?
+          # Change url to https instead of ssh to allow anonymous git clones
+          # so that users do not need to have an ssh keypair associated with a Github account
+          url = mod['url'].gsub('git@github.com:', 'https://github.com/')
+          @missing_vendored << { mod_slug => { git: url, ref: mod['ref'] } }
+          logger.debug "#{mod_name} found to be missing in Puppetfile"
+        else
+          logger.debug "#{mod_name} found in Puppetfile. Using the specified version"
+        end
+      end
+    end
+
+    # Return json from a query whom caches, or from the cache to avoid spamming github
+    def query_or_cache(url, params, filepath)
+      if (File.exist? filepath) && (@force_update == false)
+        logger.debug "Using cache: #{filepath}"
+        json = read_json_dump(filepath)
+      else
+        logger.debug "Making GET request to: #{url}"
+        json = github_get(url, params)
+        logger.debug "Caching response to: #{filepath}"
+        write_json_dump(filepath, json)
+      end
+      json
+    end
+
+    # Given a github url and optional query parameters, return the parsed json body
+    def github_get(url, params)
+      uri = URI.parse(url)
+      uri.query = URI.encode_www_form(params) if params
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = (uri.scheme == 'https')
+      request = Net::HTTP::Get.new(uri.request_uri)
+      request['Accept'] = 'application/vnd.github.raw+json'
+      request['X-GitHub-Api-Version'] = '2022-11-28'
+      response = http.request(request)
+
+      case response
+      when Net::HTTPOK # 200
+        MultiJson.load(response.body)
+      else
+        # Expose the ratelimit response headers
+        # https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28#checking-the-status-of-your-rate-limit
+        ratelimit_headers = response.to_hash.select { |k, _v| k =~ /x-ratelimit.*/ }
+        raise "#{response.code} #{response.message} #{ratelimit_headers}"
+      end
+    end
+
+    # Returns parsed json of file
+    def read_json_dump(filepath)
+      MultiJson.load(File.read(filepath))
+    end
+
+    # Writes json to a file
+    def write_json_dump(filepath, json_data)
+      File.write(filepath, MultiJson.dump(json_data))
+    end
+  end
+end