nli_pipeline 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5803338257515506b32dc63259908d23d660e701
-  data.tar.gz: b5a12fbcf9b2e604464bd3c99c6b9f3101e34539
+  metadata.gz: 0b403349bdb310782fa8b2dae71361b5ee0874e1
+  data.tar.gz: 837d3ca13607c39c8aa1d451abb80ab9a1066196
 SHA512:
-  metadata.gz: b547fd4c73d957517ca8b8cc0b30f189d042188e025d37f9ad42238bdbeae976efede708a6229f9b64a39c4fe353d9e611cdd983393830f45025ef0525b4d77a
-  data.tar.gz: 6ec520a89825c94223381bff4169f807a24d29988c7207f27d0fdbef385c5c581542a751f2ccd49b2888da274f4614e5810e13301e5ebaa544466ec1d6160baf
+  metadata.gz: de55b1cddd338648732b811fc616a2e4d9484c1f585c3413e0532e1726d3b7b3748c7f8f78bece0b81120dcfcb665b37438082911baf788652ba583734d32eb1
+  data.tar.gz: f5cf0e8b41936beb85fbd4967ef45d8d56ab2c4edcc4fb474fe322759989b880e0a2a401c11c36199948383cbe225357c82c57cf0991ed4435cdaa67e3435aec

data/bin/setup_pipeline

@@ -1,4 +1,4 @@
 #!/usr/bin/env ruby
 require 'nli_pipeline'
 
-SetupPipeline.new.main()
+NliPipeline::SetupPipeline.new.main

data/lib/nli_pipeline.rb

@@ -1,11 +1,8 @@
+# import all ruby files in nli_pipeline
+Dir["#{__dir__}/nli_pipeline/**/*.rb"].each do |file|
+  require file
+end
+
 # used to defined constants and include requirements for /bin/setup_pipeline
 module NliPipeline
-
-  # constant use in nli_pipeline.gemspec and bin/setup_pipeline
-  # to determine current version of gem
-  # should also match tag on bitbucket
-  VERSION = "0.1.2"
-
-  require_relative 'nli_pipeline/file_manager'
-  require_relative 'nli_pipeline/setup_pipeline'
 end

data/lib/nli_pipeline/abstract_util/init_attrs.rb

@@ -0,0 +1,110 @@
+module NliPipeline
+  # TODO: on v1.0.0 release, it may be possible to replace this with open struct
+  # if path is no longer a required argument
+
+  # Abstract Utilities for handling class instantiation
+  # similar to open struct, but enforces some arguments
+  module AbstractUtil
+    # override include to automatically extend classmethods
+    # i.e. functions in the module ClassMethods becom class methods
+    # when the AbstractUtil is included
+    def self.included(parent)
+      parent.extend(ClassMethods)
+    end
+
+    # @raise [ArgumentError] unless all values in kwargs are truthy
+    # @return [Boolean]
+    def raise_unless_all(**kwargs)
+      message = "All #{kwargs} must be truthy to use"\
+        " #{self.class}.#{caller_locations(1, 1)[0].label}"
+      raise ArgumentError, message unless kwargs.values.all?
+      true
+    end
+
+    # keys become instance variable names
+    # value is set using corresponding value in kwargs
+    # all instance variables have getters, but not setters
+    # add attributes to class
+    def add_attrs(**kwargs)
+      kwargs.each do |k, v|
+        instance_variable_set("@#{k}", v)
+        self.class.class_eval { attr_reader k.to_sym }
+      end
+      puts(to_s.yellow) if @debug
+    end
+
+    # support method for init_with_attrs to warn user about unused kwargs
+    def drop_forbidden_args_message(**kwargs)
+      forbidden_args = self.class.get_forbidden_kwargs(**kwargs)
+      show_warning = !forbidden_args.empty? && kwargs[:debug]
+      puts("Warning, dropping args #{forbidden_args} to #{self.class}".red) if show_warning
+    end
+
+    # similar to openstruct initialize
+    # assigns all keyword arguments as instance variables
+    # but uses attr_reader, not attr_accessor
+    def init_with_attrs(**kwargs)
+      sanitized_args = self.class.get_allowed_kwargs(**kwargs)
+      drop_forbidden_args_message(**kwargs)
+      unless self.class.required_args?(**kwargs)
+        # raise SystemWrapper::ConfigError.new()
+        msg = "#{self.class}.#{caller_locations(1, 1)[0].label}"\
+          " requires arguments: #{self.class.required_args}"
+        raise ArgumentError, msg
+      end
+      # merge passed args with default values for args
+      merged_args = self.class.supported_args.merge(sanitized_args)
+      add_attrs(**merged_args)
+    end
+
+    # override default to_s for nice formatting of instance vars
+    def to_s
+      format_instance_vars = lambda do |x, y|
+        x + format("\n%<key>-30s: %<value>s", key: y, value: instance_variable_get(y.to_sym))
+      end
+      vars = instance_variables.reduce('', &format_instance_vars)
+      "\n#{self.class} config#{vars}\n"
+    end
+
+    # methods to be loaded as static/class methods in class that extends AbstractUtil
+    module ClassMethods
+      # :nocov:
+      # use method_missing to force classes that use AbstractUtil to implement
+      #   supported_args and required_args.
+      # @param method [String]
+      # @param args   [Array]
+      def method_missing(method, *args)
+        # cast methods to array of symbols, compare to current missing method
+        if %i[supported_args required_args].include?(method.to_sym)
+          message = "Method #{method} must be implemented in #{self.class} "\
+            'in order to use AbstractUtil'
+          raise ArgumentError, message
+        end
+        super
+      end
+      # can't test call to super, so wrap in nocov tag
+      # :nocov:
+
+      # support for method_missing
+      # https://robots.thoughtbot.com/always-define-respond-to-missing-when-overriding
+      def respond_to_missing?(method, *args)
+        %i[supported_args required_args].include?(method.to_sym) || super
+      end
+
+      # @return [Hash] args that can be converted into instance vars for class
+      def get_allowed_kwargs(**kwargs)
+        kwargs.select { |k, _v| supported_args.key?(k) }
+      end
+
+      # @return [Hash] args that can't be converted into instance vars for class
+      def get_forbidden_kwargs(**kwargs)
+        kwargs.reject { |k, _v| supported_args.key?(k) }
+      end
+
+      # @return [Boolean] have all required args been passed?
+      def required_args?(**kwargs)
+        required_args.all? { |key| kwargs.key?(key) }
+      end
+    end
+  end
+end

data/lib/nli_pipeline/docker_manager.rb

@@ -0,0 +1,243 @@
+require 'colorize'
+require 'nli_pipeline/system_wrapper/call_wrapper'
+require 'nli_pipeline/abstract_util/init_attrs'
+
+module NliPipeline
+  # handle docker build, save and run for nli-pipelines
+  # this DOES NOT handle docker login
+  # if you want to push to dockerhub or pull private images
+  # you'll need to log in in advance
+  class DockerManager < SystemWrapper::CallWrapper
+    include AbstractUtil
+
+    # static methods required by NliPipeline::AbstractUtil::init_attrs
+    # @see NliPipeline::AbstractUtil#init_with_attrs
+    # @see NliPipeline::AbstractUtil#get_allowed_args
+    # @return [Hash]
+    def self.supported_args
+      {
+        path: '', debug: false, fail_on_error: false,
+        image_name: false, upstream_image_name: false, tar_name: false,
+        git_branch: false, proxy: false, container_name: false, commit: true,
+        mount: false, bash_commands: [], ports: []
+      }
+    end
+
+    # @see NliPipeline::AbstractUtil#init_with_attrs
+    # @see NliPipeline::AbstractUtil::ClassMethods#required_args?
+    # @return [Array[Symbol]]
+    def self.required_args
+      [:path]
+    end
+
+    # creates new DockerManager and assigns each keywoard argument as instance var with attr_reader
+    # uses image_name.tar as tar_name unless it's explicitly set
+    def initialize(**kwargs)
+      # add nlireland/ prefix to imagename if no upstream image name is set
+      can_guess_upstream = kwargs[:image_name] && !kwargs[:upstream_image_name]
+      kwargs[:upstream_image_name] = "nlireland/#{kwargs[:image_name]}" if can_guess_upstream
+
+      # set tar name as image name if image name is set but tar name isn't
+      can_make_tar = kwargs[:image_name] && !kwargs[:tar_name]
+      kwargs[:tar_name] = "#{kwargs[:image_name]}.tar" if can_make_tar
+
+      # set container name manually, should not be able to change it
+      kwargs[:container_name] = 'pipeline-container'
+
+      # set git branch if it's not defined
+      # pass debug flags down to GitManager (will show warning about extra args though)
+      kwargs[:git_branch] = GitManager.new(**kwargs).branch unless kwargs[:git_branch]
+
+      init_with_attrs(**kwargs)
+    end
+
+    # check if last commit contained build command
+    # @return Boolean
+    def build_commit?
+      # neeed \\ since command will be passed to the shell as \[.*\]
+      # escape chars all the way down
+      custom_error_message = "[docker build] or variant not in last commit.\nFailing Build."
+      custom_message = 'can build docker'
+      # will match any combination of 'docker' and 'build' inside square brackets
+      # TODO: refator to use GitManager and stub git log -1 call
+      last_commit_message = NliPipeline::GitManager.new.last_commit_message
+      command = "echo '#{last_commit_message}' | grep -o '\\[.*\\]' "\
+        "| grep -i 'docker' | grep -i 'build'"
+
+      pretty_print(custom_message: custom_message) do
+        call_system(command, custom_error_message: custom_error_message)
+      end
+    end
+
+    # @return [String] build arguments
+    def build_args
+      args = []
+      args.push("proxy=#{@proxy}") if @proxy
+      args.push("commit=#{NliPipeline::GitManager.new.last_commit_url}") if @commit
+      args.reduce('', &->(x, y) { x + " --build-arg #{y}" })
+    end
+
+    # build docker image
+    # @return [Boolean] success/failure
+    def build
+      can_build = build_commit?
+      custom_message = 'built docker image'
+
+      # if we can't build, fail early
+      unless can_build
+        # if --fail-on-error is passed, this will throw and exception
+        # and "Skipping build" will not be output
+        ret = pretty_print(custom_message: custom_message) { false }
+        puts('Skipping build'.yellow)
+        return ret
+      end
+
+      command = 'docker build'
+      # command += " --build-arg proxy=#{@proxy}" if @proxy
+      command += build_args
+      command += " -t #{@image_name}" if @image_name
+      pretty_print(custom_message: custom_message) { call_system("#{command} #{@path}") }
+    end
+
+    # save docker image as tarfile
+    # @return [Boolean] success/failure
+    def save
+      # throw an error is image_name is not set
+      raise_unless_all(image_name: @image_name)
+      save_command = "docker save #{@image_name} -o #{@path}/#{@tar_name}"
+      puts 'saving docker image'.yellow
+      pretty_print(custom_message: 'saved docker image') { call_system(save_command) }
+    end
+
+    # save docker image as tarfile
+    # @return [Boolean] success/failure
+    def build_and_save
+      built = build
+      # if the build fails quit
+      return false unless built
+      save
+    end
+
+    # load saved docker image
+    # @return [Boolean] success/failure
+    def load
+      load_command = "docker load -i #{@path}/#{@tar_name}"
+      pretty_print(custom_message: "loading #{@tar_name}") { call_system(load_command) }
+    end
+
+    # def tar_exist?
+    #   File.exist? "#{@path}/#{@tar_name}"
+    # end
+
+    # if the last commit was a build commit
+    # try to load and run the image
+    # otherwise pull and run the upstream image
+    #
+    # in prepare_run_func
+    # always:
+    #   1. run in detached mode with stdin open (-id)
+    #   2. pass CI=true to as a build argument
+    #   3. set the container name (pipeline-container)
+    #
+    # @raise [SystemWrapper::ConfigError] if the local image failed to be loaded /run,
+    #   or the upstream image failed to run
+    # @return [Boolean] success/failure
+    def run
+      run_func = prepare_run_func
+      # if the last commit had a build instruction, load the built image
+      if build_commit? && @image_name
+        load
+        run_func.call(@image_name)
+      # otherwise try to pull the upstream image from dockerhub
+      elsif @upstream_image_name
+        run_func.call(@upstream_image_name)
+      # if neither image nor upstream image were set, throw config error
+      # can't use raise_unless_all since some, but not all, must be set
+      else
+        error_message = 'you must set image_name or upstream_image_name to run a docker image'
+        config_error = "Config Error: #{error_message}"
+        config = {
+          image_name: @image_name, upstream_image_name: @upstream_image_name,
+          build_commit: build_commit?
+        }
+        raise SystemWrapper::ConfigError.new(config: config, msg: config_error)
+      end
+    end
+
+    # create function to bind args to on docker run
+    # @return [Boolean] success/failure
+    def prepare_run_func
+      base_command = 'docker run -id'
+      base_command += " -v #{@path}:#{@mount}" if @mount
+      base_command = @ports.reduce(base_command, &->(x, y) { x + " -p #{y}:#{y}" })
+      base_command += " -e CI=TRUE --name #{@container_name}"
+      proc do |name|
+        pretty_print(custom_message: "running #{name}") { call_system("#{base_command} #{name}") }
+      end
+    end
+
+    # send bash command to running docker container
+    # avoid clash with exec!
+    # @return [Boolean] success/failure
+    def docker_exec
+      @bash_commands.each do |cmd|
+        pretty_print { call_system("docker exec #{@container_name} bash -c '#{cmd}'") }
+      end
+    end
+
+    # relies on tag to ensure image_name and upstream_image_name are set
+    # setup_pipeline sets fail-on-error to true for all deploys
+    # so build_commit? will raise an exception and stop the build
+    # if the last commit was not a build commit
+    # @return [Boolean] success/failure
+    def deploy_master
+      build_commit?
+      master_tags = ['latest', "latest-#{Time.now.strftime('%Y-%m-%d')}"]
+      master_tags.map do |t|
+        tag(t)
+        message = "pushing tag: #{t} to #{@upstream_image_name}"
+        push_command = "docker push #{@upstream_image_name}:#{t}"
+        # deploy each tag individually on the off change
+        # that another tag for the image that wasn't created by setup_pipeline
+        # exists and should not be pushed
+        pretty_print(custom_message: message) { call_system(push_command) }
+      end
+    end
+
+    # setup_pipeline sets fail-on-error to true for all deploys
+    # so build_commit? will raise an exception and stop the build
+    # if the last commit was not a build commit
+    # @return [Boolean] success/failure
+    def deploy_branch
+      build_commit?
+      # if gitbranch is not set, raise an error
+      # todo: this may be unnecessary now with git branch default
+      raise_unless_all(git_branch: @git_branch)
+      message = "pushing tag: #{@git_branch} to #{@upstream_image_name}"
+      tag(@git_branch)
+      push_command = "docker push #{@upstream_image_name}:#{@git_branch}"
+      pretty_print(custom_message: message) { call_system(push_command) }
+    end
+
+    private
+
+    # @param  tag [String] what to tag the docker image as
+    # @return [Boolean] success/failure
+    def tag(tag)
+      # docker tag requires a local and upstream image
+      # raise an error if either of these are missing
+      raise_unless_all(image_name: @image_name, upstream_image_name: @upstream_image_name)
+      imgs_and_tag = "#{@image_name} #{@upstream_image_name}:#{tag}"
+      print_arg = "tagging #{imgs_and_tag}"
+      call_system_arg = "docker tag #{imgs_and_tag}"
+      # ret = pretty_print(custom_message: print_arg) { call_system(call_system_arg) }
+      pretty_print(custom_message: print_arg) { call_system(call_system_arg) }
+
+      # # raise an error if fail on error is set
+      # error_messsage = "Failed to tag #{@upstream_image_name}:#{tag}"
+      # err = NliPipeline::SystemWrapper::CallWrapperError.new(error_messsage)
+      # raise err if !ret && @fail_on_error
+      # ret
+    end
+  end
+end

data/lib/nli_pipeline/file_manager.rb

@@ -1,144 +1,177 @@
-require 'rubygems'
 require 'colorize'
+require 'nli_pipeline/system_wrapper/call_wrapper'
+require 'nli_pipeline/abstract_util/init_attrs'
 
-# Manage example files
-#
-# EXAMPLE:
-#   >> FileManager.new.copy_example_files()
-#   => Moving 1 .EXAMPLE files
-class FileManager
-
-  # String: must be valid file path
-  attr_reader :path
-  # String: file extension default: .EXAMPLE
-  attr_accessor :extension
-  # Bool: show all system commands run by ruby
-  attr_accessor :debug
-
-  # Arguments:
-  #   path: (string: path to directory)
-  #   extension: (string)
-  #   debug: (boolean)
-  #   no_output: (boolean: used for suppressing output during tests)
-  def initialize(path, extension: ".EXAMPLE", debug: false, no_output: false)
-    puts("config = {path: '#{path}', extension: '#{extension}', debug: #{debug}, no_output: #{no_output}}".yellow()) if debug 
-    @path = path
-    @extension = extension
-    @debug = debug
-    @no_output = no_output
-    @log_dir = "#{path}/.pipeline_logs"
-    @created_log_name = "created_files_"
-
-    unless File.directory?(@log_dir)
-      Dir.mkdir(@log_dir)
-    end 
-  end
+module NliPipeline
+  # TODO: enforce that command must be either cp or mv
 
-  # get backup dates in order from newest to oldest
-  def get_all_backups()
-    backup_files = Dir.glob("#{@path}/**/*.backup*")
-    backup_numbers = backup_files.map {|x| x.split(".backup").last}
-    return backup_numbers.uniq.sort
-  end
+  # Manage example files
+  #
+  # EXAMPLE:
+  #   >> NliPipeline::FileManager.new.copy_example_files()
+  #   => Moving 1 .EXAMPLE files
+  class FileManager < SystemWrapper::CallWrapper
+    include AbstractUtil
+
+    # static methods required by NliPipeline::AbstractUtil::init_attrs
+    # @see NliPipeline::AbstractUtil#init_with_attrs
+    # @see NliPipeline::AbstractUtil#get_allowed_args
+    # @return [Hash]
+    def self.supported_args
+      {
+        path: '', extension: '.EXAMPLE', debug: false,
+        fail_on_error: false, log_dir: '.pipeline_logs',
+        created_get_log_name: 'created_files_', backup_path: ''
+      }
+    end
 
-  # find all example files under @path
-  def get_all_example_files()
-    example_file_path = "#{@path}/**/*#{extension}"
-    example_files = Dir.glob(example_file_path)
-    if example_files.empty?
-      raise ArgumentError.new("No #{extension} Files found at #{example_file_path}")
-    else
-      puts("Moving #{example_files.count} #{extension} files".green()) unless @no_output
+    # @see NliPipeline::AbstractUtil#init_with_attrs
+    # @see NliPipeline::AbstractUtil::ClassMethods#required_args?
+    # @return [Array[Symbol]]
+    def self.required_args
+      [:path]
     end
-    return example_files
-  end
 
-  # read latest log in .pipeline_logs 
-  # @return [String] files created by last setup_pipeline
-  def get_last_created_files()
-    logs = Dir.glob("#{@log_dir}/*")
-    if logs.empty?
-      raise ArgumentError.new("No logs found in #{@log_dir}")
+    # automatically set backup path
+    # init_with_attrs handles the rest
+    # @param kwargs [Hash] all keyword (only supported_args) will be added as vars
+    def initialize(**kwargs)
+      # backup_path should not be configurable
+      # set it directly regardless of whether it's passed in
+      kwargs[:backup_path] = "#{kwargs[:path]}/**/*.backup*"
+      init_with_attrs(**kwargs)
     end
-    log_numbers = logs.map {|x| x.split("#{@created_log_name}").last}
-    latest_log_number = log_numbers.uniq.sort.first
-    return File.readlines("#{@log_dir}/#{@created_log_name}#{latest_log_number}")
-  end
 
-  # Arguments:
-  #   command: (string: system command)
-  def copy_example_files(command="cp")
-    time_stamp = Time.now.strftime("%Y%m%d%H%M")
-    puts("Setting up pipeline in #{@path}".yellow()) unless @no_output
-    example_files = get_all_example_files
-    files_to_backup = example_files.map {|f| f.gsub(@extension, "")}.select {|file| File.exists?(file) }
-    # raise ArgumentError.new("#{files_to_backup.to_s} #{example_files.to_s}")
+    # @return [Array[String]] all example files under @path
+    def all_example_files
+      example_file_path = "#{@path}/**/*#{@extension}"
+      example_files = Dir.glob(example_file_path)
+      if example_files.empty?
+        raise ArgumentError, "No #{@extension} Files found at #{example_file_path}"
+      end
+      puts("Moving #{example_files.count} #{@extension} files".green)
+      example_files
+    end
 
-    if !files_to_backup.empty?
-      # preset backup time so all backups fomr same batch have the same timestamp
-      puts("Backing up #{files_to_backup.count} files".green()) unless @no_output
-      files_to_backup.each do |file_to_backup|
-        full_command = "#{command} #{file_to_backup} #{file_to_backup}.backup#{time_stamp}"
-        puts("\t#{full_command}") if @debug
-        system(full_command)
+    # read latest log in .pipeline_logs
+    # @return [String] files created by last setup_pipeline
+    def last_created_files
+      logs = Dir.glob("#{@path}/#{@log_dir}/*")
+      if logs.empty?
+        puts("No logs found in #{@path}/#{@log_dir}".red)
+        return false
       end
+      log_numbers = logs.map { |x| x.split(@created_get_log_name.to_s).last }
+      latest_log_number = log_numbers.uniq.min
+      File.readlines(get_log_name(latest_log_number))
     end
 
-    example_files.each do |example_file|
-      target_file = example_file.gsub(@extension, "")
-      # convert the example file to real config
-      full_command = "#{command} #{example_file} #{target_file}"
-      puts("\t#{full_command}") if @debug
-      system(full_command)
+    # copy all files ending in @extension under @path.
+    # if a non-exmaple version of the file exists, back it up.
+    # @param command       [String]        command to run (cp/mv)
+    def copy_example_files(command = 'cp')
+      time_stamp = Time.now.strftime('%Y%m%d%H%M')
+      puts("Setting up pipeline in #{@path}".yellow)
+      example_files = all_example_files
+
+      backup_non_example_files(example_files, time_stamp)
 
-      if File.exists?(target_file)
-        add_to_log(target_file, time_stamp)
-        puts("\t add #{target_file} to log #{get_log_name(time_stamp)}") if @debug
+      example_files.each do |example_file|
+        target_file = example_file.gsub(@extension, '')
+        call_system("#{command} #{example_file} #{target_file}")
+
+        if File.exist?(target_file)
+          add_to_log(target_file, time_stamp)
+          puts("\t add #{target_file} to log #{get_log_name(time_stamp)}") if @debug
+        end
       end
     end
 
-  end
+    # get backup dates in order from newest to oldest
+    def all_backups
+      backup_files = Dir.glob(@backup_path)
+      backup_numbers = backup_files.map { |x| x.split('.backup').last }
+      backup_numbers.uniq.sort
+    end
 
-  # Arguments:
-  #   command: (string: system command)
-  #
-  # to remove backup, use mv
-  # to keep backup, use cp
-  def load_from_backup(command="cp")
-    puts("loading backup in #{@path}".yellow())
-    backups = get_all_backups()
-    if backups.empty?
-      # TODO: better exception class here?
-      raise "\n No backups found \n"
+    # for each example file,
+    # if a non-example version of that file exists,
+    # back it up.
+    #
+    # @param example_files [Array[string]] list of example files
+    # @param time_stamp    [String]        time of backup
+    # @param command       [String]        command to run (cp/mv)
+    def backup_non_example_files(example_files, time_stamp, command = 'cp')
+      possible_files = example_files.map { |f| f.gsub(@extension, '') }
+      files_to_backup = possible_files.select { |file| File.exist?(file) }
+      # raise ArgumentError.new("#{files_to_backup.to_s} #{example_files.to_s}")
+      return false if files_to_backup.empty?
+
+      # preset backup time so all backups fomr same batch have the same timestamp
+      puts("Backing up #{files_to_backup.count} files".green)
+      files_to_backup.each do |file_to_backup|
+        call_system("#{command} #{file_to_backup} #{file_to_backup}.backup#{time_stamp}")
+      end
     end
 
-    puts "which backup would you like to load?\n#{backups}".yellow()
-    latest_backup_date = STDIN.gets().chomp()
+    # @param command [String] system command.
+    #   use mv to remove backup,
+    #   use cp to keep backup.
+    # @return Boolean
+    def load_from_backup(command = 'cp')
+      puts("loading backup in #{@path}".yellow)
+      backups = all_backups
+      if backups.empty?
+        puts("No backups found in #{@backup_path}".red)
+        return false
+      end
+
+      backup_path, backup_date = handle_backup_user_input(backups)
+      files_to_load = Dir.glob(backup_path)
+      puts "loading #{files_to_load.count} files from #{backup_path}".green
 
-    while !backups.include?(latest_backup_date)
-      puts "#{latest_backup_date} is not in backups".red()
-      puts "please choose from: #{backups}".yellow()
-      latest_backup_date = STDIN.gets().chomp()
+      files_to_load.each do |backup_file|
+        target_file = backup_file.gsub("\.backup#{backup_date}", '')
+        call_system("#{command} #{backup_file} #{target_file}")
+      end
+    end
+
+    # keeps asking the use to choose a backup
+    # until they choose a valid one
+    # @param backups [Array[String]]
+    # @return [Array[String]]
+    def handle_backup_user_input(backups)
+      puts "which backup would you like to load?\n#{backups}".yellow
+      backup_date = STDIN.gets.chomp
+
+      until backups.include?(backup_date)
+        puts "#{backup_date} is not in backups".red
+        puts "please choose from: #{backups}".yellow
+        backup_date = STDIN.gets.chomp
+      end
+      ["#{@path}/**/*.backup#{backup_date}", backup_date]
     end
 
-    files_to_load = Dir.glob("#{@path}/**/*.backup#{latest_backup_date}")
-    puts "loading #{files_to_load.count} files from #{@path}/**/*.backup#{latest_backup_date}".green()
+    private
 
-    files_to_load.each do |backup_file|
-      target_file = backup_file.gsub("\.backup#{latest_backup_date}", "")
-      full_command = "#{command} #{backup_file} #{target_file}"
-      puts("\t#{full_command}") if @debug
-      system(full_command)
+    # create dir for pipeline logs
+    def create_log_dir
+      full_path_to_log = "#{@path}/#{@log_dir}"
+      Dir.mkdir(full_path_to_log) unless File.directory?(full_path_to_log)
     end
-  end
 
-  private
+    # @param date [String]
+    # @return [String] full path to log
     def get_log_name(date)
-      return "#{@log_dir}/#{@created_log_name}#{date}"
+      "#{@path}/#{@log_dir}/#{@created_get_log_name}#{date}"
     end
 
+    # appends to log file
+    # @param line [String] line to append to log file
+    # @param date [String]
     def add_to_log(line, date)
-      File.open(get_log_name(date), "a") {|file| file.write(line)}
+      create_log_dir # if log doesn't exist
+      File.open(get_log_name(date), 'a') { |file| file.write(line) }
     end
-
+  end
 end