git_tree 0.3.0 → 1.0.1

data/lib/commands/git_update.rb

@@ -0,0 +1,113 @@
+require 'shellwords'
+require 'timeout'
+require_relative 'abstract_command'
+require_relative '../util/git_tree_walker'
+require_relative '../util/command_runner'
+
+module GitTree
+  class UpdateCommand < GitTree::AbstractCommand
+    include Logging
+
+    attr_writer :walker, :runner
+
+    self.allow_empty_args = true
+
+    def initialize(args = ARGV, options: {})
+      $PROGRAM_NAME = 'git-update'
+      super
+      # Allow walker and runner to be injected for testing
+      @runner = @options.delete(:runner)
+      @walker = @options.delete(:walker)
+    end
+
+    def run
+      setup
+      @runner ||= CommandRunner.new
+      @walker ||= GitTreeWalker.new(@args, options: @options)
+      @walker.process do |dir, thread_id, walker|
+        process_repo(walker, dir, thread_id)
+      end
+    end
+
+    private
+
+    def help(msg = nil)
+      log(QUIET, "Error: #{msg}\n", :red) if msg
+      log QUIET, <<~END_HELP
+        git-update - Recursively updates trees of git repositories.
+
+        If no arguments are given, uses default environment variables (#{GitTreeWalker::DEFAULT_ROOTS.join(', ')}) as roots.
+        These environment variables point to roots of git repository trees to walk.
+        Skips directories containing a .ignore file, and all subdirectories.
+
+        Environment variables that point to the roots of git repository trees must have been exported, for example:
+
+          $ export work=$HOME/work
+
+        Usage: #{$PROGRAM_NAME} [OPTIONS] [ROOTS...]
+
+        OPTIONS:
+          -h, --help           Show this help message and exit.
+          -q, --quiet          Suppress normal output, only show errors.
+          -v, --verbose        Increase verbosity. Can be used multiple times (e.g., -v, -vv).
+
+        ROOTS:
+        When specifying roots, directory paths can be specified, and environment variables can be used, preceded by a dollar sign.
+
+        Usage examples:
+
+        $ #{$PROGRAM_NAME}               # Use default environment variables as roots
+        $ #{$PROGRAM_NAME} $work $sites  # Use specific environment variables
+        $ #{$PROGRAM_NAME} $work /path/to/git/tree
+      END_HELP
+      exit 1
+    end
+
+    # Updates the git repository in the given directory.
+    # @param git_walker [GitTreeWalker] The GitTreeWalker instance.
+    # @param dir [String] The path to the git repository.
+    # @param thread_id [Integer] The ID of the current worker thread.
+    # @return [nil]
+    def process_repo(git_walker, dir, thread_id)
+      abbrev_dir = git_walker.abbreviate_path(dir)
+      log NORMAL, "Updating #{abbrev_dir}", :green
+      log VERBOSE, "Thread #{thread_id}: git -C #{dir} pull", :yellow
+
+      output = nil
+      status = nil
+      begin
+        Timeout.timeout(GitTreeWalker::GIT_TIMEOUT) do
+          log VERBOSE, "Executing: git pull in #{dir}", :yellow
+          output, status_obj = @runner.run('git pull', dir)
+          status = status_obj.exitstatus
+        end
+      rescue Timeout::Error
+        log NORMAL, "[TIMEOUT] Thread #{thread_id}: git pull timed out in #{abbrev_dir}", :red
+        status = -1
+      rescue StandardError => e
+        log NORMAL, "[ERROR] Thread #{thread_id}: #{e.class} in #{abbrev_dir}; #{e.message}\n#{e.backtrace.join("\n")}", :red
+        status = -1
+      end
+
+      if !status.zero?
+        log NORMAL, "[ERROR] git pull failed in #{abbrev_dir} (exit code #{status}):", :red
+        log NORMAL, output.strip, :red unless output.to_s.strip.empty?
+      elsif Logging.verbosity >= VERBOSE
+        # Output from a successful pull is considered NORMAL level
+        log NORMAL, output.strip, :green
+      end
+    end
+  end
+end
+
+if $PROGRAM_NAME == __FILE__ || $PROGRAM_NAME.end_with?('git-update')
+  begin
+    GitTree::UpdateCommand.new(ARGV).run
+  rescue Interrupt
+    log NORMAL, "\nInterrupted by user", :yellow
+    exit! 130
+  rescue StandardError => e
+    log QUIET, "#{e.class}: #{e.message}\n#{e.backtrace.join("\n")}", :red
+    exit! 1
+  end
+end

data/lib/git_tree/version.rb

@@ -1,3 +1,3 @@
 module GitUrlsVersion
-  VERSION = '0.3.0'.freeze
+  VERSION = '1.0.1'.freeze
 end

data/lib/git_tree.rb

@@ -1,31 +1,14 @@
-require 'find'
-require 'rainbow/refinement'
-require 'rugged'
-require_relative 'util'
+require_relative 'git_tree/version'
 
 module GitTree
-  using Rainbow
-
-  # @return array containing directory names to process
-  #   Each directory name ends with a slash, to ensure symlinks are dereferences
-  def self.directories_to_process(root)
-    root_fq = File.expand_path root
-    abort "Error: #{root_fq} is a file, instead of a directory. Cannot recurse.".red if File.file? root_fq
-
-    root_fq = MslinnUtil.deref_symlink(root_fq).to_s
-    abort "Error: #{root_fq} does not exist. Halting.".red unless Dir.exist? root_fq
-
-    result = []
-    Find.find(root_fq) do |path|
-      next if File.file? path
+  # Helper to require all .rb files in a subdirectory
+  def self.require_all(relative_path)
+    Dir[File.join(__dir__, relative_path, '*.rb')].sort.each { |file| require file }
+  end
 
-      Find.prune if File.exist?("#{path}/.ignore")
+  # Require utilities first, as commands may depend on them.
+  require_all 'util'
+  require_all 'commands'
 
-      if Dir.exist?("#{path}/.git")
-        result << path.to_s
-        Find.prune
-      end
-    end
-    result.map { |x| x.delete_prefix("#{root_fq}/") }
-  end
+  include Logging
 end

data/lib/util/command_runner.rb

@@ -0,0 +1,12 @@
+require 'open3'
+
+class CommandRunner
+  # Executes a shell command in a specified directory.
+  # This is wrapped in a class to make it easy to mock in tests.
+  # @param command [String] The shell command to execute.
+  # @param dir [String] The directory to execute the command in.
+  # @return [Array] A tuple containing the output and the status object.
+  def run(command, dir)
+    Open3.capture2e(command, chdir: dir)
+  end
+end

data/lib/{util.rb → util/gem_support.rb}

@@ -1,5 +1,8 @@
-module MslinnUtil
-  # @param paths [Array[String]] all start with a leading '/' (they are assumed to be absolute paths).
+require 'pathname'
+
+module GemSupport
+  # @param paths [Array<String>] all start with a leading '/' (they are assumed to be absolute paths).
+  # @param allow_root_match [Boolean]
   # @return [String] the longest path prefix that is a prefix of all paths in array.
   #   If array is empty, return ''.
   #   If only the leading slash matches, and allow_root_match is true, return '/', else return ''.
@@ -25,8 +28,10 @@ module MslinnUtil
     result.empty? && allow_root_match ? '/' : result
   end
 
-  # @param paths [Array[String]] absolute paths to examine
+  # @param paths [Array<String>] absolute paths to examine
   # @param level [Int] minimum # of leading directory names in result, origin 1
+  # @param allow_root_match [Boolean] Whether to return '/' if only the root matches.
+  #        Defaults to false.
   def self.roots(paths, level, allow_root_match: false)
     abort "Error: level must be positive, but it is #{level}." unless level.positive?
     return allow_root_match ? '/' : '' if paths.empty?
@@ -51,27 +56,32 @@ module MslinnUtil
     allow_root_match ? '/' : ''
   end
 
-  # @param paths [Array[String]] absolute paths to examine
+  # @param paths [Array<String>] absolute paths to examine
   # @param level is origin 1
   def self.trim_to_level(paths, level)
     result = paths.map do |x|
       elements = x.split('/').reject(&:empty?)
-      '/' + elements[0..level - 1].join('/')
+      '/' + elements[0..(level - 1)].join('/')
     end
     result.sort.uniq
   end
 
   # @return Path to symlink
   def self.deref_symlink(symlink)
-    require 'pathname'
     Pathname.new(symlink).realpath
   end
 
+  # @param string [String] The string to ensure ends with the suffix.
+  # @param suffix [String] The suffix to ensure the string ends with.
   def self.ensure_ends_with(string, suffix)
     string = string.delete_suffix suffix
     "#{string}#{suffix}"
   end
 
+  # Expands environment variables in a string.
+  #
+  # @param str [String] The string containing environment variables to expand.
+  # @return [String] The string with environment variables expanded.
   def self.expand_env(str)
     str.gsub(/\$([a-zA-Z_][a-zA-Z0-9_]*)|\${\g<1>}|%\g<1>%/) do
       ENV.fetch(Regexp.last_match(1), nil)

data/lib/util/git_tree_walker.rb

@@ -0,0 +1,75 @@
+require 'English'
+require 'etc'
+require 'shellwords'
+require 'optparse'
+require 'timeout'
+require_relative 'thread_pool_manager'
+require_relative 'log'
+
+class GitTreeWalker
+  include Logging
+
+  attr_reader :display_roots, :root_map
+
+  DEFAULT_ROOTS = %w[sites sitesUbuntu work].freeze
+  GIT_TIMEOUT = 10 # TODO: for debuggin only; should be 300 # 5 minutes per git pull
+  IGNORED_DIRECTORIES = ['.', '..', '.venv'].freeze
+
+  def initialize(args = ARGV, options: {})
+    @options = options
+    @root_map = {}
+    @display_roots = []
+    determine_roots(args)
+  end
+
+  def abbreviate_path(dir)
+    @root_map.each do |display_root, expanded_paths|
+      expanded_paths.each do |expanded_path|
+        return dir.sub(expanded_path, display_root) if dir.start_with?(expanded_path)
+      end
+    end
+    dir # Return original if no match
+  end
+
+  def process(&) # Accepts a block
+    log VERBOSE, "Processing #{@display_roots.join(' ')}", :green
+    if @options[:serial]
+      log VERBOSE, "Running in serial mode.", :yellow
+      find_and_process_repos do |dir, _root_arg|
+        yield(dir, 0, self) # task, thread_id, walker
+      end
+    else
+      pool = FixedThreadPoolManager.new(0.75)
+      # The block passed to pool.start now receives the walker instance (self)
+      pool.start do |_worker, dir, thread_id|
+        yield(dir, thread_id, self)
+      end
+      # Run the directory scanning in a separate thread so the main thread can handle interrupts.
+      producer_thread = Thread.new do
+        find_and_process_repos do |dir, _root_arg|
+          pool.add_task(dir)
+        end
+      end
+
+      # Wait for the producer to finish, then wait for the pool to complete.
+      producer_thread.join
+      pool.wait_for_completion
+    end
+  rescue Interrupt
+    # If interrupted, ensure the pool is shut down and then let the main command handle the exit.
+    pool&.shutdown
+    raise
+  end
+
+  # Finds git repos and yields them to the block. Does not use thread pool.
+  def find_and_process_repos(&)
+    visited = Set.new
+    @root_map.each do |root_arg, paths|
+      paths.sort.each do |root_path|
+        find_git_repos_recursive(root_path, visited) { |dir| yield(dir, root_arg) }
+      end
+    end
+  end
+end
+
+require_relative 'git_tree_walker_private'

data/lib/util/git_tree_walker_private.rb

@@ -0,0 +1,60 @@
+require_relative 'log'
+
+class GitTreeWalker
+  include Logging
+
+  private
+
+  def determine_roots(args)
+    if args.empty?
+      @display_roots = DEFAULT_ROOTS.map { |r| "$#{r}" }
+      DEFAULT_ROOTS.each do |r|
+        @root_map["$#{r}"] = ENV[r].split.map { |p| File.expand_path(p) } if ENV[r]
+      end
+    else
+      processed_args = args.flat_map { |arg| arg.strip.split(/\s+/) }
+      @display_roots = processed_args.dup
+      processed_args.each do |arg|
+        path = arg
+        if (match = arg.match(/\A'?\$([a-zA-Z_]\w*)'?\z/))
+          var_name = match[1]
+          path = ENV.fetch(var_name, nil)
+        end
+        @root_map[arg] = [File.expand_path(path)] if path
+      end
+    end
+  end
+
+  def sort_directory_entries(directory_path)
+    Dir.children(directory_path).select do |entry|
+      File.directory?(File.join(directory_path, entry))
+    end.sort
+  end
+
+  def find_git_repos_recursive(root_path, visited, &block)
+    return unless File.directory?(root_path)
+
+    return if File.exist?(File.join(root_path, '.ignore'))
+
+    log DEBUG, "Scanning #{root_path}", :green
+    git_dir_or_file = File.join(root_path, '.git')
+    if File.exist?(git_dir_or_file)
+      log DEBUG, "  Found #{git_dir_or_file}", :green
+      unless visited.include?(root_path)
+        visited.add(root_path)
+        yield root_path
+      end
+      return # Prune search
+    else
+      log DEBUG, "  #{root_path} is not a git directory", :green
+    end
+
+    sort_directory_entries(root_path).each do |entry|
+      next if IGNORED_DIRECTORIES.include?(entry)
+
+      find_git_repos_recursive(File.join(root_path, entry), visited, &block)
+    end
+  rescue SystemCallError => e
+    log NORMAL, "Error scanning #{root_path}: #{e.message}", :red
+  end
+end

data/lib/util/log.rb

@@ -0,0 +1,49 @@
+require 'rainbow/refinement'
+
+module Logging
+  using Rainbow
+
+  # Verbosity levels
+  QUIET = 0
+  NORMAL = 1
+  VERBOSE = 2
+  DEBUG = 3
+
+  # Class-level instance variables to hold the verbosity setting for the module
+  @verbosity = NORMAL
+
+  # @return [Integer] The current verbosity level.
+  def self.verbosity
+    @verbosity
+  end
+
+  # @param level [Integer] The new verbosity level.
+  # @return [nil]
+  def self.verbosity=(level)
+    @verbosity = level
+  end
+
+  # A thread-safe output method for colored text to STDERR.
+  # @param level [Integer] The verbosity level of the message.
+  # @param multiline_string [String] The message to log.
+  # @param color [Symbol, nil] The color method to apply from Rainbow, e.g., :red, :green.  If nil, no color is applied.
+  # @return [nil]
+  def log(level, multiline_string, color = nil)
+    return unless Logging.verbosity >= level
+
+    multiline_string.to_s.each_line do |line|
+      line_to_print = line.chomp
+      line_to_print = line_to_print.public_send(color) if color
+      warn line_to_print
+    end
+    $stderr.flush
+  end
+
+  # A thread-safe output method for uncolored text to STDOUT.
+  # @param multiline_string [String] The message to log.
+  # @return [nil]
+  def log_stdout(multiline_string)
+    $stdout.puts multiline_string.to_s
+    $stdout.flush
+  end
+end

data/lib/util/thread_pool_manager.rb

@@ -0,0 +1,116 @@
+require 'etc'
+require_relative 'log'
+
+class FixedThreadPoolManager
+  include Logging
+
+  SHUTDOWN_SIGNAL = :shutdown
+
+  # Calculate the number of worker threads as 75% of available processors
+  # (less one for the monitor thread), with a minimum of 1.
+  # @param percent_available_processors [Float] The percentage of available processors to use for worker threads.
+  def initialize(percent_available_processors = 0.75)
+    if percent_available_processors > 1 || percent_available_processors <= 0
+      msg = <<~END_MSG
+        Error: The allowable range for the ThreadPool.initialize percent_available_processors is between 0 and 1.
+        You provided #{percent_available_processors}.
+      END_MSG
+      log QUIET, msg, :red
+      exit! 1
+    end
+    @worker_count = [(Etc.nprocessors * percent_available_processors).floor, 1].max
+    @main_work_queue = Queue.new
+    @workers = []
+  end
+
+  # Adds a single task to the work queue.
+  # The pool must have been started with `start` first.
+  def add_task(task)
+    @main_work_queue.push(task)
+  end
+
+  # Signals the pool to shut down after all currently queued tasks are processed.
+  # This is a non-blocking method.
+  # When you call it, it simply places a special SHUTDOWN_SIGNAL message onto the
+  # main work queue. The method returns immediately, allowing your main thread to
+  # continue with other tasks.
+  # It's like telling the pool, "I'm not going to give you any more tasks,
+  # so start wrapping things up when you're done with what you have."
+  def shutdown
+    @main_work_queue.push(SHUTDOWN_SIGNAL)
+  end
+
+  # Starts the workers and the monitor, but does not wait for them to complete.
+  # This is for "drip-feeding" tasks.
+  # @param Block of code to execute for each task.
+  # @return nil
+  def start(&)
+    initialize_workers(&)
+  end
+
+  # This is the last method to call when using FixedPoolManager.
+  # This is a blocking method.
+  # It pauses the execution of your main thread and waits until the monitor and all worker threads have
+  # fully completed their work and terminated.
+  def wait_for_completion
+    @worker_count.times { @main_work_queue.push(SHUTDOWN_SIGNAL) }
+
+    last_active_count = -1
+    loop do
+      active_workers = @workers.count(&:alive?)
+      break if active_workers.zero?
+
+      if active_workers != last_active_count
+        warn format("Waiting for %d worker threads to complete...", active_workers) + "\r" if Logging.verbosity > NORMAL
+        last_active_count = active_workers
+      end
+      begin
+        sleep 0.1
+      rescue Interrupt
+        # This can be interrupted by Ctrl-C. We catch it here to allow the main thread's
+        # rescue block to handle the exit gracefully without a stack trace.
+      end
+    end
+
+    warn (" " * 60) + "\r" # Clear the line
+    log NORMAL, "All work is complete.", :green
+  end
+
+  private
+
+  def initialize_workers
+    log NORMAL, "Initializing #{@worker_count} worker threads...", :green
+    @worker_count.times do |i|
+      worker_thread = Thread.new do
+        log NORMAL, "  [Worker #{i}] Started.", :cyan
+        start_time = Time.now
+        start_cpu = Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID)
+        tasks_processed = 0
+
+        loop do
+          task = @main_work_queue.pop # The worker blocks here, waiting for a task.
+          break if task == SHUTDOWN_SIGNAL
+
+          yield(self, task, i) # Execute the provided block of work.
+          tasks_processed += 1
+        end
+
+        if Logging.verbosity >= VERBOSE
+          elapsed_time = Time.now - start_time
+          cpu_time = Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) - start_cpu
+          shutdown_msg = format(
+            "  [Worker #{i}] Shutting down. Processed #{tasks_processed} tasks. Elapsed: %.2fs, CPU: %.2fs",
+            elapsed_time, cpu_time
+          )
+          log VERBOSE, shutdown_msg, :cyan
+        end
+      rescue Interrupt
+        # This thread was interrupted by Ctrl-C, likely while waiting on the queue.
+        # Exit gracefully without a stack trace.
+      end
+      # Suppress automatic error reporting for this thread. The error should be handled elsewhere.
+      worker_thread.report_on_exception = false
+      @workers << worker_thread
+    end
+  end
+end

data/lib/util/zowee_optimizer.rb

@@ -0,0 +1,136 @@
+class ZoweeOptimizer
+  # The ZoweeOptimizer class is responsible for optimizing the environment variable definitions.
+  # It is used by the `git-evars` command to generate a script with shorter and more readable variable names.
+  def initialize(initial_vars = {})
+    @defined_vars = {}
+    initial_vars.each do |var_ref, paths|
+      var_name = var_ref.tr("'$", '')
+      @defined_vars[var_name] = paths.first if paths.any?
+    end
+  end
+
+  # Optimizes a list of paths to generate a script with environment variable definitions.
+  # @param paths [Array<String>] are provided in breadth-first order, so no sorting is needed.
+  # @param initial_roots [Array<String>] a list of initial root variables.
+  # @return [Array<String>] a list of strings, where each string is an export statement for an environment variable.
+  def optimize(paths, initial_roots)
+    output = []
+
+    # Find common prefixes and define intermediate variables
+    define_intermediate_vars(paths)
+
+    paths.each do |path|
+      var_name = generate_var_name(path)
+      next if var_name.nil?
+
+      # Skip defining a var for a root that was passed in.
+      next if initial_roots.include?("$#{var_name}") && @defined_vars[var_name] == path
+
+      best_substitution = find_best_substitution(path)
+
+      value = if best_substitution
+                "$#{best_substitution[:var]}/#{path.sub("#{best_substitution[:path]}/", '')}"
+              else
+                path
+              end
+
+      output << "export #{var_name}=#{value}"
+      @defined_vars[var_name] = path
+    end
+
+    (@intermediate_vars.values + output).uniq
+  end
+
+  # Generates a valid environment variable name from a path.
+  # @param path [String] the path to generate the variable name from.
+  # @return [String] a valid environment variable name.
+  def generate_var_name(path)
+    basename = File.basename(path)
+    return nil if basename.empty?
+
+    parts = basename.split('.')
+    name = if parts.first == 'www' && parts.length > 1
+             parts[1]
+           else
+             parts.first
+           end.tr('-', '_')
+
+    if @defined_vars.key?(name) && @defined_vars[name] != path
+      # Collision. Try to disambiguate.
+      parent_name = File.basename(File.dirname(path))
+      name = "#{parent_name}_#{name}"
+    end
+
+    # Sanitize the name
+    name.gsub!(/[^a-zA-Z0-9_]/, '_')
+
+    # Prepend underscore if it starts with a digit
+    name = "_#{name}" if name.match?(/^[0-9]/)
+
+    name
+  end
+
+  private
+
+  # Defines intermediate variables based on common prefixes in the given paths.
+  # @param paths [Array<String>] a list of paths.
+  def define_intermediate_vars(paths)
+    @intermediate_vars = {}
+    prefixes = {}
+    paths.each do |path|
+      parts = path.split('/')
+      (1...parts.length).each do |i|
+        prefix = parts.take(i).join('/')
+        prefixes[prefix] ||= 0
+        prefixes[prefix] += 1
+      end
+    end
+
+    # Sort by length to define shorter prefixes first
+    sorted_prefixes = prefixes.keys.sort_by(&:length)
+
+    sorted_prefixes.each do |prefix|
+      # An intermediate variable is useful if it is a prefix to at least 2 paths
+      # and is not one of the paths to be defined.
+      # Also, it should not be created if a more specific path from the input list can be used.
+      is_useful = prefixes[prefix] > 1 &&
+                  !@defined_vars.value?(prefix) &&
+                  paths.none? { |p| File.dirname(p) == prefix } &&
+                  @defined_vars.values.compact.none? { |v| prefix.start_with?(v) || v.start_with?(prefix) }
+      is_not_an_input_path = !paths.include?(prefix)
+      next unless is_useful && is_not_an_input_path
+
+      var_name = generate_var_name(prefix)
+      next if var_name.nil?
+
+      best_substitution = find_best_substitution(prefix)
+      value = if best_substitution
+                "$#{best_substitution[:var]}/#{prefix.sub("#{best_substitution[:path]}/", '')}"
+              else
+                prefix
+              end
+
+      unless @defined_vars.key?(var_name)
+        @defined_vars[var_name] = prefix
+        @intermediate_vars[prefix] = "export #{var_name}=#{value}"
+      end
+    end
+  end
+
+  # Finds the best substitution for a given path from the currently defined variables.
+  # @param path [String] the path to find the best substitution for.
+  # @return [Hash] a hash containing the best substitution variable and path, or nil if no substitution is found.
+  def find_best_substitution(path)
+    best_substitution = nil
+    longest_match = 0
+
+    # Find the best existing variable to substitute.
+    @defined_vars.each do |sub_var, sub_path|
+      if path.start_with?("#{sub_path}/") && sub_path.length > longest_match
+        best_substitution = { var: sub_var, path: sub_path }
+        longest_match = sub_path.length
+      end
+    end
+    best_substitution
+  end
+end