toys-core 0.11.3 → 0.12.1

data/lib/toys/utils/gems.rb

@@ -132,8 +132,8 @@ module Toys
                       end
         @on_conflict = on_conflict || :error
         @terminal = terminal
-        @input = input || ::STDIN
-        @output = output || ::STDOUT
+        @input = input || $stdin
+        @output = output || $stdout
       end
 
       ##
@@ -170,22 +170,28 @@ module Toys
       #     recognized as Gemfiles, when searching because gemfile_path is not
       #     given. Defaults to {DEFAULT_GEMFILE_NAMES}.
       #
+      # @param retries [Integer] Number of times to retry bundler operations.
+      #     Optional.
+      #
       # @return [void]
       #
       def bundle(groups: nil,
                  gemfile_path: nil,
                  search_dirs: nil,
-                 gemfile_names: nil)
+                 gemfile_names: nil,
+                 retries: nil)
         Array(search_dirs).each do |dir|
           break if gemfile_path
           gemfile_path = Gems.find_gemfile(dir, gemfile_names: gemfile_names)
         end
         raise GemfileNotFoundError, "Gemfile not found" unless gemfile_path
+        gemfile_path = ::File.absolute_path(gemfile_path)
         Gems.synchronize do
           if configure_gemfile(gemfile_path)
             activate("bundler", "~> 2.1")
             require "bundler"
-            setup_bundle(gemfile_path, groups || [])
+            lockfile_path = find_lockfile_path(gemfile_path)
+            setup_bundle(gemfile_path, lockfile_path, groups: groups, retries: retries)
           end
         end
       end
@@ -287,21 +293,47 @@ module Toys
         true
       end
 
-      def setup_bundle(gemfile_path, groups)
+      def find_lockfile_path(gemfile_path)
+        if ::File.basename(gemfile_path) == "gems.rb"
+          ::File.join(::File.dirname(gemfile_path), "gems.locked")
+        else
+          "#{gemfile_path}.lock"
+        end
+      end
+
+      def setup_bundle(gemfile_path, lockfile_path, groups: nil, retries: nil)
+        groups = Array(groups)
+        old_lockfile_contents = save_old_lockfile(lockfile_path)
         begin
-          modify_bundle_definition(gemfile_path)
+          modify_bundle_definition(gemfile_path, lockfile_path)
           ::Bundler.ui.silence { ::Bundler.setup(*groups) }
         rescue ::Bundler::GemNotFound, ::Bundler::VersionConflict
           restore_toys_libs
-          install_bundle(gemfile_path)
+          install_bundle(gemfile_path, retries: retries)
+          old_lockfile_contents = save_old_lockfile(lockfile_path)
           ::Bundler.reset!
-          modify_bundle_definition(gemfile_path)
+          modify_bundle_definition(gemfile_path, lockfile_path)
           ::Bundler.ui.silence { ::Bundler.setup(*groups) }
         end
         restore_toys_libs
+      ensure
+        restore_old_lockfile(lockfile_path, old_lockfile_contents)
+      end
+
+      def save_old_lockfile(lockfile_path)
+        return nil unless ::File.readable?(lockfile_path) && ::File.writable?(lockfile_path)
+        ::File.read(lockfile_path)
+      end
+
+      def restore_old_lockfile(lockfile_path, contents)
+        return unless contents
+        ::File.open(lockfile_path, "w") do |file|
+          file.write(contents)
+        end
       end
 
-      def modify_bundle_definition(gemfile_path)
+      def modify_bundle_definition(gemfile_path, lockfile_path)
+        ::Bundler.configure
         builder = ::Bundler::Dsl.new
         builder.eval_gemfile(gemfile_path)
         toys_gems = ["toys-core"]
@@ -312,7 +344,7 @@ module Toys
           add_gem_to_definition(builder, "toys")
           toys_gems << "toys"
         end
-        definition = builder.to_definition(gemfile_path + ".lock", { gems: toys_gems })
+        definition = builder.to_definition(lockfile_path, { gems: toys_gems })
         ::Bundler.instance_variable_set(:@definition, definition)
       end
 
@@ -357,19 +389,21 @@ module Toys
         end
       end
 
-      def install_bundle(gemfile_path)
+      def install_bundle(gemfile_path, retries: nil)
         gemfile_dir = ::File.dirname(gemfile_path)
         unless permission_to_bundle?
           raise BundleNotInstalledError,
                 "Your bundle is not installed. Consider running" \
                   " `cd #{gemfile_dir} && bundle install`"
         end
+        retries = retries.to_i
+        args = retries.positive? ? ["--retry=#{retries}"] : []
         require "bundler/cli"
         begin
-          ::Bundler::CLI.start(["install"])
-        rescue ::Bundler::GemNotFound, ::Bundler::InstallError
+          ::Bundler::CLI.start(["install"] + args)
+        rescue ::Bundler::GemNotFound, ::Bundler::InstallError, ::Bundler::VersionConflict
           terminal.puts("Failed to install. Trying update...")
-          ::Bundler::CLI.start(["update"])
+          ::Bundler::CLI.start(["update"] + args)
         end
       end
     end

data/lib/toys/utils/git_cache.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+require "digest"
+require "fileutils"
+require "toys/utils/exec"
+require "toys/utils/xdg"
+
+module Toys
+  module Utils
+    ##
+    # This object provides cached access to remote git data. Given a remote
+    # repository, a path, and a commit, it makes the files availble in the
+    # local filesystem. Access is cached, so repeated requests do not hit the
+    # remote repository again.
+    #
+    # This class is used by the Loader to load tools from git. Tools can also
+    # use the `:git_cache` mixin for direct access to this class.
+    #
+    class GitCache
+      ##
+      # GitCache encountered a failure
+      #
+      class Error < ::StandardError
+        ##
+        # Create a GitCache::Error.
+        #
+        # @param message [String] The error message
+        # @param result [Toys::Utils::Exec::Result] The result of a git
+        #     command execution, or `nil` if this error was not due to a git
+        #     command error.
+        #
+        def initialize(message, result)
+          super(message)
+          @exec_result = result
+        end
+
+        ##
+        # @return [Toys::Utils::Exec::Result] The result of a git command
+        #     execution, or `nil` if this error was not due to a git command
+        #     error.
+        #
+        attr_reader :exec_result
+      end
+
+      ##
+      # Access a git cache.
+      #
+      # @param cache_dir [String] The path to the cache directory. Defaults to
+      #     a specific directory in the user's XDG cache.
+      #
+      def initialize(cache_dir: nil)
+        @cache_dir = ::File.expand_path(cache_dir || default_cache_dir)
+        @exec = Utils::Exec.new(out: :capture, err: :capture)
+      end
+
+      ##
+      # Find the given git-based files from the git cache, loading from the
+      # remote repo if necessary.
+      #
+      # @param remote [String] The URL of the git repo. Required.
+      # @param path [String] The path to the file or directory within the repo.
+      #     Optional. Defaults to the entire repo.
+      # @param commit [String] The commit reference, which may be a SHA or any
+      #     git ref such as a branch or tag. Optional. Defaults to `HEAD`.
+      # @param update [Boolean] Force update of non-SHA commit references, even
+      #     if it has previously been loaded.
+      #
+      # @return [String] The full path to the cached files.
+      #
+      def find(remote, path: nil, commit: nil, update: false)
+        path ||= ""
+        commit ||= "HEAD"
+        dir = ensure_dir(remote)
+        lock_repo(dir) do
+          ensure_repo(dir, remote)
+          sha = ensure_commit(dir, commit, update)
+          ensure_source(dir, sha, path.to_s)
+        end
+      end
+
+      ##
+      # The cache directory.
+      #
+      # @return [String]
+      #
+      attr_reader :cache_dir
+
+      # @private Used for testing
+      def repo_dir_for(remote)
+        ::File.join(@cache_dir, remote_dir_name(remote), "repo")
+      end
+
+      private
+
+      def remote_dir_name(remote)
+        ::Digest::MD5.hexdigest(remote)
+      end
+
+      def source_name(sha, path)
+        digest = ::Digest::MD5.hexdigest("#{sha}#{path}")
+        "#{digest}#{::File.extname(path)}"
+      end
+
+      def repo_dir_name
+        "repo"
+      end
+
+      def default_cache_dir
+        ::File.join(XDG.new.cache_home, "toys", "git")
+      end
+
+      def git(dir, cmd, error_message: nil)
+        result = @exec.exec(["git"] + cmd, chdir: dir)
+        if result.failed?
+          raise GitCache::Error.new("Could not run git command line", result)
+        end
+        if block_given?
+          yield result
+        elsif result.error? && error_message
+          raise GitCache::Error.new(error_message, result)
+        else
+          result
+        end
+      end
+
+      def ensure_dir(remote)
+        dir = ::File.join(@cache_dir, remote_dir_name(remote))
+        ::FileUtils.mkdir_p(dir)
+        dir
+      end
+
+      def lock_repo(dir)
+        lock_path = ::File.join(dir, "repo.lock")
+        ::File.open(lock_path, ::File::RDWR | ::File::CREAT) do |file|
+          file.flock(::File::LOCK_EX)
+          yield
+        end
+      end
+
+      def ensure_repo(dir, remote)
+        repo_dir = ::File.join(dir, repo_dir_name)
+        ::FileUtils.mkdir_p(repo_dir)
+        result = git(repo_dir, ["remote", "get-url", "origin"])
+        unless result.success? && result.captured_out.strip == remote
+          ::FileUtils.rm_rf(repo_dir)
+          ::FileUtils.mkdir_p(repo_dir)
+          git(repo_dir, ["init"],
+              error_message: "Unable to initialize git repository")
+          git(repo_dir, ["remote", "add", "origin", remote],
+              error_message: "Unable to add git remote")
+        end
+      end
+
+      def ensure_commit(dir, commit, update = false)
+        local_commit = "toys-git-cache/#{commit}"
+        repo_dir = ::File.join(dir, repo_dir_name)
+        is_sha = commit =~ /^[0-9a-f]{40}$/
+        if update && !is_sha || !commit_exists?(repo_dir, local_commit)
+          git(repo_dir, ["fetch", "--depth=1", "origin", "#{commit}:#{local_commit}"],
+              error_message: "Unable to to fetch commit: #{commit}")
+        end
+        result = git(repo_dir, ["rev-parse", local_commit],
+                     error_message: "Unable to retrieve commit: #{local_commit}")
+        result.captured_out.strip
+      end
+
+      def commit_exists?(repo_dir, commit)
+        result = git(repo_dir, ["cat-file", "-t", commit])
+        result.success? && result.captured_out.strip == "commit"
+      end
+
+      def ensure_source(dir, sha, path)
+        source_path = ::File.join(dir, source_name(sha, path))
+        unless ::File.exist?(source_path)
+          repo_dir = ::File.join(dir, repo_dir_name)
+          git(repo_dir, ["checkout", sha])
+          from_path = ::File.join(repo_dir, path)
+          ::FileUtils.cp_r(from_path, source_path)
+        end
+        source_path
+      end
+    end
+  end
+end

data/lib/toys/utils/help_text.rb

@@ -44,11 +44,12 @@ module Toys
       ##
       # Create a usage helper.
       #
-      # @param tool [Toys::Tool] The tool to document.
+      # @param tool [Toys::ToolDefinition] The tool to document.
       # @param loader [Toys::Loader] A loader that can provide subcommands.
       # @param executable_name [String] The name of the executable.
       #     e.g. `"toys"`.
-      # @param delegates [Array<Toys::Tool>] The delegation path to the tool.
+      # @param delegates [Array<Toys::ToolDefinition>] The delegation path to
+      #     the tool.
       #
       # @return [Toys::Utils::HelpText]
       #
@@ -60,8 +61,8 @@ module Toys
       end
 
       ##
-      # The Tool being documented.
-      # @return [Toys::Tool]
+      # The ToolDefinition being documented.
+      # @return [Toys::ToolDefinition]
       #
       attr_reader :tool
 
@@ -72,6 +73,8 @@ module Toys
       #     display all subtools recursively. Defaults to false.
       # @param include_hidden [Boolean] Include hidden subtools (i.e. whose
       #     names begin with underscore.) Default is false.
+      # @param separate_sources [Boolean] Split up tool list by source root.
+      #     Defaults to false.
       # @param left_column_width [Integer] Width of the first column. Default
       #     is {DEFAULT_LEFT_COLUMN_WIDTH}.
       # @param indent [Integer] Indent width. Default is {DEFAULT_INDENT}.
@@ -80,13 +83,14 @@ module Toys
       #
       # @return [String] A usage string.
       #
-      def usage_string(recursive: false, include_hidden: false,
+      def usage_string(recursive: false, include_hidden: false, separate_sources: false,
                        left_column_width: nil, indent: nil, wrap_width: nil)
         left_column_width ||= DEFAULT_LEFT_COLUMN_WIDTH
         indent ||= DEFAULT_INDENT
-        subtools = find_subtools(recursive, nil, include_hidden)
+        subtools = collect_subtool_info(recursive, nil, include_hidden, separate_sources)
         assembler = UsageStringAssembler.new(
-          @tool, @executable_name, subtools, indent, left_column_width, wrap_width
+          @tool, @executable_name, subtools, separate_sources,
+          indent, left_column_width, wrap_width
         )
         assembler.result
       end
@@ -102,6 +106,8 @@ module Toys
       #     names begin with underscore.) Default is false.
       # @param show_source_path [Boolean] If true, shows the source path
       #     section. Defaults to false.
+      # @param separate_sources [Boolean] Split up tool list by source root.
+      #     Defaults to false.
       # @param indent [Integer] Indent width. Default is {DEFAULT_INDENT}.
       # @param indent2 [Integer] Second indent width. Default is
       #     {DEFAULT_INDENT}.
@@ -112,13 +118,14 @@ module Toys
       # @return [String] A usage string.
       #
       def help_string(recursive: false, search: nil, include_hidden: false,
-                      show_source_path: false,
+                      show_source_path: false, separate_sources: false,
                       indent: nil, indent2: nil, wrap_width: nil, styled: true)
         indent ||= DEFAULT_INDENT
         indent2 ||= DEFAULT_INDENT
-        subtools = find_subtools(recursive, search, include_hidden)
+        subtools = collect_subtool_info(recursive, search, include_hidden, separate_sources)
         assembler = HelpStringAssembler.new(
-          @tool, @executable_name, @delegates, subtools, search, show_source_path,
+          @tool, @executable_name, @delegates, subtools, search,
+          show_source_path, separate_sources,
           indent, indent2, wrap_width, styled
         )
         assembler.result
@@ -133,6 +140,8 @@ module Toys
       #     listing subtools. Defaults to `nil` which finds all subtools.
       # @param include_hidden [Boolean] Include hidden subtools (i.e. whose
       #     names begin with underscore.) Default is false.
+      # @param separate_sources [Boolean] Split up tool list by source root.
+      #     Defaults to false.
       # @param indent [Integer] Indent width. Default is {DEFAULT_INDENT}.
       # @param wrap_width [Integer,nil] Wrap width of the column, or `nil` to
       #     disable wrap. Default is `nil`.
@@ -141,17 +150,23 @@ module Toys
       # @return [String] A usage string.
       #
       def list_string(recursive: false, search: nil, include_hidden: false,
-                      indent: nil, wrap_width: nil, styled: true)
+                      separate_sources: false, indent: nil, wrap_width: nil, styled: true)
         indent ||= DEFAULT_INDENT
-        subtools = find_subtools(recursive, search, include_hidden)
-        assembler = ListStringAssembler.new(@tool, subtools, recursive, search,
+        subtools = collect_subtool_info(recursive, search, include_hidden, separate_sources)
+        assembler = ListStringAssembler.new(@tool, subtools, recursive, search, separate_sources,
                                             indent, wrap_width, styled)
         assembler.result
       end
 
       private
 
-      def find_subtools(recursive, search, include_hidden)
+      def collect_subtool_info(recursive, search, include_hidden, separate_sources)
+        subtools_by_name = list_subtools(recursive, include_hidden)
+        filter_subtools(subtools_by_name, search)
+        arrange_subtools(subtools_by_name, separate_sources)
+      end
+
+      def list_subtools(recursive, include_hidden)
         subtools_by_name = {}
         ([@tool] + @delegates).each do |tool|
           name_len = tool.full_name.length
@@ -162,20 +177,37 @@ module Toys
             subtools_by_name[local_name] = subtool
           end
         end
+        subtools_by_name
+      end
+
+      def filter_subtools(subtools_by_name, search)
+        if !search.nil? && !search.empty?
+          regex = ::Regexp.new(search, ::Regexp::IGNORECASE)
+          subtools_by_name.delete_if do |local_name, tool|
+            !regex.match?(local_name) && !regex.match?(tool.desc.to_s)
+          end
+        end
+      end
+
+      def arrange_subtools(subtools_by_name, separate_sources)
         subtool_list = subtools_by_name.sort_by { |(local_name, _tool)| local_name }
-        return subtool_list if search.nil? || search.empty?
-        regex = ::Regexp.new(search, ::Regexp::IGNORECASE)
-        subtool_list.find_all do |local_name, tool|
-          regex =~ local_name || regex =~ tool.desc.to_s
+        result = {}
+        subtool_list.each do |(local_name, subtool)|
+          key = separate_sources ? subtool.source_root : nil
+          (result[key] ||= []) << [local_name, subtool]
         end
+        result.sort_by { |source, _subtools| -(source&.priority || -999_999) }
+              .map { |source, subtools| [source&.source_name || "unknown source", subtools] }
       end
 
       ## @private
       class UsageStringAssembler
-        def initialize(tool, executable_name, subtools, indent, left_column_width, wrap_width)
+        def initialize(tool, executable_name, subtools, separate_sources,
+                       indent, left_column_width, wrap_width)
           @tool = tool
           @executable_name = executable_name
           @subtools = subtools
+          @separate_sources = separate_sources
           @indent = indent
           @left_column_width = left_column_width
           @wrap_width = wrap_width
@@ -193,7 +225,8 @@ module Toys
           add_flag_group_sections
           add_positional_arguments_section if @tool.runnable?
           add_subtool_list_section
-          @result = @lines.join("\n") + "\n"
+          joined_lines = @lines.join("\n")
+          @result = "#{joined_lines}\n"
         end
 
         def add_synopsis_section
@@ -226,7 +259,7 @@ module Toys
             @lines << ""
             desc_str = group.desc.to_s
             desc_str = "Flags" if desc_str.empty?
-            @lines << desc_str + ":"
+            @lines << "#{desc_str}:"
             group.flags.each do |flag|
               add_flag(flag)
             end
@@ -254,11 +287,12 @@ module Toys
         end
 
         def add_subtool_list_section
-          return if @subtools.empty?
-          @lines << ""
-          @lines << "Tools:"
-          @subtools.each do |local_name, subtool|
-            add_right_column_desc(local_name, wrap_desc(subtool.desc))
+          @subtools.each do |source_name, subtool_list|
+            @lines << ""
+            @lines << (@separate_sources ? "Tools from #{source_name}:" : "Tools:")
+            subtool_list.each do |local_name, subtool|
+              add_right_column_desc(local_name, wrap_desc(subtool.desc))
+            end
           end
         end
 
@@ -299,7 +333,7 @@ module Toys
       ## @private
       class HelpStringAssembler
         def initialize(tool, executable_name, delegates, subtools, search_term,
-                       show_source_path, indent, indent2, wrap_width, styled)
+                       show_source_path, separate_sources, indent, indent2, wrap_width, styled)
           require "toys/utils/terminal"
           @tool = tool
           @executable_name = executable_name
@@ -307,6 +341,7 @@ module Toys
           @subtools = subtools
           @search_term = search_term
           @show_source_path = show_source_path
+          @separate_sources = separate_sources
           @indent = indent
           @indent2 = indent2
           @wrap_width = wrap_width
@@ -532,8 +567,17 @@ module Toys
             @lines << indent_str("Showing search results for \"#{@search_term}\"")
             @lines << ""
           end
-          @subtools.each do |local_name, subtool|
-            add_prefix_with_desc(bold(local_name), subtool.desc)
+          first_section = true
+          @subtools.each do |source_name, subtool_list|
+            @lines << "" unless first_section
+            if @separate_sources
+              @lines << indent_str(underline("From #{source_name}"))
+              @lines << ""
+            end
+            subtool_list.each do |local_name, subtool|
+              add_prefix_with_desc(bold(local_name), subtool.desc)
+            end
+            first_section = false
           end
         end
 
@@ -596,12 +640,14 @@ module Toys
 
       ## @private
       class ListStringAssembler
-        def initialize(tool, subtools, recursive, search_term, indent, wrap_width, styled)
+        def initialize(tool, subtools, recursive, search_term, separate_sources,
+                       indent, wrap_width, styled)
           require "toys/utils/terminal"
           @tool = tool
           @subtools = subtools
           @recursive = recursive
           @search_term = search_term
+          @separate_sources = separate_sources
           @indent = indent
           @wrap_width = wrap_width
           assemble(styled)
@@ -626,16 +672,22 @@ module Toys
             else
               "#{top_line} under #{bold(@tool.display_name)}:"
             end
-          @lines << ""
           if @search_term
-            @lines << "Showing search results for \"#{@search_term}\""
             @lines << ""
+            @lines << "Showing search results for \"#{@search_term}\""
           end
         end
 
         def add_list
-          @subtools.each do |local_name, subtool|
-            add_prefix_with_desc(bold(local_name), subtool.desc)
+          @subtools.each do |source_name, subtool_list|
+            @lines << ""
+            if @separate_sources
+              @lines << underline("From: #{source_name}")
+              @lines << ""
+            end
+            subtool_list.each do |local_name, subtool|
+              add_prefix_with_desc(bold(local_name), subtool.desc)
+            end
           end
         end
 
@@ -662,6 +714,10 @@ module Toys
           @lines.apply_styles(str, :bold)
         end
 
+        def underline(str)
+          @lines.apply_styles(str, :underline)
+        end
+
         def indent_str(str)
           "#{' ' * @indent}#{str}"
         end