toys-core 0.11.5 → 0.12.0

data/lib/toys/errors.rb

@@ -54,6 +54,7 @@ module Toys
       @config_line = config_line
       @tool_name = tool_name
       @tool_args = tool_args
+      set_backtrace(cause.backtrace)
     end
 
     attr_reader :cause

data/lib/toys/flag.rb

@@ -85,11 +85,11 @@ module Toys
     #     true.
     # @param group [Toys::FlagGroup] Group containing this flag.
     # @param desc [String,Array<String>,Toys::WrappableString] Short
-    #     description for the flag. See {Toys::Tool#desc=} for a description of
-    #     allowed formats. Defaults to the empty string.
+    #     description for the flag. See {Toys::ToolDefinition#desc} for a
+    #     description of allowed formats. Defaults to the empty string.
     # @param long_desc [Array<String,Array<String>,Toys::WrappableString>]
-    #     Long description for the flag. See {Toys::Tool#long_desc=} for a
-    #     description of allowed formats. Defaults to the empty array.
+    #     Long description for the flag. See {Toys::ToolDefinition#long_desc}
+    #     for a description of allowed formats. Defaults to the empty array.
     # @param display_name [String] A display name for this flag, used in help
     #     text and error messages.
     # @param used_flags [Array<String>] An array of flags already in use.
@@ -368,7 +368,7 @@ module Toys
       key_str = key.to_s
       flag_str =
         if key_str.length == 1
-          "-#{key_str}" if key_str =~ /[a-zA-Z0-9\?]/
+          "-#{key_str}" if key_str =~ /[a-zA-Z0-9?]/
         elsif key_str.length > 1
           key_str = key_str.downcase.tr("_", "-").gsub(/[^a-z0-9-]/, "").sub(/^-+/, "")
           "--#{key_str}" unless key_str.empty?
@@ -455,23 +455,19 @@ module Toys
       #
       def initialize(str)
         case str
-        when /\A(-([\?\w]))\z/
+        when /\A(-([?\w]))\z/
           setup(str, $1, nil, $1, $2, :short, nil, nil, nil, nil)
-        when /\A(-([\?\w]))( ?)\[(\w+)\]\z/
-          setup(str, $1, nil, $1, $2, :short, :value, :optional, $3, $4)
-        when /\A(-([\?\w]))\[( )(\w+)\]\z/
-          setup(str, $1, nil, $1, $2, :short, :value, :optional, $3, $4)
-        when /\A(-([\?\w]))( ?)(\w+)\z/
+        when /\A(-([?\w]))(?:( ?)\[|\[( ))(\w+)\]\z/
+          setup(str, $1, nil, $1, $2, :short, :value, :optional, $3 || $4, $5)
+        when /\A(-([?\w]))( ?)(\w+)\z/
           setup(str, $1, nil, $1, $2, :short, :value, :required, $3, $4)
-        when /\A--\[no-\](\w[\?\w-]*)\z/
+        when /\A--\[no-\](\w[?\w-]*)\z/
           setup(str, "--#{$1}", "--no-#{$1}", str, $1, :long, :boolean, nil, nil, nil)
-        when /\A(--(\w[\?\w-]*))\z/
+        when /\A(--(\w[?\w-]*))\z/
           setup(str, $1, nil, $1, $2, :long, nil, nil, nil, nil)
-        when /\A(--(\w[\?\w-]*))([= ])\[(\w+)\]\z/
-          setup(str, $1, nil, $1, $2, :long, :value, :optional, $3, $4)
-        when /\A(--(\w[\?\w-]*))\[([= ])(\w+)\]\z/
-          setup(str, $1, nil, $1, $2, :long, :value, :optional, $3, $4)
-        when /\A(--(\w[\?\w-]*))([= ])(\w+)\z/
+        when /\A(--(\w[?\w-]*))(?:([= ])\[|\[([= ]))(\w+)\]\z/
+          setup(str, $1, nil, $1, $2, :long, :value, :optional, $3 || $4, $5)
+        when /\A(--(\w[?\w-]*))([= ])(\w+)\z/
           setup(str, $1, nil, $1, $2, :long, :value, :required, $3, $4)
         else
           raise ToolDefinitionError, "Illegal flag: #{str.inspect}"
@@ -731,6 +727,7 @@ module Toys
       # @param include_negative [Boolean] Whether to include `--no-*` forms.
       #
       def initialize(flag:, include_short: true, include_long: true, include_negative: true)
+        super()
         @flag = flag
         @include_short = include_short
         @include_long = include_long

data/lib/toys/flag_group.rb

@@ -17,11 +17,12 @@ module Toys
     #
     # @param type [Symbol] The type of group. Default is `:optional`.
     # @param desc [String,Array<String>,Toys::WrappableString] Short
-    #     description for the group. See {Toys::Tool#desc=} for a description
-    #     of allowed formats. Defaults to `"Flags"`.
+    #     description for the group. See {Toys::ToolDefinition#desc} for a
+    #     description of allowed formats. Defaults to `"Flags"`.
     # @param long_desc [Array<String,Array<String>,Toys::WrappableString>]
-    #     Long description for the flag group. See {Toys::Tool#long_desc=} for
-    #     a description of allowed formats. Defaults to the empty array.
+    #     Long description for the flag group. See
+    #     {Toys::ToolDefinition#long_desc} for a description of allowed
+    #     formats. Defaults to the empty array.
     # @param name [String,Symbol,nil] The name of the group, or nil for no
     #     name.
     # @return [Toys::FlagGroup::Base] A flag group of the correct subclass.

data/lib/toys/input_file.rb

@@ -11,11 +11,11 @@ module Toys::InputFile # rubocop:disable Style/ClassAndModuleChildren
   end
 
   ## @private
-  def self.evaluate(tool_class, remaining_words, source)
+  def self.evaluate(tool_class, words, priority, remaining_words, source, loader)
     namespace = ::Module.new
     namespace.module_eval do
       include ::Toys::Context::Key
-      @tool_class = tool_class
+      @__tool_class = tool_class
     end
     path = source.source_path
     basename = ::File.basename(path).tr(".-", "_").gsub(/\W/, "")
@@ -23,7 +23,7 @@ module Toys::InputFile # rubocop:disable Style/ClassAndModuleChildren
     str = build_eval_string(name, ::IO.read(path))
     if str
       const_set(name, namespace)
-      ::Toys::DSL::Tool.prepare(tool_class, remaining_words, source) do
+      ::Toys::DSL::Internal.prepare(tool_class, words, priority, remaining_words, source, loader) do
         ::Toys::ContextualError.capture_path("Error while loading Toys config!", path) do
           # rubocop:disable Security/Eval
           eval(str, __binding, path, -2)
@@ -39,7 +39,7 @@ module Toys::InputFile # rubocop:disable Style/ClassAndModuleChildren
     return nil if index.nil?
     "#{string[0, index]}\n" \
       "module #{module_name}\n" \
-      "@tool_class.class_eval do\n" \
+      "@__tool_class.class_eval do\n" \
       "#{string[index..-1]}\n" \
       "end\n" \
       "end\n"

data/lib/toys/loader.rb

@@ -54,7 +54,8 @@ module Toys
       extra_delimiters: "",
       mixin_lookup: nil,
       middleware_lookup: nil,
-      template_lookup: nil
+      template_lookup: nil,
+      git_cache: nil
     )
       if index_file_name && ::File.extname(index_file_name) != ".rb"
         raise ::ArgumentError, "Illegal index file name #{index_file_name.inspect}"
@@ -71,30 +72,82 @@ module Toys
       @loading_started = false
       @worklist = []
       @tool_data = {}
+      @roots_by_priority = {}
       @max_priority = @min_priority = 0
       @stop_priority = BASE_PRIORITY
       @min_loaded_priority = 999_999
       @middleware_stack = Middleware.stack(middleware_stack)
       @delimiter_handler = DelimiterHandler.new(extra_delimiters)
+      @git_cache = git_cache
       get_tool([], BASE_PRIORITY)
     end
 
     ##
     # Add a configuration file/directory to the loader.
     #
-    # @param paths [String,Array<String>] One or more paths to add.
+    # @param path [String] A single path to add.
     # @param high_priority [Boolean] If true, add this path at the top of the
     #     priority list. Defaults to false, indicating the new path should be
     #     at the bottom of the priority list.
+    # @param source_name [String] A custom name for the root source. Optional.
+    # @param context_directory [String,nil,:path,:parent] The context directory
+    #     for tools loaded from this path. You can pass a directory path as a
+    #     string, `:path` to denote the given path, `:parent` to denote the
+    #     given path's parent directory, or `nil` to denote no context.
+    #     Defaults to `:parent`.
     # @return [self]
     #
-    def add_path(paths, high_priority: false)
-      paths = Array(paths)
+    def add_path(path,
+                 high_priority: false,
+                 source_name: nil,
+                 context_directory: :parent)
       @mutex.synchronize do
         raise "Cannot add a path after tool loading has started" if @loading_started
         priority = high_priority ? (@max_priority += 1) : (@min_priority -= 1)
-        paths.each do |path|
-          source = SourceInfo.create_path_root(path, @data_dir_name, @lib_dir_name)
+        source = SourceInfo.create_path_root(path, priority,
+                                             context_directory: context_directory,
+                                             data_dir_name: @data_dir_name,
+                                             lib_dir_name: @lib_dir_name,
+                                             source_name: source_name)
+        @roots_by_priority[priority] = source
+        @worklist << [source, [], priority]
+      end
+      self
+    end
+
+    ##
+    # Add a set of configuration files/directories from a common directory to
+    # the loader. The set of paths will be added at the same priority level and
+    # will share a root.
+    #
+    # @param root_path [String] A root path to be seen as the root source. This
+    #     should generally be a directory containing the paths to add.
+    # @param relative_paths [String,Array<String>] One or more paths to add, as
+    #     relative paths from the common root.
+    # @param high_priority [Boolean] If true, add the paths at the top of the
+    #     priority list. Defaults to false, indicating the new paths should be
+    #     at the bottom of the priority list.
+    # @param context_directory [String,nil,:path,:parent] The context directory
+    #     for tools loaded from this path. You can pass a directory path as a
+    #     string, `:path` to denote the given root path, `:parent` to denote
+    #     the given root path's parent directory, or `nil` to denote no context.
+    #     Defaults to `:path`.
+    # @return [self]
+    #
+    def add_path_set(root_path, relative_paths,
+                     high_priority: false,
+                     context_directory: :path)
+      relative_paths = Array(relative_paths)
+      @mutex.synchronize do
+        raise "Cannot add a path after tool loading has started" if @loading_started
+        priority = high_priority ? (@max_priority += 1) : (@min_priority -= 1)
+        root_source = SourceInfo.create_path_root(root_path, priority,
+                                                  context_directory: context_directory,
+                                                  data_dir_name: @data_dir_name,
+                                                  lib_dir_name: @lib_dir_name)
+        @roots_by_priority[priority] = root_source
+        relative_paths.each do |path, individual_name|
+          source = root_source.relative_child(path, source_name: individual_name)
           @worklist << [source, [], priority]
         end
       end
@@ -107,19 +160,65 @@ module Toys
     # @param high_priority [Boolean] If true, add this block at the top of the
     #     priority list. Defaults to false, indicating the block should be at
     #     the bottom of the priority list.
-    # @param name [String] The source name that will be shown in documentation
-    #     for tools defined in this block. If omitted, a default unique string
-    #     will be generated.
+    # @param source_name [String] The source name that will be shown in
+    #     documentation for tools defined in this block. If omitted, a default
+    #     unique string will be generated.
     # @param block [Proc] The block of configuration, executed in the context
     #     of the tool DSL {Toys::DSL::Tool}.
+    # @param context_directory [String,nil] The context directory for tools
+    #     loaded from this block. You can pass a directory path as a string, or
+    #     `nil` to denote no context. Defaults to `nil`.
     # @return [self]
     #
-    def add_block(high_priority: false, name: nil, &block)
-      name ||= "(Code block #{block.object_id})"
+    def add_block(high_priority: false,
+                  source_name: nil,
+                  context_directory: nil,
+                  &block)
       @mutex.synchronize do
         raise "Cannot add a block after tool loading has started" if @loading_started
         priority = high_priority ? (@max_priority += 1) : (@min_priority -= 1)
-        source = SourceInfo.create_proc_root(block, name, @data_dir_name, @lib_dir_name)
+        source = SourceInfo.create_proc_root(block, priority,
+                                             context_directory: context_directory,
+                                             source_name: source_name,
+                                             data_dir_name: @data_dir_name,
+                                             lib_dir_name: @lib_dir_name)
+        @roots_by_priority[priority] = source
+        @worklist << [source, [], priority]
+      end
+      self
+    end
+
+    ##
+    # Add a configuration git source to the loader.
+    #
+    # @param git_remote [String] The git repo URL
+    # @param git_path [String] The path to the relevant file or directory in
+    #     the repo. Specify the empty string to use the entire repo.
+    # @param git_commit [String] The git ref (i.e. SHA, tag, or branch name)
+    # @param high_priority [Boolean] If true, add this path at the top of the
+    #     priority list. Defaults to false, indicating the new path should be
+    #     at the bottom of the priority list.
+    # @param update [Boolean] If the commit is not a SHA, pulls any updates
+    #     from the remote. Defaults to false, which uses a local cache and does
+    #     not update if the commit has been fetched previously.
+    # @param context_directory [String,nil] The context directory for tools
+    #     loaded from this source. You can pass a directory path as a string,
+    #     or `nil` to denote no context. Defaults to `nil`.
+    # @return [self]
+    #
+    def add_git(git_remote, git_path, git_commit,
+                high_priority: false,
+                update: false,
+                context_directory: nil)
+      @mutex.synchronize do
+        raise "Cannot add a git source after tool loading has started" if @loading_started
+        priority = high_priority ? (@max_priority += 1) : (@min_priority -= 1)
+        path = git_cache.find(git_remote, path: git_path, commit: git_commit, update: update)
+        source = SourceInfo.create_git_root(git_remote, git_path, git_commit, path, priority,
+                                            context_directory: context_directory,
+                                            data_dir_name: @data_dir_name,
+                                            lib_dir_name: @lib_dir_name)
+        @roots_by_priority[priority] = source
         @worklist << [source, [], priority]
       end
       self
@@ -136,7 +235,7 @@ module Toys
     # that are not part of the tool name and should be passed as tool args.
     #
     # @param args [Array<String>] Command line arguments
-    # @return [Array(Toys::Tool,Array<String>)]
+    # @return [Array(Toys::ToolDefinition,Array<String>)]
     #
     def lookup(args)
       orig_prefix, args = @delimiter_handler.find_orig_prefix(args)
@@ -157,13 +256,13 @@ module Toys
     # the given name, returns `nil`.
     #
     # @param words [Array<String>] The tool name
-    # @return [Toys::Tool] if the tool was found
+    # @return [Toys::ToolDefinition] if the tool was found
     # @return [nil] if no such tool exists
     #
     def lookup_specific(words)
       words = @delimiter_handler.split_path(words.first) if words.size == 1
       load_for_prefix(words)
-      tool = get_tool_data(words).cur_definition
+      tool = get_tool_data(words, false)&.cur_definition
       finish_definitions_in_tree(words) if tool
       tool
     end
@@ -177,7 +276,7 @@ module Toys
     #     rather than just the immediate children (the default)
     # @param include_hidden [Boolean] If true, include hidden subtools,
     #     e.g. names beginning with underscores.
-    # @return [Array<Toys::Tool>] An array of subtools.
+    # @return [Array<Toys::ToolDefinition>] An array of subtools.
     #
     def list_subtools(words, recursive: false, include_hidden: false)
       load_for_prefix(words)
@@ -234,8 +333,8 @@ module Toys
     #
     # @private
     #
-    def get_tool(words, priority)
-      get_tool_data(words).get_tool(priority, self)
+    def get_tool(words, priority, tool_class = nil)
+      get_tool_data(words, true).get_tool(priority, self, tool_class)
     end
 
     ##
@@ -248,7 +347,7 @@ module Toys
     # @private
     #
     def activate_tool(words, priority)
-      get_tool_data(words).activate_tool(priority, self)
+      get_tool_data(words, true).activate_tool(priority, self)
     end
 
     ##
@@ -267,10 +366,11 @@ module Toys
     #
     # @private
     #
-    def build_tool(words, priority)
+    def build_tool(words, priority, tool_class = nil)
       parent = words.empty? ? nil : get_tool(words.slice(0..-2), priority)
       middleware_stack = parent ? parent.subtool_middleware_stack : @middleware_stack
-      Tool.new(self, parent, words, priority, middleware_stack, @middleware_lookup)
+      ToolDefinition.new(parent, words, priority, @roots_by_priority[priority],
+                         middleware_stack, @middleware_lookup, tool_class)
     end
 
     ##
@@ -335,12 +435,31 @@ module Toys
     # @private
     #
     def load_path(parent_source, path, words, remaining_words, priority)
+      if parent_source.git_remote
+        raise LoaderError,
+              "Git source #{parent_source.source_name} tried to load from the local file system"
+      end
       source = parent_source.absolute_child(path)
       @mutex.synchronize do
         load_validated_path(source, words, remaining_words, priority)
       end
     end
 
+    ##
+    # Load configuration from the given git remote. This is called from the
+    # `load_git` directive in the DSL.
+    #
+    # @private
+    #
+    def load_git(parent_source, git_remote, git_path, git_commit, words, remaining_words, priority,
+                 update: false)
+      path = git_cache.find(git_remote, path: git_path, commit: git_commit, update: update)
+      source = parent_source.git_child(git_remote, git_path, git_commit, path)
+      @mutex.synchronize do
+        load_validated_path(source, words, remaining_words, priority)
+      end
+    end
+
     ##
     # Load a subtool block. Called from the `tool` directive in the DSL.
     #
@@ -353,6 +472,18 @@ module Toys
       end
     end
 
+    ##
+    # Get a GitCache.
+    #
+    # @private
+    #
+    def git_cache
+      @git_cache ||= begin
+        require "toys/utils/git_cache"
+        Utils::GitCache.new
+      end
+    end
+
     ##
     # Determine the next setting for remaining_words, given a word.
     #
@@ -381,8 +512,10 @@ module Toys
       result
     end
 
-    def get_tool_data(words)
-      @mutex.synchronize { @tool_data[words] ||= ToolData.new(words) }
+    def get_tool_data(words, create)
+      @mutex.synchronize do
+        create ? (@tool_data[words] ||= ToolData.new(words)) : @tool_data[words]
+      end
     end
 
     ##
@@ -403,7 +536,7 @@ module Toys
       if remaining_words
         update_min_loaded_priority(priority)
         tool_class = get_tool(words, priority).tool_class
-        DSL::Tool.prepare(tool_class, remaining_words, source) do
+        DSL::Internal.prepare(tool_class, words, priority, remaining_words, source, self) do
           ContextualError.capture("Error while loading Toys config!") do
             tool_class.class_eval(&source.source_proc)
           end
@@ -425,7 +558,7 @@ module Toys
       if source.source_type == :file
         update_min_loaded_priority(priority)
         tool_class = get_tool(words, priority).tool_class
-        InputFile.evaluate(tool_class, remaining_words, source)
+        InputFile.evaluate(tool_class, words, priority, remaining_words, source, self)
       else
         do_preload(source.source_path)
         load_index_in(source, words, remaining_words, priority)
@@ -467,16 +600,20 @@ module Toys
       if @preload_dir_name
         preload_dir = ::File.join(path, @preload_dir_name)
         if ::File.directory?(preload_dir) && ::File.readable?(preload_dir)
-          ::Dir.entries(preload_dir).each do |child|
-            next unless ::File.extname(child) == ".rb"
-            preload_file = ::File.join(preload_dir, child)
-            next if !::File.file?(preload_file) || !::File.readable?(preload_file)
-            require preload_file
-          end
+          preload_dir_contents(preload_dir)
         end
       end
     end
 
+    def preload_dir_contents(preload_dir)
+      ::Dir.entries(preload_dir).each do |child|
+        next unless ::File.extname(child) == ".rb"
+        preload_file = ::File.join(preload_dir, child)
+        next if !::File.file?(preload_file) || !::File.readable?(preload_file)
+        require preload_file
+      end
+    end
+
     def sort_tools_by_name(tools)
       tools.sort! do |a, b|
         a = a.full_name
@@ -520,7 +657,7 @@ module Toys
     class ToolData
       # @private
       def initialize(words)
-        @words = words
+        @words = validate_words(words)
         @definitions = {}
         @top_priority = @active_priority = nil
         @mutex = ::Monitor.new
@@ -537,12 +674,15 @@ module Toys
       end
 
       # @private
-      def get_tool(priority, loader)
+      def get_tool(priority, loader, tool_class = nil)
         @mutex.synchronize do
           if @top_priority.nil? || @top_priority < priority
             @top_priority = priority
           end
-          @definitions[priority] ||= loader.build_tool(@words, priority)
+          if tool_class && @definitions.include?(priority)
+            raise ToolDefinitionError, "Tool already defined for #{@words.inspect}"
+          end
+          @definitions[priority] ||= loader.build_tool(@words, priority, tool_class)
         end
       end
 
@@ -558,6 +698,14 @@ module Toys
 
       private
 
+      def validate_words(words)
+        words.each do |word|
+          if /[[:cntrl:] #"$&'()*;<>\[\\\]\^`{|}]/.match(word)
+            raise ToolDefinitionError, "Illegal characters in name #{word.inspect}"
+          end
+        end
+      end
+
       def top_definition
         @top_priority ? @definitions[@top_priority] : nil
       end
@@ -573,32 +721,23 @@ module Toys
     # @private
     #
     class DelimiterHandler
-      ## @private
-      ALLOWED_DELIMITERS = %r{^[\./:]*$}.freeze
-      private_constant :ALLOWED_DELIMITERS
-
-      ## @private
       def initialize(extra_delimiters)
-        unless ALLOWED_DELIMITERS =~ extra_delimiters
+        unless %r{^[[:space:]./:]*$}.match?(extra_delimiters)
           raise ::ArgumentError, "Illegal delimiters in #{extra_delimiters.inspect}"
         end
         chars = ::Regexp.escape(extra_delimiters.chars.uniq.join)
-        @extra_delimiters = chars.empty? ? nil : ::Regexp.new("[#{chars}]")
+        @delimiters = ::Regexp.new("[[:space:]#{chars}]")
       end
 
-      ## @private
       def split_path(str)
-        @extra_delimiters ? str.split(@extra_delimiters) : [str]
+        str.split(@delimiters)
       end
 
-      ## @private
       def find_orig_prefix(args)
-        if @extra_delimiters
-          first_split = (args.first || "").split(@extra_delimiters)
-          if first_split.size > 1
-            args = first_split + args.slice(1..-1)
-            return [first_split, args]
-          end
+        first_split = (args.first || "").split(@delimiters)
+        if first_split.size > 1
+          args = first_split + args.slice(1..-1)
+          return [first_split, args]
         end
         orig_prefix = args.take_while { |arg| !arg.start_with?("-") }
         [orig_prefix, args]