toys-core 0.11.5 → 0.13.0

data/lib/toys/loader.rb

@@ -8,9 +8,6 @@ module Toys
   # appropriate tool given a set of command line arguments.
   #
   class Loader
-    # @private
-    BASE_PRIORITY = -999_999
-
     ##
     # Create a Loader
     #
@@ -44,18 +41,17 @@ module Toys
     # @param template_lookup [Toys::ModuleLookup] A lookup for
     #     well-known template classes. Defaults to an empty lookup.
     #
-    def initialize(
-      index_file_name: nil,
-      preload_dir_name: nil,
-      preload_file_name: nil,
-      data_dir_name: nil,
-      lib_dir_name: nil,
-      middleware_stack: [],
-      extra_delimiters: "",
-      mixin_lookup: nil,
-      middleware_lookup: nil,
-      template_lookup: nil
-    )
+    def initialize(index_file_name: nil,
+                   preload_dir_name: nil,
+                   preload_file_name: nil,
+                   data_dir_name: nil,
+                   lib_dir_name: nil,
+                   middleware_stack: [],
+                   extra_delimiters: "",
+                   mixin_lookup: nil,
+                   middleware_lookup: nil,
+                   template_lookup: nil,
+                   git_cache: nil)
       if index_file_name && ::File.extname(index_file_name) != ".rb"
         raise ::ArgumentError, "Illegal index file name #{index_file_name.inspect}"
       end
@@ -71,30 +67,82 @@ module Toys
       @loading_started = false
       @worklist = []
       @tool_data = {}
+      @roots_by_priority = {}
       @max_priority = @min_priority = 0
-      @stop_priority = BASE_PRIORITY
+      @stop_priority = -999_999
       @min_loaded_priority = 999_999
       @middleware_stack = Middleware.stack(middleware_stack)
       @delimiter_handler = DelimiterHandler.new(extra_delimiters)
-      get_tool([], BASE_PRIORITY)
+      @git_cache = git_cache
+      get_tool([], -999_999)
     end
 
     ##
     # Add a configuration file/directory to the loader.
     #
-    # @param paths [String,Array<String>] One or more paths to add.
+    # @param path [String] A single path to add.
     # @param high_priority [Boolean] If true, add this path at the top of the
     #     priority list. Defaults to false, indicating the new path should be
     #     at the bottom of the priority list.
+    # @param source_name [String] A custom name for the root source. Optional.
+    # @param context_directory [String,nil,:path,:parent] The context directory
+    #     for tools loaded from this path. You can pass a directory path as a
+    #     string, `:path` to denote the given path, `:parent` to denote the
+    #     given path's parent directory, or `nil` to denote no context.
+    #     Defaults to `:parent`.
+    # @return [self]
+    #
+    def add_path(path,
+                 high_priority: false,
+                 source_name: nil,
+                 context_directory: :parent)
+      @mutex.synchronize do
+        raise "Cannot add a path after tool loading has started" if @loading_started
+        priority = high_priority ? (@max_priority += 1) : (@min_priority -= 1)
+        source = SourceInfo.create_path_root(path, priority,
+                                             context_directory: context_directory,
+                                             data_dir_name: @data_dir_name,
+                                             lib_dir_name: @lib_dir_name,
+                                             source_name: source_name)
+        @roots_by_priority[priority] = source
+        @worklist << [source, [], priority]
+      end
+      self
+    end
+
+    ##
+    # Add a set of configuration files/directories from a common directory to
+    # the loader. The set of paths will be added at the same priority level and
+    # will share a root.
+    #
+    # @param root_path [String] A root path to be seen as the root source. This
+    #     should generally be a directory containing the paths to add.
+    # @param relative_paths [String,Array<String>] One or more paths to add, as
+    #     relative paths from the common root.
+    # @param high_priority [Boolean] If true, add the paths at the top of the
+    #     priority list. Defaults to false, indicating the new paths should be
+    #     at the bottom of the priority list.
+    # @param context_directory [String,nil,:path,:parent] The context directory
+    #     for tools loaded from this path. You can pass a directory path as a
+    #     string, `:path` to denote the given root path, `:parent` to denote
+    #     the given root path's parent directory, or `nil` to denote no context.
+    #     Defaults to `:path`.
     # @return [self]
     #
-    def add_path(paths, high_priority: false)
-      paths = Array(paths)
+    def add_path_set(root_path, relative_paths,
+                     high_priority: false,
+                     context_directory: :path)
+      relative_paths = Array(relative_paths)
       @mutex.synchronize do
         raise "Cannot add a path after tool loading has started" if @loading_started
         priority = high_priority ? (@max_priority += 1) : (@min_priority -= 1)
-        paths.each do |path|
-          source = SourceInfo.create_path_root(path, @data_dir_name, @lib_dir_name)
+        root_source = SourceInfo.create_path_root(root_path, priority,
+                                                  context_directory: context_directory,
+                                                  data_dir_name: @data_dir_name,
+                                                  lib_dir_name: @lib_dir_name)
+        @roots_by_priority[priority] = root_source
+        relative_paths.each do |path, individual_name|
+          source = root_source.relative_child(path, source_name: individual_name)
           @worklist << [source, [], priority]
         end
       end
@@ -107,19 +155,65 @@ module Toys
     # @param high_priority [Boolean] If true, add this block at the top of the
     #     priority list. Defaults to false, indicating the block should be at
     #     the bottom of the priority list.
-    # @param name [String] The source name that will be shown in documentation
-    #     for tools defined in this block. If omitted, a default unique string
-    #     will be generated.
+    # @param source_name [String] The source name that will be shown in
+    #     documentation for tools defined in this block. If omitted, a default
+    #     unique string will be generated.
     # @param block [Proc] The block of configuration, executed in the context
     #     of the tool DSL {Toys::DSL::Tool}.
+    # @param context_directory [String,nil] The context directory for tools
+    #     loaded from this block. You can pass a directory path as a string, or
+    #     `nil` to denote no context. Defaults to `nil`.
     # @return [self]
     #
-    def add_block(high_priority: false, name: nil, &block)
-      name ||= "(Code block #{block.object_id})"
+    def add_block(high_priority: false,
+                  source_name: nil,
+                  context_directory: nil,
+                  &block)
       @mutex.synchronize do
         raise "Cannot add a block after tool loading has started" if @loading_started
         priority = high_priority ? (@max_priority += 1) : (@min_priority -= 1)
-        source = SourceInfo.create_proc_root(block, name, @data_dir_name, @lib_dir_name)
+        source = SourceInfo.create_proc_root(block, priority,
+                                             context_directory: context_directory,
+                                             source_name: source_name,
+                                             data_dir_name: @data_dir_name,
+                                             lib_dir_name: @lib_dir_name)
+        @roots_by_priority[priority] = source
+        @worklist << [source, [], priority]
+      end
+      self
+    end
+
+    ##
+    # Add a configuration git source to the loader.
+    #
+    # @param git_remote [String] The git repo URL
+    # @param git_path [String] The path to the relevant file or directory in
+    #     the repo. Specify the empty string to use the entire repo.
+    # @param git_commit [String] The git ref (i.e. SHA, tag, or branch name)
+    # @param high_priority [Boolean] If true, add this path at the top of the
+    #     priority list. Defaults to false, indicating the new path should be
+    #     at the bottom of the priority list.
+    # @param update [Boolean] If the commit is not a SHA, pulls any updates
+    #     from the remote. Defaults to false, which uses a local cache and does
+    #     not update if the commit has been fetched previously.
+    # @param context_directory [String,nil] The context directory for tools
+    #     loaded from this source. You can pass a directory path as a string,
+    #     or `nil` to denote no context. Defaults to `nil`.
+    # @return [self]
+    #
+    def add_git(git_remote, git_path, git_commit,
+                high_priority: false,
+                update: false,
+                context_directory: nil)
+      @mutex.synchronize do
+        raise "Cannot add a git source after tool loading has started" if @loading_started
+        priority = high_priority ? (@max_priority += 1) : (@min_priority -= 1)
+        path = git_cache.get(git_remote, path: git_path, commit: git_commit, update: update)
+        source = SourceInfo.create_git_root(git_remote, git_path, git_commit, path, priority,
+                                            context_directory: context_directory,
+                                            data_dir_name: @data_dir_name,
+                                            lib_dir_name: @lib_dir_name)
+        @roots_by_priority[priority] = source
         @worklist << [source, [], priority]
       end
       self
@@ -136,7 +230,7 @@ module Toys
     # that are not part of the tool name and should be passed as tool args.
     #
     # @param args [Array<String>] Command line arguments
-    # @return [Array(Toys::Tool,Array<String>)]
+    # @return [Array(Toys::ToolDefinition,Array<String>)]
     #
     def lookup(args)
       orig_prefix, args = @delimiter_handler.find_orig_prefix(args)
@@ -157,13 +251,13 @@ module Toys
     # the given name, returns `nil`.
     #
     # @param words [Array<String>] The tool name
-    # @return [Toys::Tool] if the tool was found
+    # @return [Toys::ToolDefinition] if the tool was found
     # @return [nil] if no such tool exists
     #
     def lookup_specific(words)
       words = @delimiter_handler.split_path(words.first) if words.size == 1
       load_for_prefix(words)
-      tool = get_tool_data(words).cur_definition
+      tool = get_tool_data(words, false)&.cur_definition
       finish_definitions_in_tree(words) if tool
       tool
     end
@@ -177,7 +271,7 @@ module Toys
     #     rather than just the immediate children (the default)
     # @param include_hidden [Boolean] If true, include hidden subtools,
     #     e.g. names beginning with underscores.
-    # @return [Array<Toys::Tool>] An array of subtools.
+    # @return [Array<Toys::ToolDefinition>] An array of subtools.
     #
     def list_subtools(words, recursive: false, include_hidden: false)
       load_for_prefix(words)
@@ -234,8 +328,8 @@ module Toys
     #
     # @private
     #
-    def get_tool(words, priority)
-      get_tool_data(words).get_tool(priority, self)
+    def get_tool(words, priority, tool_class = nil)
+      get_tool_data(words, true).get_tool(priority, self, tool_class)
     end
 
     ##
@@ -248,7 +342,7 @@ module Toys
     # @private
     #
     def activate_tool(words, priority)
-      get_tool_data(words).activate_tool(priority, self)
+      get_tool_data(words, true).activate_tool(priority, self)
     end
 
     ##
@@ -267,10 +361,11 @@ module Toys
     #
     # @private
     #
-    def build_tool(words, priority)
+    def build_tool(words, priority, tool_class = nil)
       parent = words.empty? ? nil : get_tool(words.slice(0..-2), priority)
       middleware_stack = parent ? parent.subtool_middleware_stack : @middleware_stack
-      Tool.new(self, parent, words, priority, middleware_stack, @middleware_lookup)
+      ToolDefinition.new(parent, words, priority, @roots_by_priority[priority],
+                         middleware_stack, @middleware_lookup, tool_class)
     end
 
     ##
@@ -335,12 +430,31 @@ module Toys
     # @private
     #
     def load_path(parent_source, path, words, remaining_words, priority)
+      if parent_source.git_remote
+        raise LoaderError,
+              "Git source #{parent_source.source_name} tried to load from the local file system"
+      end
       source = parent_source.absolute_child(path)
       @mutex.synchronize do
         load_validated_path(source, words, remaining_words, priority)
       end
     end
 
+    ##
+    # Load configuration from the given git remote. This is called from the
+    # `load_git` directive in the DSL.
+    #
+    # @private
+    #
+    def load_git(parent_source, git_remote, git_path, git_commit, words, remaining_words, priority,
+                 update: false)
+      path = git_cache.get(git_remote, path: git_path, commit: git_commit, update: update)
+      source = parent_source.git_child(git_remote, git_path, git_commit, path)
+      @mutex.synchronize do
+        load_validated_path(source, words, remaining_words, priority)
+      end
+    end
+
     ##
     # Load a subtool block. Called from the `tool` directive in the DSL.
     #
@@ -353,6 +467,18 @@ module Toys
       end
     end
 
+    ##
+    # Get a GitCache.
+    #
+    # @private
+    #
+    def git_cache
+      @git_cache ||= begin
+        require "toys/utils/git_cache"
+        Utils::GitCache.new
+      end
+    end
+
     ##
     # Determine the next setting for remaining_words, given a word.
     #
@@ -368,6 +494,120 @@ module Toys
       end
     end
 
+    ##
+    # Tool data
+    #
+    # @private
+    #
+    class ToolData
+      ##
+      # @private
+      #
+      def initialize(words)
+        @words = validate_words(words)
+        @definitions = {}
+        @top_priority = @active_priority = nil
+        @mutex = ::Monitor.new
+      end
+
+      ##
+      # @private
+      #
+      def cur_definition
+        @mutex.synchronize { active_definition || top_definition }
+      end
+
+      ##
+      # @private
+      #
+      def empty?
+        @definitions.empty?
+      end
+
+      ##
+      # @private
+      #
+      def get_tool(priority, loader, tool_class = nil)
+        @mutex.synchronize do
+          if @top_priority.nil? || @top_priority < priority
+            @top_priority = priority
+          end
+          if tool_class && @definitions.include?(priority)
+            raise ToolDefinitionError, "Tool already defined for #{@words.inspect}"
+          end
+          @definitions[priority] ||= loader.build_tool(@words, priority, tool_class)
+        end
+      end
+
+      ##
+      # @private
+      #
+      def activate_tool(priority, loader)
+        @mutex.synchronize do
+          return active_definition if @active_priority == priority
+          return nil if @active_priority && @active_priority > priority
+          @active_priority = priority
+          get_tool(priority, loader)
+        end
+      end
+
+      private
+
+      def validate_words(words)
+        words.each do |word|
+          if /[[:cntrl:] #"$&'()*;<>\[\\\]\^`{|}]/.match(word)
+            raise ToolDefinitionError, "Illegal characters in name #{word.inspect}"
+          end
+        end
+      end
+
+      def top_definition
+        @top_priority ? @definitions[@top_priority] : nil
+      end
+
+      def active_definition
+        @active_priority ? @definitions[@active_priority] : nil
+      end
+    end
+
+    ##
+    # An object that handles name delimiting.
+    #
+    # @private
+    #
+    class DelimiterHandler
+      ##
+      # @private
+      #
+      def initialize(extra_delimiters)
+        unless %r{^[[:space:]./:]*$}.match?(extra_delimiters)
+          raise ::ArgumentError, "Illegal delimiters in #{extra_delimiters.inspect}"
+        end
+        chars = ::Regexp.escape(extra_delimiters.chars.uniq.join)
+        @delimiters = ::Regexp.new("[[:space:]#{chars}]")
+      end
+
+      ##
+      # @private
+      #
+      def split_path(str)
+        str.split(@delimiters)
+      end
+
+      ##
+      # @private
+      #
+      def find_orig_prefix(args)
+        first_split = (args.first || "").split(@delimiters)
+        if first_split.size > 1
+          args = first_split + args.slice(1..-1)
+          return [first_split, args]
+        end
+        orig_prefix = args.take_while { |arg| !arg.start_with?("-") }
+        [orig_prefix, args]
+      end
+    end
+
     private
 
     def all_cur_definitions
@@ -381,8 +621,10 @@ module Toys
       result
     end
 
-    def get_tool_data(words)
-      @mutex.synchronize { @tool_data[words] ||= ToolData.new(words) }
+    def get_tool_data(words, create)
+      @mutex.synchronize do
+        create ? (@tool_data[words] ||= ToolData.new(words)) : @tool_data[words]
+      end
     end
 
     ##
@@ -403,7 +645,7 @@ module Toys
       if remaining_words
         update_min_loaded_priority(priority)
         tool_class = get_tool(words, priority).tool_class
-        DSL::Tool.prepare(tool_class, remaining_words, source) do
+        DSL::Internal.prepare(tool_class, words, priority, remaining_words, source, self) do
           ContextualError.capture("Error while loading Toys config!") do
             tool_class.class_eval(&source.source_proc)
           end
@@ -425,7 +667,7 @@ module Toys
       if source.source_type == :file
         update_min_loaded_priority(priority)
         tool_class = get_tool(words, priority).tool_class
-        InputFile.evaluate(tool_class, remaining_words, source)
+        InputFile.evaluate(tool_class, words, priority, remaining_words, source, self)
       else
         do_preload(source.source_path)
         load_index_in(source, words, remaining_words, priority)
@@ -467,16 +709,20 @@ module Toys
       if @preload_dir_name
         preload_dir = ::File.join(path, @preload_dir_name)
         if ::File.directory?(preload_dir) && ::File.readable?(preload_dir)
-          ::Dir.entries(preload_dir).each do |child|
-            next unless ::File.extname(child) == ".rb"
-            preload_file = ::File.join(preload_dir, child)
-            next if !::File.file?(preload_file) || !::File.readable?(preload_file)
-            require preload_file
-          end
+          preload_dir_contents(preload_dir)
         end
       end
     end
 
+    def preload_dir_contents(preload_dir)
+      ::Dir.entries(preload_dir).each do |child|
+        next unless ::File.extname(child) == ".rb"
+        preload_file = ::File.join(preload_dir, child)
+        next if !::File.file?(preload_file) || !::File.readable?(preload_file)
+        require preload_file
+      end
+    end
+
     def sort_tools_by_name(tools)
       tools.sort! do |a, b|
         a = a.full_name
@@ -511,98 +757,5 @@ module Toys
         index += 1
       end
     end
-
-    ##
-    # Tool data
-    #
-    # @private
-    #
-    class ToolData
-      # @private
-      def initialize(words)
-        @words = words
-        @definitions = {}
-        @top_priority = @active_priority = nil
-        @mutex = ::Monitor.new
-      end
-
-      # @private
-      def cur_definition
-        @mutex.synchronize { active_definition || top_definition }
-      end
-
-      # @private
-      def empty?
-        @definitions.empty?
-      end
-
-      # @private
-      def get_tool(priority, loader)
-        @mutex.synchronize do
-          if @top_priority.nil? || @top_priority < priority
-            @top_priority = priority
-          end
-          @definitions[priority] ||= loader.build_tool(@words, priority)
-        end
-      end
-
-      # @private
-      def activate_tool(priority, loader)
-        @mutex.synchronize do
-          return active_definition if @active_priority == priority
-          return nil if @active_priority && @active_priority > priority
-          @active_priority = priority
-          get_tool(priority, loader)
-        end
-      end
-
-      private
-
-      def top_definition
-        @top_priority ? @definitions[@top_priority] : nil
-      end
-
-      def active_definition
-        @active_priority ? @definitions[@active_priority] : nil
-      end
-    end
-
-    ##
-    # An object that handles name delimiting.
-    #
-    # @private
-    #
-    class DelimiterHandler
-      ## @private
-      ALLOWED_DELIMITERS = %r{^[\./:]*$}.freeze
-      private_constant :ALLOWED_DELIMITERS
-
-      ## @private
-      def initialize(extra_delimiters)
-        unless ALLOWED_DELIMITERS =~ extra_delimiters
-          raise ::ArgumentError, "Illegal delimiters in #{extra_delimiters.inspect}"
-        end
-        chars = ::Regexp.escape(extra_delimiters.chars.uniq.join)
-        @extra_delimiters = chars.empty? ? nil : ::Regexp.new("[#{chars}]")
-      end
-
-      ## @private
-      def split_path(str)
-        @extra_delimiters ? str.split(@extra_delimiters) : [str]
-      end
-
-      ## @private
-      def find_orig_prefix(args)
-        if @extra_delimiters
-          first_split = (args.first || "").split(@extra_delimiters)
-          if first_split.size > 1
-            args = first_split + args.slice(1..-1)
-            return [first_split, args]
-          end
-        end
-        orig_prefix = args.take_while { |arg| !arg.start_with?("-") }
-        [orig_prefix, args]
-      end
-    end
   end
 end

data/lib/toys/middleware.rb

@@ -37,7 +37,7 @@ module Toys
     # This basic implementation does nothing and simply yields to the next
     # middleware.
     #
-    # @param tool [Toys::Tool] The tool definition to modify.
+    # @param tool [Toys::ToolDefinition] The tool definition to modify.
     # @param loader [Toys::Loader] The loader that loaded this tool.
     # @return [void]
     #
@@ -140,7 +140,9 @@ module Toys
         end
       end
 
-      ## @private
+      ##
+      # @private
+      #
       def spec_from_array(array)
         middleware = array.first
         if !middleware.is_a?(::String) && !middleware.is_a?(::Symbol) && !middleware.is_a?(::Class)
@@ -238,16 +240,12 @@ module Toys
       #
       attr_reader :block
 
-      ## @private
-      def initialize(object, name, args, kwargs, block)
-        @object = object
-        @name = name
-        @args = args
-        @kwargs = kwargs
-        @block = block
-      end
-
-      ## @private
+      ##
+      # Equality check
+      #
+      # @param other [Object]
+      # @return [Boolean]
+      #
       def ==(other)
         other.is_a?(Spec) &&
           object.eql?(other.object) &&
@@ -258,10 +256,25 @@ module Toys
       end
       alias eql? ==
 
-      ## @private
+      ##
+      # Return the hash code
+      #
+      # @return [Integer]
+      #
       def hash
         [object, name, args, kwargs, block].hash
       end
+
+      ##
+      # @private
+      #
+      def initialize(object, name, args, kwargs, block)
+        @object = object
+        @name = name
+        @args = args
+        @kwargs = kwargs
+        @block = block
+      end
     end
 
     ##
@@ -319,14 +332,12 @@ module Toys
         (@pre_specs + @default_specs + @post_specs).map { |spec| spec.build(middleware_lookup) }
       end
 
-      ## @private
-      def initialize(default_specs: nil, pre_specs: nil, post_specs: nil)
-        @pre_specs = pre_specs || []
-        @post_specs = post_specs || []
-        @default_specs = default_specs || []
-      end
-
-      ## @private
+      ##
+      # Equality check
+      #
+      # @param other [Object]
+      # @return [Boolean]
+      #
       def ==(other)
         other.is_a?(Stack) &&
           pre_specs.eql?(other.pre_specs) &&
@@ -335,10 +346,23 @@ module Toys
       end
       alias eql? ==
 
-      ## @private
+      ##
+      # Return the hash code
+      #
+      # @return [Integer]
+      #
       def hash
         [@pre_specs, @default_specs, @post_specs].hash
       end
+
+      ##
+      # @private
+      #
+      def initialize(default_specs: nil, pre_specs: nil, post_specs: nil)
+        @pre_specs = pre_specs || []
+        @post_specs = post_specs || []
+        @default_specs = default_specs || []
+      end
     end
   end
 end