puppet-strings 0.4.0

data/lib/puppet_x/puppetlabs/strings/yard/handlers/provider_handler.rb

@@ -0,0 +1,95 @@
+# Handles `dispatch` calls within a future parser function declaration. For
+# now, it just treats any docstring as an `@overlaod` tag and attaches the
+# overload to the parent function.
+class PuppetX::PuppetLabs::Strings::YARD::Handlers::PuppetProviderHandler < YARD::Handlers::Ruby::Base
+  include PuppetX::PuppetLabs::Strings::YARD::CodeObjects
+
+  handles :command_call, :call
+
+  process do
+    @heredoc_helper = HereDocHelper.new
+    # Puppet types always begin with:
+    # Puppet::Types.newtype...
+    # Therefore, we match the corresponding trees which look like this:
+    # s(:call,
+    #   s(:const_path_ref,
+    #     s(:var_ref, s(:const, "Puppet", ...), ...),
+    #   s(:const, "Type", ...),
+    # You think this is ugly? It's better than the alternative.
+    return unless statement.children.length > 2
+    first = statement.children.first.first
+    return unless (first.source == 'Puppet::Type') ||
+      (first.type == :var_ref &&
+      first.source == 'Type') &&
+      statement[2].source == 'provide'
+    i = statement.index { |s| YARD::Parser::Ruby::AstNode === s && s.type == :ident && s.source == 'provide' }
+    provider_name = statement[i+1].jump(:ident).source
+    type_name = statement.jump(:symbol).first.source
+
+    obj = ProviderObject.new(:root, "#{provider_name}_provider")
+
+    docstring = nil
+    features = []
+    commands = []
+    confines = {}
+    defaults = {}
+    do_block = statement.jump(:do_block)
+    do_block.traverse do |node|
+      if is_a_func_call_named?('desc', node)
+        content = node.jump(:tstring_content)
+        # if we found the string content extract its source
+        if content != node
+          # The docstring is either the source stripped of heredoc
+          # annotations or the raw source.
+          if @heredoc_helper.is_heredoc?(content.source)
+            docstring = @heredoc_helper.process_heredoc content.source
+          else
+            docstring = content.source
+          end
+        end
+      elsif is_a_func_call_named?('confine', node)
+        node.traverse do |s|
+          if s.type == :assoc
+            k = s.first.jump(:ident).source
+            v = s[1].first.source
+            confines[k] = v
+          end
+        end
+      elsif is_a_func_call_named?('has_feature', node)
+        list = node.jump :list
+        if list != nil && list != node
+          features += list.map { |s| s.source if YARD::Parser::Ruby::AstNode === s }.compact
+        end
+      elsif is_a_func_call_named?('commands', node)
+        assoc = node.jump(:assoc)
+        if assoc && assoc != node
+          ident = assoc.jump(:ident)
+          if ident && ident != assoc
+            commands << ident.source
+          end
+        end
+      elsif is_a_func_call_named?('defaultfor', node)
+        node.traverse do |s|
+          if s.type == :assoc
+            k = s.first.jump(:ident).source
+            v = s[1].first.source
+            defaults[k] = v
+          end
+        end
+      end
+    end
+    obj.features = features
+    obj.commands = commands
+    obj.confines = confines
+    obj.defaults = defaults
+    obj.type_name = type_name
+    obj.header_name = provider_name
+
+    register_docstring(obj, docstring, nil)
+    register obj
+  end
+
+  def is_a_func_call_named?(name, node)
+    (node.type == :fcall || node.type == :command || node.type == :vcall) && node.children.first.source == name
+  end
+end

data/lib/puppet_x/puppetlabs/strings/yard/handlers/puppet_3x_function_handler.rb

@@ -0,0 +1,54 @@
+require File.join(File.dirname(__FILE__),'./heredoc_helper')
+
+class PuppetX::PuppetLabs::Strings::YARD::Handlers::Puppet3xFunctionHandler < YARD::Handlers::Ruby::Base
+  include PuppetX::PuppetLabs::Strings::YARD::CodeObjects
+
+  handles method_call(:newfunction)
+
+  process do
+    @heredoc_helper = HereDocHelper.new
+    name, options = @heredoc_helper.process_parameters statement
+
+    obj = MethodObject.new(function_namespace, name)
+    obj[:puppet_3x_function] = true
+
+    register obj
+    if options['doc']
+      register_docstring(obj, options['doc'], nil)
+    end
+
+    # This has to be done _after_ register_docstring as all tags on the
+    # object are overwritten by tags parsed out of the docstring.
+    return_type = options['type']
+    return_type ||= 'statement' # Default for newfunction
+    obj.add_tag YARD::Tags::Tag.new(:return, '', return_type)
+
+    # FIXME: This is a hack that allows us to document the Puppet Core which
+    # uses `--no-transitive-tag api` and then only shows things explicitly
+    # tagged with `public` or `private` api. This is kind of insane and
+    # should be fixed upstream.
+    obj.add_tag YARD::Tags::Tag.new(:api, 'public')
+  end
+
+  private
+
+  # Returns a {PuppetNamespaceObject} for holding functions. Creates this
+  # object if necessary.
+  #
+  # @return [PuppetNamespaceObject]
+  def function_namespace
+    # NOTE: This tricky. If there is ever a Ruby class or module with the
+    # name ::Puppet3xFunctions, then there will be a clash. Hopefully the name
+    # is sufficiently uncommon.
+    obj = P(:root, 'Puppet3xFunctions')
+    if obj.is_a? Proxy
+      namespace_obj = PuppetNamespaceObject.new(:root, 'Puppet3xFunctions')
+      namespace_obj.add_tag YARD::Tags::Tag.new(:api, 'public')
+
+      register namespace_obj
+    end
+
+    obj
+  end
+
+end

data/lib/puppet_x/puppetlabs/strings/yard/handlers/puppet_4x_function_handler.rb

@@ -0,0 +1,234 @@
+# Handles `dispatch` calls within a future parser function declaration. For
+# now, it just treats any docstring as an `@overlaod` tag and attaches the
+# overload to the parent function.
+class PuppetX::PuppetLabs::Strings::YARD::Handlers::Puppet4xFunctionHandler < YARD::Handlers::Ruby::Base
+  include PuppetX::PuppetLabs::Strings::YARD::CodeObjects
+
+  handles method_call(:dispatch)
+
+  process do
+    return unless owner.is_a?(MethodObject) && owner['puppet_4x_function']
+    return unless statement.docstring
+
+    docstring = ::YARD::Docstring.new(statement.docstring, nil)
+
+    # FIXME: This does a wholesale copy of all possible tags. But, we're only
+    # interested in the @overload tag.
+    owner.add_tag *docstring.tags
+  end
+end
+
+class Puppet4xFunctionHandler < YARD::Handlers::Ruby::Base
+  include PuppetX::PuppetLabs::Strings::YARD::CodeObjects
+
+  handles method_call(:create_function)
+
+  # Given a command node which represents code like this:
+  # param  'Optional[Type]',   :value_type
+  # Extract the type name and type signature and return them as a array.
+  def extract_type_from_command command
+    return [] if command.children.length < 2 or command.children[1].children.length < 2
+    type_specifier = command.children[1]
+    # the parameter signature is the first child of the specifier and an
+    # identifier. Jump to the content inside the quotes and convert it to a
+    # string.
+    param_signature = type_specifier.children[0].jump(:tstring_content).source
+    # The parameter name is the second child of the specifier and a symbol.
+    # convert it to a string.
+    param_name_ident = type_specifier.jump :ident
+    return [] if param_name_ident == type_specifier
+    param_name = param_name_ident.source
+    [param_name, param_signature]
+  end
+
+  process do
+    name = process_parameters
+
+    method_arguments = []
+
+    # To attach the method parameters to the new code object, traverse the
+    # ruby AST until a node is found which defines a array of parameters.
+    # Then, traverse the children of the parameters, storing each identifier
+    # in the array of method arguments.
+    obj = MethodObject.new(function_namespace, name) do |o|
+    end
+
+    # The data structure for overload_signatures is an array of hashes. Each
+    # hash represents the arguments a single function dispatch (aka overload)
+    # can take.
+    # overload_signatures = [
+    #   { # First function dispatch arguments
+    #     # argument name, argument type
+    #     'arg0': 'Variant[String,Array[String]]',
+    #     'arg1': 'Optional[Type]'
+    #   },
+    #   { # Second function dispatch arguments
+    #     'arg0': 'Variant[String,Array[String]]',
+    #     'arg1': 'Optional[Type]',
+    #     'arg2': 'Any'
+    #   }
+    # ]
+    # Note that the order for arguments to a function doesn't actually matter
+    # because we allow users flexibility when listing their arguments in the
+    # comments.
+    overload_signatures = []
+    statement.traverse do |node|
+      # Find all of the dispatch methods
+      if node.type == :ident and node.source == 'dispatch'
+        command = node.parent
+        do_block = command.jump :do_block
+        # If the command doesn't have a do_block we can't extract type info
+        if do_block == command
+          next
+        end
+        signature = {}
+        # Iterate through each of the children of the do block and build
+        # tuples of parameter names and parameter type signatures
+        do_block.children.first.children.each do |child|
+          name, type = extract_type_from_command(child)
+          # This can happen if there is a function or something we aren't
+          # expecting.
+          if name != nil and type != nil
+            signature[name] = type
+          end
+        end
+        overload_signatures <<= signature
+      end
+    end
+
+    # If the overload_signatures list is empty because we couldn't find any
+    # dispatch blocks, then there must be one function named the same as the
+    # name of the function being created.
+    if overload_signatures.length == 0
+      statement.traverse do |node|
+        # Find the function definition with the same name as the puppet
+        # function being created.
+        if (node.type == :def and node.children.first.type == :ident and
+            node.children.first.source == obj.name.to_s)
+          signature = {}
+          # Find its parameters. If they don't exist, fine
+          params = node.jump :params
+          break if params == node
+          params.traverse do |param|
+            if param.type == :ident
+              # The parameters of Puppet functions with no defined dispatch are
+              # as though they are Any type.
+              signature[param[0]] =  'Any'
+            end
+          end
+          overload_signatures <<= signature
+          # Now that the parameters have been found, break out of the traversal
+          break
+        end
+      end
+    end
+
+    # Preserve this type information. We'll need it later when we look
+    # at the docstring.
+    obj.type_info = overload_signatures
+
+    # The yard docstring parser expects a list of lists, not a list of lists of
+    # lists.
+    obj.parameters = overload_signatures.map { |sig| sig.to_a }.flatten(1)
+
+    obj['puppet_4x_function'] = true
+
+    register obj
+
+    obj.add_tag YARD::Tags::Tag.new(:api, 'public')
+
+    blk = statement.block.children.first
+    parse_block(blk, :owner => obj)
+  end
+
+  private
+
+  # Returns a {PuppetNamespaceObject} for holding functions. Creates this
+  # object if necessary.
+  #
+  # @return [PuppetNamespaceObject]
+  def function_namespace
+    # NOTE: This tricky. If there is ever a Ruby class or module with the
+    # name ::Puppet4xFunctions, then there will be a clash. Hopefully the name
+    # is sufficiently uncommon.
+    obj = P(:root, 'Puppet4xFunctions')
+    if obj.is_a? Proxy
+      namespace_obj = PuppetNamespaceObject.new(:root, 'Puppet4xFunctions')
+
+      register namespace_obj
+      # FIXME: The docstring has to be cleared. Otherwise, the namespace
+      # object will be registered using the docstring of the
+      # `create_function` call that is currently being processed.
+      #
+      # Figure out how to properly register the namespace without using the
+      # function handler object.
+      register_docstring(namespace_obj, '', nil)
+      namespace_obj.add_tag YARD::Tags::Tag.new(:api, 'public')
+    end
+
+    obj
+  end
+
+  # NOTE: The following methods duplicate functionality from
+  # Puppet::Util::Reference and Puppet::Parser::Functions.functiondocs
+  #
+  # However, implementing this natively in YARD is a good test for the
+  # feasibility of extracting custom Ruby documentation. In the end, the
+  # existing approach taken by Puppet::Util::Reference may be the best due to
+  # the heavy use of metaprogramming in Types and Providers.
+
+  # Extracts the Puppet function name and options hash from the parsed
+  # definition.
+  #
+  # @return [(String, Hash{String => String})]
+  def process_parameters
+    # Passing `false` to parameters excludes the block param from the returned
+    # array.
+    name, _ = statement.parameters(false).compact
+
+    name = process_element(name)
+
+
+    name
+  end
+
+  # Sometimes the YARD parser returns Heredoc strings that start with `<-`
+  # instead of `<<-`.
+  HEREDOC_START = /^<?<-/
+
+    # Turns an entry in the method parameter array into a string.
+    #
+    # @param ele [YARD::Parser::Ruby::AstNode]
+    # @return [String]
+    def process_element(ele)
+      ele = ele.jump(:ident, :string_content, :tstring_content)
+
+      case ele.type
+      when :ident
+        ele.source
+      when :string_content, :tstring_content
+        source = ele.source
+        if HEREDOC_START.match(source)
+          process_heredoc(source)
+        else
+          source
+        end
+      end
+    end
+
+  # Cleans up and formats Heredoc contents parsed by YARD.
+  #
+  # @param source [String]
+  # @return [String]
+  def process_heredoc(source)
+    source = source.lines.to_a
+
+    # YARD adds a line of source context on either side of the Heredoc
+    # contents.
+    source.shift
+    source.pop
+
+    # This utility method normalizes indentation and trims whitespace.
+    Puppet::Util::Docs.scrub(source.join)
+  end
+end

data/lib/puppet_x/puppetlabs/strings/yard/handlers/type_handler.rb

@@ -0,0 +1,295 @@
+# Handles `dispatch` calls within a future parser function declaration. For
+# now, it just treats any docstring as an `@overlaod` tag and attaches the
+# overload to the parent function.
+class PuppetX::PuppetLabs::Strings::YARD::Handlers::PuppetTypeHandler < YARD::Handlers::Ruby::Base
+  include PuppetX::PuppetLabs::Strings::YARD::CodeObjects
+
+  handles :call
+
+  process do
+    @heredoc_helper = HereDocHelper.new
+    # Puppet types always begin with:
+    # Puppet::Types.newtype...
+    # Therefore, we match the corresponding trees which look like this:
+    # s(:call,
+    #   s(:const_path_ref,
+    #     s(:var_ref, s(:const, "Puppet", ...), ...),
+    #   s(:const, "Type", ...),
+    # You think this is ugly? It's better than the alternative.
+    return unless statement.children.length > 2
+    first = statement.children.first
+    return unless (first.type == :const_path_ref and
+      first.source == 'Puppet::Type') or
+      (first.type == :var_ref and
+      first.source == 'Type') and
+      statement.children[1].source == "newtype"
+
+    # Fetch the docstring for the types. The docstring is the string literal
+    # assigned to the @doc parameter or absent, like this:
+    # @doc "docstring goes here"
+    # We assume that docstrings nodes have the following shape in the source
+    # code:
+    # ...
+    # s(s(:assign,
+    #        s(:..., s(:ivar, "@doc", ...), ...),
+    #        s(:...,
+    #           s(:...,
+    #              s(:tstring_content,
+    #                 "Manages files, including their content, etc.", ...
+    # Initialize the docstring to nil, the default value if we don't find
+    # anything
+    docstring = nil
+    # Walk the tree searching for assignments
+    statement.traverse do |node|
+      if node.type == :assign
+        # Once we have found and assignment, jump to the first ivar
+        # (the l-value)
+        # If we can't find an ivar return the node.
+        ivar = node.jump(:ivar)
+        # If we found and  ivar and its source reads '@doc' then...
+        if ivar != node and ivar.source == '@doc'
+          # find the next string content
+          content = node.jump(:tstring_content)
+          # if we found the string content extract its source
+          if content != node
+            # The docstring is either the source stripped of heredoc
+            # annotations or the raw source.
+            if @heredoc_helper.is_heredoc? content.source
+              docstring = @heredoc_helper.process_heredoc content.source
+            else
+              docstring = content.source
+            end
+          end
+          # Since we found the @doc parameter (regardless of whether we
+          # successfully extracted its source), we're done.
+          break
+        # But if we didn't find the ivar loop around again.
+        else
+          next
+        end
+      end
+    end
+
+    # The types begin with:
+    # Puppet::Types.newtype(:symbol)
+    # Jump to the first identifier (':symbol') after the third argument
+    # ('(:symbol)') to the current statement
+    name = statement.children[2].jump(:ident).source
+    parameter_details = []
+    property_details = []
+    features = []
+    obj = TypeObject.new(:root, name)
+    obj.parameters = []
+
+    # Find the do block following the Type.
+    do_block = statement.jump(:do_block)
+    # traverse the do block's children searching for function calls whose
+    # identifier is newparam (we're calling the newparam function)
+    do_block.traverse do |node|
+      if is_param? node
+        # The first member of the parameter tuple is the parameter name.
+        # Find the second identifier node under the fcall tree. The first one
+        # is 'newparam', the second one is the function name.
+        # Get its source.
+        # The second parameter is nil because we cannot infer types for these
+        # functions. In fact, that's a silly thing to ask because ruby
+        # types were deprecated with puppet 4 at the same time the type
+        # system was created.
+
+        # Because of a ripper bug a symbol identifier is sometimes incorrectly parsed as a keyword.
+        # That is, the symbol `:true` will be represented as s(:symbol s(:kw, true...
+        param_name = node.children[1].jump(:ident)
+        if param_name == node.children[1]
+          param_name = node.children[1].jump(:kw)
+        end
+        param_name = param_name.source
+        obj.parameters << [param_name, nil]
+        parameter_details << {:name => param_name,
+          :desc => fetch_description(node), :exists? => true,
+          :puppet_type => true,
+          :default => fetch_default(node),
+          :namevar => is_namevar?(node, param_name, name),
+          :parameter => true,
+          :allowed_values => get_parameter_allowed_values(node),
+        }
+      elsif is_prop? node
+        # Because of a ripper bug a symbol identifier is sometimes incorrectly parsed as a keyword.
+        # That is, the symbol `:true` will be represented as s(:symbol s(:kw, true...
+        prop_name = node.children[1].jump(:ident)
+        if prop_name == node.children[1]
+          prop_name = node.children[1].jump(:kw)
+        end
+        prop_name = prop_name.source
+        property_details << {:name => prop_name,
+          :desc => fetch_description(node), :exists? => true,
+          :default => fetch_default(node),
+          :puppet_type => true,
+          :property => true,
+          :allowed_values => get_property_allowed_values(node),
+        }
+      elsif is_feature? node
+        features << get_feature(node)
+      elsif is_a_func_call_named? 'ensurable', node
+        # Someone could call the ensurable method and create an ensure 
+        # property. If that happens, they it will be documented twice. Serves
+        # them right.
+        property_details << {:name => 'ensure',
+          :desc => '', :exists? => true,
+          :default => nil,
+          :puppet_type => true,
+          :property => true,
+          :allowed_values => [],
+        }
+      end
+    end
+    obj.parameter_details = parameter_details
+    obj.property_details = property_details
+    obj.features = features
+    obj.header_name = name
+
+    register obj
+    # Register docstring after the object. If the object already has a
+    # docstring, or more likely has parameters documented with the type
+    # directive and an empty docstring, we want to override it with the
+    # docstring we found, assuming we found one.
+    register_docstring(obj, docstring, nil) if docstring
+  end
+
+
+  # See:
+  # https://docs.puppetlabs.com/guides/custom_types.html#namevar
+  # node should be a parameter
+  def is_namevar? node, param_name, type_name
+    # Option 1:
+    # Puppet::Type.newtype(:name) do
+    # ...
+    # newparam(:name) do
+    #   ...
+    # end
+    if type_name == param_name
+      return true
+    end
+    # Option 2:
+    # newparam(:path, :namevar => true) do
+    #   ...
+    # end
+    if node.children.length >= 2
+      node.traverse do |s|
+        if s.type == :assoc and s.jump(:ident).source == 'namevar' and s.jump(:kw).source == 'true'
+          return true
+        end
+      end
+    end
+    # Option 3:
+    # newparam(:path) do
+    #   isnamevar
+    #   ...
+    # end
+    do_block = node.jump(:do_block).traverse do |s|
+      if is_a_func_call_named? 'isnamevar', s
+        return true
+      end
+    end
+    # Crazy implementations of types may just call #isnamevar directly on the object.
+    # We don't handle this today.
+    return false
+  end
+
+  def is_param? node
+    is_a_func_call_named? 'newparam', node
+  end
+  def is_prop? node
+    is_a_func_call_named? 'newproperty', node
+  end
+
+  def is_feature? node
+    is_a_func_call_named? 'feature', node
+  end
+
+  def is_a_func_call_named? name, node
+    (node.type == :fcall or node.type == :command or node.type == :vcall) and node.children.first.source == name
+  end
+
+  def get_feature node
+    name = node[1].jump(:ident).source
+    desc = node[1].jump(:tstring_content).source
+    methods = []
+    if node[1].length == 4 and node.children[1][2].jump(:ident).source == 'methods'
+      arr = node[1][2].jump(:array)
+      if arr != node[1][2]
+        arr.traverse do |s|
+          if s.type == :ident
+            methods << s.source
+          end
+        end
+      end
+    end
+    {
+      :name => name,
+      :desc => desc,
+      :methods => methods != [] ? methods : nil,
+    }
+  end
+
+  def get_parameter_allowed_values node
+    vals = []
+    node.traverse do |s|
+      if is_a_func_call_named? 'newvalues', s
+        list = s.jump(:list)
+        if list != s
+          vals += list.map { |item| [item.source] if YARD::Parser::Ruby::AstNode === item }
+        end
+      end
+    end
+    vals.compact
+  end
+
+  # Calls to newvalue only apply to properties, according to Dan & Nan's
+  # "Puppet Types and Providers", page 30.
+  def get_property_allowed_values node
+    vals = get_parameter_allowed_values node
+    node.traverse do |s|
+      if is_a_func_call_named? 'newvalue', s
+        required_features = nil
+        s.traverse do |ss|
+          if ss.type == :assoc and ss[0].source == ':required_features'
+            required_features = ss[1].source
+          end
+        end
+        list = s.jump(:list)
+        if list != s
+          vals << [list[0].source, required_features].compact
+        end
+      end
+    end
+    vals
+  end
+
+  def fetch_default node
+    do_block = node.jump(:do_block)
+    do_block.traverse do |s|
+      if is_a_func_call_named? 'defaultto', s
+        return s[-1].source
+      end
+    end
+    nil
+  end
+
+  def fetch_description(fcall)
+    fcall.traverse do |node|
+      if is_a_func_call_named? 'desc', node
+        content = node.jump(:string_content)
+        if content != node
+          @heredoc_helper = HereDocHelper.new
+          if @heredoc_helper.is_heredoc? content.source
+            docstring = @heredoc_helper.process_heredoc content.source
+          else
+            docstring = content.source
+          end
+          return docstring
+        end
+      end
+    end
+    return nil
+  end
+end