puppet-strings 0.4.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3c5d45a8489fb4754aebb00abe3afb2e85b4e75c
+  data.tar.gz: cb074d7c812e50b1a1ffbbe42cafde1a909fb125
+SHA512:
+  metadata.gz: 5bc48f92940b474aeddff28cad23338216db262815dec6abfc275aaf39f7c458a9d2a4d160c5131898724310e4ef75fe657f597164b80ede62fd9066d616ec9e
+  data.tar.gz: 9ce5ee6be34c1727855f9c1f99bb2049146b3434ab4bc5b51579d698cd15e531b70ae3a466beb77b08599ff0439cbf6c2e077c4556cb3d01448e5dd5adaf0a20

data/lib/puppet-strings/rake_tasks.rb

@@ -0,0 +1,18 @@
+require 'rake'
+require 'rake/tasklib'
+require 'puppet/face'
+require 'puppet_x/puppetlabs/strings/util'
+
+namespace :strings do
+  desc 'Generate Puppet documentation with YARD.'
+  task :generate do
+    PuppetX::PuppetLabs::Strings::Util.generate([
+      {emit_json: 'strings.json'}
+    ])
+  end
+
+  desc 'Serve YARD documentation for modules.'
+  task :serve do
+    PuppetX::PuppetLabs::Strings::Util.serve
+  end
+end

data/lib/puppet/application/strings.rb

@@ -0,0 +1,4 @@
+require 'puppet/application/face_base'
+
+class Puppet::Application::Strings < Puppet::Application::FaceBase
+end

data/lib/puppet/face/strings.rb

@@ -0,0 +1,64 @@
+require 'puppet/face'
+require 'puppet_x/puppetlabs/strings/yard/tags/directives'
+
+Puppet::Face.define(:strings, '0.0.1') do
+  summary "Generate Puppet documentation with YARD."
+
+  # Ensures that the user has the needed features to use puppet strings
+  def check_required_features
+    unless Puppet.features.yard?
+      raise RuntimeError, "The 'yard' gem must be installed in order to use this face."
+    end
+
+    unless Puppet.features.rgen?
+      raise RuntimeError, "The 'rgen' gem must be installed in order to use this face."
+    end
+
+    if RUBY_VERSION.match(/^1\.8/)
+      raise RuntimeError, "This face requires Ruby 1.9 or greater."
+    end
+  end
+
+  action(:yardoc) do
+    default
+
+    option "--emit-json-stdout" do
+      summary "Print json representation of the documentation to stdout"
+    end
+    option "--emit-json FILE" do
+      summary "Write json representation of the documentation to FILE"
+    end
+
+    summary "Generate YARD documentation from files."
+    arguments "[manifest_file.pp ...]"
+
+    when_invoked do |*args|
+      check_required_features
+      require 'puppet_x/puppetlabs/strings/util'
+
+      PuppetX::PuppetLabs::Strings::Util.generate(args)
+
+      # Puppet prints the return value of the action. The return value of this
+      # action is that of the yardoc_actions invocation, which is the boolean
+      # "true". This clutters the statistics yard prints, so instead return the
+      # empty string. Note an extra newline will also be printed.
+      ""
+    end
+  end
+
+  # NOTE: Modeled after the `yard gems` command which builds doc indicies
+  # (.yardoc directories) for Ruby Gems. Currently lacks the fine-grained
+  # control over where these indicies are created and just dumps them in the
+  # module roots.
+
+  action(:server) do
+    summary "Serve YARD documentation for modules."
+
+    when_invoked do |*args|
+      check_required_features
+      require 'puppet_x/puppetlabs/strings/util'
+
+      PuppetX::PuppetLabs::Strings::Util.serve(args)
+    end
+  end
+end

data/lib/puppet/feature/rgen.rb

@@ -0,0 +1,3 @@
+require 'puppet/util/feature'
+
+Puppet.features.add(:rgen, :libs => ['rgen/metamodel_builder', 'rgen/ecore/ecore'])

data/lib/puppet/feature/yard.rb

@@ -0,0 +1,3 @@
+require 'puppet/util/feature'
+
+Puppet.features.add(:yard, :libs => ['yard'])

data/lib/puppet_x/puppetlabs/strings.rb

@@ -0,0 +1,64 @@
+require 'puppet'
+require 'puppet/pops'
+require 'puppet/util/docs'
+require 'yard'
+
+module PuppetX
+end
+
+# Nothing to see here except forward declarations.
+module PuppetX::PuppetLabs
+  module Strings
+    # This submodule contains bits that operate on the Pops module produced by
+    # the Future parser.
+    module Pops
+      require 'puppet_x/puppetlabs/strings/pops/yard_statement'
+      require 'puppet_x/puppetlabs/strings/pops/yard_transformer'
+    end
+
+    # This submodule contains bits that interface with the YARD plugin system.
+    module YARD
+      require 'puppet_x/puppetlabs/strings/yard/monkey_patches'
+      require 'puppet_x/puppetlabs/strings/yard/parser'
+
+      module Tags
+        require 'puppet_x/puppetlabs/strings/yard/tags/directives'
+      end
+
+      # This submodule contains code objects which are used to represent relevant
+      # aspects of puppet code in YARD's Registry
+      module CodeObjects
+        require 'puppet_x/puppetlabs/strings/yard/code_objects/puppet_namespace_object'
+        require 'puppet_x/puppetlabs/strings/yard/code_objects/method_object'
+        require 'puppet_x/puppetlabs/strings/yard/code_objects/defined_type_object'
+        require 'puppet_x/puppetlabs/strings/yard/code_objects/host_class_object'
+        require 'puppet_x/puppetlabs/strings/yard/code_objects/type_object'
+        require 'puppet_x/puppetlabs/strings/yard/code_objects/provider_object'
+      end
+
+      # This submodule contains handlers which are used to extract relevant data about
+      # puppet code from the ASTs produced by the Ruby and Puppet parsers
+      module Handlers
+        # This utility library contains some tools for working with Puppet docstrings
+        require 'puppet_x/puppetlabs/strings/yard/handlers/base'
+        require 'puppet_x/puppetlabs/strings/yard/handlers/defined_type_handler'
+        require 'puppet_x/puppetlabs/strings/yard/handlers/host_class_handler'
+        require 'puppet_x/puppetlabs/strings/yard/handlers/puppet_3x_function_handler'
+        require 'puppet_x/puppetlabs/strings/yard/handlers/puppet_4x_function_handler'
+        require 'puppet_x/puppetlabs/strings/yard/handlers/type_handler'
+        require 'puppet_x/puppetlabs/strings/yard/handlers/provider_handler'
+      end
+
+      ::YARD::Parser::SourceParser.register_parser_type(:puppet,
+        PuppetX::PuppetLabs::Strings::YARD::PuppetParser,
+        ['pp'])
+      ::YARD::Handlers::Processor.register_handler_namespace(:puppet,
+        PuppetX::PuppetLabs::Strings::YARD::Handlers)
+
+      # FIXME: Might not be the best idea to have the template code on the Ruby
+      # LOAD_PATH as the contents of this directory really aren't library code.
+      ::YARD::Templates::Engine.register_template_path(
+        File.join(File.dirname(__FILE__), 'strings', 'yard', 'templates'))
+    end
+  end
+end

data/lib/puppet_x/puppetlabs/strings/actions.rb

@@ -0,0 +1,92 @@
+require 'puppet_x/puppetlabs/strings'
+
+class PuppetX::PuppetLabs::Strings::Actions
+
+  # Creates a new instance of the Actions class by determining
+  # whether or not debug and backtrace arguments should be sent
+  # to YARD
+  def initialize(puppet_debug, puppet_backtrace)
+    @debug = puppet_debug
+    @backtrace = puppet_backtrace
+  end
+
+  # Holds the name of a module and the file path to its YARD index
+  ModuleIndex = Struct.new(:name, :index_path)
+
+
+  # Maps things like the Puppet `--debug` flag to YARD options.
+  def merge_puppet_args!(yard_args)
+    yard_args.unshift '--debug'     if @debug
+    yard_args.unshift '--backtrace' if @backtrace
+
+    yard_args
+  end
+
+  # Builds doc indices (.yardoc directories) for modules.
+  # Currently lacks the fine-grained control over where these
+  # indices are created and just dumps them in the module roots.
+  #
+  # @return [Array<Module>] the modules to be documented
+  #
+  # @param [Array<String>] module_names a list of the module source files
+  # @param [Array<String>] module_sourcefiles default list of module files
+  def index_documentation_for_modules(module_names, module_sourcefiles)
+    # NOTE: The return value of the `module` Face seems to have changed in
+    # 3.6.x. This part of the code will blow up if run under an earlier
+    # version of Puppet.
+    modules = Puppet::Face[:module, :current].list
+    module_list = modules[:modules_by_path].values.flatten
+
+    module_list.select! {|m| module_names.include? m.name} unless module_names.empty?
+
+    # Invoke `yardoc` with -n so that it doesn't generate any HTML output but
+    # does build a `.yardoc` index that other tools can generate output from.
+    yard_args = %w[--no-stats -n] + module_sourcefiles
+    merge_puppet_args!(yard_args)
+
+    module_list.each do |m|
+      Dir.chdir(m.path) do
+        YARD::CLI::Yardoc.run(*yard_args)
+
+        # Clear the global Registry so that objects from one module don't
+        # bleed into the next.
+        YARD::Registry.clear
+      end
+    end
+  end
+
+  # Extracts the needed information of the modules we're documenting
+  #
+  # @return [Array<ModuleIndex>] An array of representation of the modules
+  # to produce documentation for. Each ModuleIndex contains the module name
+  # and the path to its YARD index
+  #
+  # @param [Array<String>] module_list a list of the module source files
+  def generate_module_tuples(module_list)
+    module_list.map do |mod|
+      name = (mod.forge_name || mod.name).gsub('/', '-')
+      yard_index = File.join(mod.path, '.yardoc')
+
+      ModuleIndex.new(name, yard_index)
+    end
+  end
+
+  # Hands off the needed information to YARD so it may
+  # serve the documentation
+  #
+  # @param [Array<String>] yard_args a list of all the arguments to pass to YARD
+  def serve_documentation(*yard_args)
+    merge_puppet_args!(yard_args)
+    YARD::CLI::Server.run(*yard_args)
+  end
+
+  # Hands off the needed information to YARD so it may
+  # generate the documentation
+  #
+  # @param [Array<String>] yard_args a list of all the arguments to pass to YARD
+  def generate_documentation(*yard_args)
+    merge_puppet_args!(yard_args)
+    YARD::CLI::Yardoc.run(*yard_args)
+  end
+end
+

data/lib/puppet_x/puppetlabs/strings/pops/yard_statement.rb

@@ -0,0 +1,79 @@
+require 'ostruct'
+
+# An adapter class that conforms a Pops model instance + adapters to the
+# interface expected by YARD handlers.
+#
+# FIXME: Inhertiting from OpenStruct is a bit of a hack. It allows attributes
+# to be declared as needed but in the long run understandibility of the code
+# would be improved by having a concrete model.
+class PuppetX::PuppetLabs::Strings::Pops::YARDStatement < OpenStruct
+  attr_reader :pops_obj, :comments
+
+  def initialize(pops_obj)
+    # Initialize OpenStruct
+    super({})
+
+    unless pops_obj.is_a? Puppet::Pops::Model::PopsObject
+      raise ArgumentError, "A YARDStatement can only be initialized from a PopsObject. Got a: #{pops_obj.class}"
+    end
+
+    @pops_obj = pops_obj
+    @pos_adapter = Puppet::Pops::Adapters::SourcePosAdapter.adapt(@pops_obj)
+    # FIXME: Perhaps this should be a seperate adapter?
+    @comments = extract_comments
+  end
+
+  def type
+    pops_obj.class
+  end
+
+  def line
+    @line ||= @pos_adapter.line
+  end
+
+  def source
+    @source ||= @pos_adapter.extract_text
+  end
+
+  # FIXME: I don't know quite what these are supposed to do, but they show up
+  # quite often in the YARD handler code. Figure out whether they are
+  # necessary.
+  alias_method :show, :source
+  def comments_hash_flag; nil end
+  def comments_range; nil end
+
+  private
+  # TODO: This stuff should probably be part of a separate class/adapter.
+  COMMENT_PATTERN = /^\s*#.*\n/
+
+  def extract_comments
+    comments = []
+    program = pops_obj.eAllContainers.find {|c| c.is_a?(Puppet::Pops::Model::Program) }
+    # FIXME: Horribly inefficient. Multiple copies. Generator pattern would
+    # be much better.
+    source_text = program.source_text.lines.to_a
+
+    source_text.slice(0, line-1).reverse.each do |line|
+      if COMMENT_PATTERN.match(line)
+        # FIXME: The gsub trims comments, but is extremely optimistic: It
+        # assumes only one space separates the comment body from the
+        # comment character.
+        # NOTE: We cannot just trim any amount of whitespace as indentation
+        # is sometimes significant in markdown. We would need a real parser.
+
+        # Comments which begin with some whitespace, a hash and then some
+        # tabs and spaces should be scrubbed. Comments which just have a
+        # solitary hash then a newline should keep that newline since newlines
+        # are significant in markdown.
+        comments.unshift line.gsub(/^\s*#[\t ]/, '').gsub(/^\s*#\n/, "\n")
+      else
+        # No comment found on this line. We must be done piecing together a
+        # comment block.
+        break
+      end
+    end
+
+    # Stick everything back together.
+    comments.join
+  end
+end

data/lib/puppet_x/puppetlabs/strings/pops/yard_transformer.rb

@@ -0,0 +1,47 @@
+# Loosely based on the TreeDumper classes in Pops::Model. The responsibility of
+# this class is to walk a Pops::Model and output objects that can be consumed
+# by YARD handlers.
+#
+# @note Currently, this class only extracts node, host class and type
+#   definitions.
+class PuppetX::PuppetLabs::Strings::Pops::YARDTransformer
+  def initialize
+    @transform_visitor = Puppet::Pops::Visitor.new(self, 'transform')
+  end
+
+  def transform(o)
+    @transform_visitor.visit(o)
+  end
+
+  private
+
+  def transform_Factory(o)
+    transform(o.current)
+  end
+
+  def transform_Program(o)
+    o.definitions.map{|d| transform(d)}
+  end
+
+  # Extract comments from type definitions and class definitions. Wrap them
+  # into YARDStatement objects that provide an interface for YARD handlers.
+  def transform_NamedDefinition(o)
+    obj = PuppetX::PuppetLabs::Strings::Pops::YARDStatement.new(o)
+    obj.parameters = o.parameters.map do |p|
+      param_tuple = [transform(p)]
+      param_tuple << ( p.value.nil? ? nil : transform(p.value) )
+    end
+
+    obj
+  end
+
+  # Catch-all visitor.
+  def transform_Positioned(o)
+    PuppetX::PuppetLabs::Strings::Pops::YARDStatement.new(o)
+  end
+
+  # nil in... nil out!
+  def transform_NilClass(o)
+    nil
+  end
+end

data/lib/puppet_x/puppetlabs/strings/util.rb

@@ -0,0 +1,65 @@
+require 'puppet_x/puppetlabs/strings/actions'
+
+module PuppetX::PuppetLabs::Strings::Util
+  MODULE_SOURCEFILES = ['manifests/**/*.pp', 'lib/**/*.rb']
+
+  def self.generate(args = [])
+    yardoc_actions = PuppetX::PuppetLabs::Strings::Actions.new(Puppet[:debug], Puppet[:trace])
+
+    # The last element of the argument array should be the options hash.
+    # We don't have any options yet, so for now just pop the hash off and
+    # toss it.
+    #
+    # NOTE: The Puppet Face will throw 'unrecognized option' errors if any
+    # YARD options are passed to it. The best way to approach this problem is
+    # by using the `.yardopts` file. YARD will autoload any options placed in
+    # that file.
+    options = args.pop
+    YARD::Config.options = YARD::Config.options.merge(options) if options
+
+    # For now, assume the remaining positional args are a list of manifest
+    # and ruby files to parse.
+    yard_args = (args.empty? ? MODULE_SOURCEFILES : args)
+
+    # If json is going to be emitted to stdout, suppress statistics.
+    if options && options[:emit_json_stdout]
+      yard_args.push('--no-stats')
+    end
+
+    # This line monkeypatches yard's progress indicator so it doesn't write
+    # all over the terminal. This should definitely not be in real code, but
+    # it's very handy for debugging with pry
+    #class YARD::Logger; def progress(*args); end; end
+    YARD::Tags::Library.define_directive("puppet.type.param",
+      :with_types_and_name,
+      PuppetX::PuppetLabs::Strings::YARD::Tags::PuppetTypeParameterDirective)
+
+    yardoc_actions.generate_documentation(*yard_args)
+  end
+
+  def self.serve(args = [])
+    server_actions = PuppetX::PuppetLabs::Strings::Actions.new(Puppet[:debug], Puppet[:trace])
+
+    args.pop
+
+    module_names = args
+
+    # FIXME: This is pretty inefficient as it forcibly re-generates the YARD
+    # indicies each time the server is started. However, it ensures things are
+    # generated properly.
+    module_list = server_actions.index_documentation_for_modules(module_names, MODULE_SOURCEFILES)
+
+    module_tuples = server_actions.generate_module_tuples(module_list)
+
+    module_tuples.map! do |mod|
+      [mod[:name], mod[:index_path]]
+    end
+
+    # The `-m` flag means a list of name/path pairs will follow. The name is
+    # used as the module name and the path indicates which `.yardoc` index to
+    # generate documentation from.
+    yard_args = %w[-m -q] + module_tuples.flatten
+
+    server_actions.serve_documentation(*yard_args)
+  end
+end