ascii_binder 0.1.9 → 0.1.10

data/lib/ascii_binder/distro_map.rb

@@ -0,0 +1,67 @@
+require 'ascii_binder/distro'
+require 'trollop'
+require 'yaml'
+
+module AsciiBinder
+  class DistroMap
+    def initialize(distro_map_filepath)
+      @distro_yaml = YAML.load_file(distro_map_filepath)
+      @distro_map  = {}
+      @distro_yaml.each do |distro_key,distro_config|
+        if @distro_map.has_key?(distro_key)
+          Trollop::die "Error parsing '#{distro_map_filepath}': distro key '#{distro_key}' is used more than once."
+        end
+        distro = AsciiBinder::Distro.new(distro_map_filepath,distro_key,distro_config)
+        @distro_map[distro_key] = distro
+      end
+    end
+
+    def get_distro(distro_key)
+      unless @distro_map.has_key?(distro_key)
+        Trollop::die "Distro key '#{distro_key}' does not exist"
+      end
+      @distro_map[distro_key]
+    end
+
+    def include_distro_key?(distro_key)
+      @distro_map.has_key?(distro_key)
+    end
+
+    def distro_keys
+      @distro_map.keys
+    end
+
+    def distros
+      @distro_map.values
+    end
+
+    def distro_branches(distro_key='')
+      if distro_key == ''
+        branch_list = []
+        distros.each do |distro|
+          branch_list.concat(distro.branch_ids)
+        end
+        return branch_list.uniq
+      else
+        return get_distro(distro_key).branch_ids
+      end
+    end
+
+    def is_valid?
+      @distro_map.values.each do |distro|
+        next if distro.is_valid?
+        return false
+      end
+      return true
+    end
+
+    def errors
+      errors = []
+      @distro_map.values.each do |distro|
+        next if distro.is_valid?
+        errors << distro.errors
+      end
+      return errors
+    end
+  end
+end

data/lib/ascii_binder/engine.rb

@@ -0,0 +1,581 @@
+require 'ascii_binder/distro_branch'
+require 'ascii_binder/distro_map'
+require 'ascii_binder/helpers'
+require 'ascii_binder/site_map'
+require 'ascii_binder/template_renderer'
+require 'ascii_binder/topic_map'
+require 'asciidoctor'
+require 'asciidoctor/cli'
+require 'asciidoctor-diagram'
+require 'fileutils'
+require 'find'
+require 'git'
+require 'logger'
+require 'pandoc-ruby'
+require 'pathname'
+require 'sitemap_generator'
+require 'trollop'
+require 'yaml'
+
+include AsciiBinder::Helpers
+
+module AsciiBinder
+  module Engine
+
+    def build_date
+      Time.now.utc
+    end
+
+    def notice(hey,message,newline = false)
+      # TODO: (maybe) redirect everything to stderr
+      if newline
+        puts "\n"
+      end
+      puts "#{hey}: #{message}"
+    end
+
+    def warning(message,newline = false)
+      notice("WARNING",message,newline)
+    end
+
+    def nl_warning(message)
+      warning(message,true)
+    end
+
+    def git
+      @git ||= Git.open(git_root_dir)
+    end
+
+    def git_checkout branch_name
+      target_branch = git.branches.local.select{ |b| b.name == branch_name }[0]
+      if not target_branch.nil? and not target_branch.current
+        target_branch.checkout
+      end
+    end
+
+    def git_stash_all
+      # See if there are any changes in need of stashing
+      @stash_needed = `cd #{git_root_dir} && git status --porcelain` !~ /^\s*$/
+      if @stash_needed
+        puts "\nNOTICE: Stashing uncommited changes and files in working branch."
+        `cd #{docs_root_dir} && git stash -u`
+      end
+    end
+
+    def git_apply_and_drop
+      return unless @stash_needed
+      puts "\nNOTE: Re-applying uncommitted changes and files to working branch."
+      if system("cd #{docs_root_dir} && git stash pop")
+        puts "NOTE: Stash application successful."
+      else
+        puts "ERROR: Could not apply stashed code. Run `git stash apply` manually."
+      end
+      @stash_needed = false
+    end
+
+    # Returns the local git branches; current branch is always first
+    def local_branches
+      @local_branches ||= begin
+        branches = []
+        if not git.branches.local.empty?
+          branches << git.branches.local.select{ |b| b.current }[0].name
+          branches << git.branches.local.select{ |b| not b.current }.map{ |b| b.name }
+        end
+        branches.flatten
+      end
+    end
+
+    def working_branch
+      @working_branch ||= local_branches[0]
+    end
+
+    def dir_empty?(dir)
+      Dir.entries(dir).select{ |f| not f.start_with?('.') }.empty?
+    end
+
+
+    # Protip: Don't cache these! The topic map needs to be reread every time we change branches.
+    def topic_map_file
+      topic_file = TOPIC_MAP_FILENAME
+      unless File.exist?(File.join(docs_root_dir,topic_file))
+        # The new filename '_topic_map.yml' couldn't be found;
+        # switch to the old one and warn the user.
+        topic_file = BUILD_FILENAME
+        unless File.exist?(File.join(docs_root_dir,topic_file))
+          # Critical error - no topic map file at all.
+          Trollop::die "Could not find any topic map file ('#{TOPIC_MAP_FILENAME}' or '#{BUILD_FILENAME}') at #{docs_root_dir} in branch '#{git.branch}'"
+        end
+        warning "'#{BUILD_FILENAME}' is a deprecated filename. Rename this to '#{TOPIC_MAP_FILENAME}'."
+      end
+      topic_file
+    end
+
+    def topic_map
+      topic_map = AsciiBinder::TopicMap.new(topic_map_file,distro_map.distro_keys)
+      unless topic_map.is_valid?
+        errors = topic_map.errors
+        Trollop::die "The topic map file at '#{topic_map_file}' contains the following errors:\n- " + errors.join("\n- ") + "\n"
+      end
+      return topic_map
+    end
+
+    def create_new_repo
+      gem_template_dir = File.join(gem_root_dir,"templates")
+
+      # Create the new repo dir
+      FileUtils.mkdir_p(docs_root_dir)
+
+      # Copy the basic repo content into the new repo dir
+      Find.find(gem_template_dir).each do |path|
+        next if path == gem_template_dir
+        src_path = Pathname.new(path)
+        tgt_path = src_path.sub(gem_template_dir,docs_root_dir)
+        if src_path.directory?
+          FileUtils.mkdir_p(tgt_path.to_s)
+        else
+          FileUtils.cp src_path.to_s, tgt_path.to_s
+        end
+      end
+
+      # Initialize the git repo
+      Git.init(docs_root_dir)
+    end
+
+    def find_topic_files
+      file_list = []
+      Find.find(docs_root_dir).each do |path|
+        # Only consider .adoc files and ignore README, and anything in
+        # directories whose names begin with 'old' or '_' (underscore)
+        next if path.nil? or not path =~ /.*\.adoc/ or path =~ /README/ or path =~ /\/old\// or path =~ /\/_/
+        src_path = Pathname.new(path).sub(docs_root_dir,'').to_s
+        next if src_path.split('/').length < 3
+        file_list << src_path
+      end
+      file_list.map{ |path|
+        parts = path.split('/').slice(1..-1);
+        parts.slice(0..-2).join('/') + '/' + parts[-1].split('.')[0]
+      }
+    end
+
+    def remove_found_topic_files(branch,branch_topic_map,branch_topic_files)
+      nonexistent_topics = []
+      branch_topic_map.filepaths.each do |topic_map_filepath|
+        result = branch_topic_files.delete(topic_map_filepath)
+        if result.nil?
+          nonexistent_topics << topic_map_filepath
+        end
+      end
+      if nonexistent_topics.length > 0
+        nl_warning "The #{topic_map_file} file on branch '#{branch}' references nonexistent topics:\n" + nonexistent_topics.map{ |topic| "- #{topic}" }.join("\n")
+      end
+    end
+
+    def distro_map
+      @distro_map ||= begin
+        distro_map_file = File.join(docs_root_dir, DISTRO_MAP_FILENAME)
+        distro_map = AsciiBinder::DistroMap.new(distro_map_file)
+        unless distro_map.is_valid?
+          errors = distro_map.errors
+          Trollop::die "The distro map file at '#{distro_map_file}' contains the following errors:\n- " + errors.join("\n- ") + "\n"
+        end
+        distro_map
+      end
+    end
+
+    def site_map
+      @site_map ||= AsciiBinder::SiteMap.new(distro_map)
+    end
+
+    def branch_group_branches
+      @branch_group_branches ||= begin
+        group_branches = Hash.new
+        group_branches[:working_only] = [local_branches[0]]
+        group_branches[:publish]      = distro_map.distro_branches
+        site_map.sites.each do |site|
+          group_branches["publish_#{site.id}".to_sym] = site.branches
+        end
+        group_branches[:all] = local_branches
+        group_branches
+      end
+    end
+
+    def page(args)
+      # TODO: This process of rebuilding the entire nav for every page will not scale well.
+      #       As the doc set increases, we will need to think about refactoring this.
+      args[:breadcrumb_root], args[:breadcrumb_group], args[:breadcrumb_subgroup], args[:breadcrumb_topic] = extract_breadcrumbs(args)
+
+      args[:breadcrumb_subgroup_block] = ''
+      args[:subtopic_shim]             = ''
+      if args[:breadcrumb_subgroup]
+        args[:breadcrumb_subgroup_block] = "<li class=\"hidden-xs active\">#{args[:breadcrumb_subgroup]}</li>"
+        args[:subtopic_shim]             = '../'
+      end
+
+      template_path = File.expand_path("#{docs_root_dir}/_templates/page.html.erb")
+      template_renderer.render(template_path, args)
+    end
+
+    def extract_breadcrumbs(args)
+      breadcrumb_root = breadcrumb_group = breadcrumb_subgroup = breadcrumb_topic = nil
+
+      root_group          = args[:navigation].first
+      selected_group      = args[:navigation].detect { |group| group[:id] == args[:group_id] }
+      selected_subgroup   = selected_group[:topics].detect { |subgroup| subgroup[:id] == args[:subgroup_id] }
+      current_is_subtopic = selected_subgroup ? true : false
+
+      if root_group
+        root_topic = root_group[:topics].first
+        breadcrumb_root = linkify_breadcrumb(root_topic[:path], "#{args[:distro]} #{args[:version]}", current_is_subtopic) if root_topic
+      end
+
+      if selected_group
+        group_topic = selected_group[:topics].first
+        breadcrumb_group = linkify_breadcrumb(group_topic[:path], selected_group[:name], current_is_subtopic) if group_topic
+
+        if selected_subgroup
+          subgroup_topic = selected_subgroup[:topics].first
+          breadcrumb_subgroup = linkify_breadcrumb(subgroup_topic[:path], selected_subgroup[:name], current_is_subtopic) if subgroup_topic
+
+          selected_topic = selected_subgroup[:topics].detect { |topic| topic[:id] == args[:topic_id] }
+          breadcrumb_topic = linkify_breadcrumb(nil, selected_topic[:name], current_is_subtopic) if selected_topic
+        else
+          selected_topic = selected_group[:topics].detect { |topic| topic[:id] == args[:topic_id] }
+          breadcrumb_topic = linkify_breadcrumb(nil, selected_topic[:name], current_is_subtopic) if selected_topic
+        end
+      end
+
+      return breadcrumb_root, breadcrumb_group, breadcrumb_subgroup, breadcrumb_topic
+    end
+
+    def linkify_breadcrumb(href, text, extra_level)
+      addl_level = extra_level ? '../' : ''
+      href ? "<a href=\"#{addl_level}#{href}\">#{text}</a>" : text
+    end
+
+    def asciidoctor_page_attrs(more_attrs=[])
+      [
+        'source-highlighter=coderay',
+        'coderay-css=style',
+        'linkcss!',
+        'icons=font',
+        'idprefix=',
+        'idseparator=-',
+        'sectanchors',
+        'data-uri',
+      ].concat(more_attrs)
+    end
+
+    def generate_docs(branch_group,build_distro,single_page)
+      # First, test to see if the docs repo has any commits. If the user has just
+      # run `asciibinder create`, there will be no commits to work from, yet.
+      if local_branches.empty?
+        raise "Before you can build the docs, you need at least one commit in your docs repo."
+      end
+
+      # Make a filepath in list form from the single_page argument
+      single_page_path = []
+      if not single_page.nil?
+        single_page_path = single_page.split(':')[0].split('/')
+        single_page_path << single_page.split(':')[1]
+        puts "Rebuilding '#{single_page_path.join('/')}' on branch '#{working_branch}'."
+      end
+
+      if not build_distro == ''
+        if not distro_map.include_distro_key?(build_distro)
+          exit
+        else
+          puts "Building only the #{distro_map.get_distro(build_distro).name} distribution."
+        end
+      elsif single_page.nil?
+        puts "Building all distributions."
+      end
+
+      # Notify the user of missing local branches
+      missing_branches = []
+      distro_map.distro_branches(build_distro).sort.each do |dbranch|
+        next if local_branches.include?(dbranch)
+        missing_branches << dbranch
+      end
+      if missing_branches.length > 0 and single_page.nil?
+        puts "\nNOTE: The following branches do not exist in your local git repo:"
+        missing_branches.each do |mbranch|
+          puts "- #{mbranch}"
+        end
+        puts "The build will proceed but these branches will not be generated."
+      end
+
+      # Generate all distros for all branches in the indicated branch group
+      branch_group_branches[branch_group].each do |local_branch|
+        # Skip known missing branches; this will only come up for the :publish branch group
+        next if missing_branches.include?(local_branch)
+
+        # Single-page regen only occurs for the working branch
+        if not local_branch == working_branch
+          if single_page.nil?
+            # Checkout the branch
+            puts "\nCHANGING TO BRANCH '#{local_branch}'"
+            git_checkout(local_branch)
+          else
+            next
+          end
+        end
+
+        # Note the image files checked in to this branch.
+        branch_image_files = Find.find(docs_root_dir).select{ |path| not path.nil? and (path =~ /.*\.png$/ or path =~ /.*\.png\.cache$/) }
+
+        first_branch = single_page.nil?
+
+        if local_branch =~ /^\(detached from .*\)/
+          local_branch = 'detached'
+        end
+
+        # The branch_orphan_files list starts with the set of all
+        # .adoc files found in the repo, and will be whittled
+        # down from there.
+        branch_orphan_files = find_topic_files
+        branch_topic_map    = topic_map
+        remove_found_topic_files(local_branch,branch_topic_map,branch_orphan_files)
+
+        if branch_orphan_files.length > 0 and single_page.nil?
+          nl_warning "Branch '#{local_branch}' includes the following .adoc files that are not referenced in the #{topic_map_file} file:\n" + branch_orphan_files.map{ |file| "- #{file}" }.join("\n")
+        end
+
+        # Run all distros.
+        distro_map.distros.each do |distro|
+          if not build_distro == ''
+            # Only building a single distro; build for all indicated branches, skip the others.
+            next unless build_distro == distro.id
+          else
+            current_distro_branches = distro_map.distro_branches(distro.id)
+
+            # In publish mode we only build "valid" distro-branch combos from the distro map
+            if branch_group.to_s.start_with?("publish") and not current_distro_branches.include?(local_branch)
+              next
+            end
+
+            # In "build all" mode we build every distro on the working branch plus the publish distro-branch combos
+            if branch_group == :all and not local_branch == working_branch and not current_distro_branches.include?(local_branch)
+              next
+            end
+          end
+
+          # Get the current distro / branch object
+          branch_config = AsciiBinder::DistroBranch.new('',{ "name" => "Branch Build", "dir" => local_branch },distro)
+          dev_branch    = true
+          if distro.branch_ids.include?(local_branch)
+            branch_config = distro.branch(local_branch)
+            dev_branch    = false
+          end
+
+          if first_branch
+            puts "\nBuilding #{distro.name} for branch '#{local_branch}'"
+            first_branch = false
+          end
+
+          # Copy files into the preview area.
+          [[stylesheet_dir, '*css', branch_config.branch_stylesheet_dir],
+           [javascript_dir, '*js',  branch_config.branch_javascript_dir],
+           [image_dir,      '*',    branch_config.branch_image_dir]].each do |dgroup|
+            src_dir = dgroup[0]
+            glob    = dgroup[1]
+            tgt_dir = dgroup[2]
+            if Dir.exist?(src_dir) and not dir_empty?(src_dir)
+              FileUtils.mkdir_p tgt_dir
+              FileUtils.cp_r Dir.glob(File.join(src_dir,glob)), tgt_dir
+            end
+          end
+
+          # Build the navigation structure for this branch / distro
+           navigation = branch_topic_map.nav_tree(distro.id)
+
+          # Build the topic files for this branch & distro
+          process_topic_entity_list(branch_config,single_page_path,navigation,branch_topic_map.list)
+        end
+
+        # In single-page context, we're done.
+        if not single_page.nil?
+          #exit 200
+          return
+        end
+
+        # Remove DITAA-generated images
+        ditaa_image_files = Find.find(docs_root_dir).select{ |path| not path.nil? and not (path =~ /_preview/ or path =~ /_package/) and (path =~ /.*\.png$/ or path =~ /.*\.png\.cache$/) and not branch_image_files.include?(path) }
+        if not ditaa_image_files.empty?
+          puts "\nRemoving ditaa-generated files from repo before changing branches."
+          ditaa_image_files.each do |dfile|
+            File.unlink(dfile)
+          end
+        end
+
+        if local_branch == working_branch
+          # We're moving away from the working branch, so save off changed files
+          git_stash_all
+        end
+      end
+
+      # Return to the original branch
+      git_checkout(working_branch)
+
+      # If necessary, restore temporarily stashed files
+      git_apply_and_drop
+
+      puts "\nAll builds completed."
+    end
+
+    def process_topic_entity_list(branch_config,single_page_path,navigation,topic_entity_list,preview_path='')
+      # When called from a topic group entity, create the preview dir for that group
+      Dir.mkdir(preview_path) unless preview_path == '' or File.exists?(preview_path)
+
+      topic_entity_list.each do |topic_entity|
+        # If this topic entity or any potential subentities are outside of the distro or single-page params, skip it.
+        next unless topic_entity.include?(branch_config.distro.id,single_page_path)
+
+        if topic_entity.is_group?
+          preview_path = topic_entity.preview_path(branch_config.distro.id,branch_config.dir)
+          process_topic_entity_list(branch_config,single_page_path,navigation,topic_entity.subitems,preview_path)
+        elsif topic_entity.is_topic?
+          if single_page_path.length == 0
+            puts "  - #{topic_entity.repo_path}"
+          end
+          configure_and_generate_page(topic_entity,branch_config,navigation)
+        end
+      end
+    end
+
+    def configure_and_generate_page(topic,branch_config,navigation)
+      distro = branch_config.distro
+      topic_adoc = File.open(topic.source_path,'r').read
+
+      page_attrs = asciidoctor_page_attrs([
+        "imagesdir=#{File.join(topic.parent.source_path,'images')}",
+        branch_config.distro.id,
+        "product-title=#{branch_config.distro_name}",
+        "product-version=#{branch_config.name}",
+        "product-author=#{branch_config.distro_author}"
+      ])
+
+      doc = Asciidoctor.load topic_adoc, :header_footer => false, :safe => :unsafe, :attributes => page_attrs
+      article_title = doc.doctitle || topic.name
+
+      topic_html = doc.render
+      dir_depth  = ''
+      if branch_config.dir.split('/').length > 1
+        dir_depth = '../' * (branch_config.dir.split('/').length - 1)
+      end
+
+      # This is logic bridges newer arbitrary-depth-tolerant code to
+      # older depth-limited code. Truly removing depth limitations will
+      # require changes to page templates in user docs repos.
+      breadcrumb     = topic.breadcrumb
+      group_title    = breadcrumb[0][:name]
+      group_id       = breadcrumb[0][:id]
+      topic_title    = breadcrumb[-1][:name]
+      topic_id       = breadcrumb[-1][:id]
+      subgroup_title = nil
+      subgroup_id    = nil
+      if breadcrumb.length == 3
+        subgroup_title = breadcrumb[1][:name]
+        subgroup_id    = breadcrumb[1][:id]
+        dir_depth      = '../' + dir_depth
+      end
+
+      preview_path = topic.preview_path(distro.id,branch_config.dir)
+
+      page_args = {
+        :distro_key       => distro.id,
+        :distro           => branch_config.distro_name,
+        :site_name        => distro.site.name,
+        :site_url         => distro.site.url,
+        :topic_url        => preview_path,
+        :version          => branch_config.name,
+        :group_title      => group_title,
+        :subgroup_title   => subgroup_title,
+        :topic_title      => topic_title,
+        :article_title    => article_title,
+        :content          => topic_html,
+        :navigation       => navigation,
+        :group_id         => group_id,
+        :subgroup_id      => subgroup_id,
+        :topic_id         => topic_id,
+        :css_path         => "../../#{dir_depth}#{branch_config.dir}/#{STYLESHEET_DIRNAME}/",
+        :javascripts_path => "../../#{dir_depth}#{branch_config.dir}/#{JAVASCRIPT_DIRNAME}/",
+        :images_path      => "../../#{dir_depth}#{branch_config.dir}/#{IMAGE_DIRNAME}/",
+        :site_home_path   => "../../#{dir_depth}index.html",
+        :template_path    => template_dir,
+      }
+      full_file_text = page(page_args)
+      File.write(preview_path,full_file_text)
+    end
+
+    # package_docs
+    # This method generates the docs and then organizes them the way they will be arranged
+    # for the production websites.
+    def package_docs(package_site)
+      site_map.sites.each do |site|
+        next if not package_site == '' and not package_site == site.id
+        site.distros.each do |distro_id,branches|
+          branches.each do |branch|
+            src_dir  = File.join(preview_dir,distro_id,branch.dir)
+            tgt_tdir = branch.dir.split('/')
+            tgt_tdir.pop
+            tgt_dir  = ''
+            if tgt_tdir.length > 0
+              tgt_dir = File.join(package_dir,site.id,tgt_tdir.join('/'))
+            else
+              tgt_dir = File.join(package_dir,site.id)
+            end
+            next if not File.directory?(src_dir)
+            FileUtils.mkdir_p(tgt_dir)
+            FileUtils.cp_r(src_dir,tgt_dir)
+          end
+          site_dir = File.join(package_dir,site.id)
+          if File.directory?(site_dir)
+            puts "\nPackaging #{distro_id} for #{site.id} site."
+
+            # Any files in the root of the docs repo with names ending in:
+            #     *-#{site}.html
+            # will get copied into the root dir of the packaged site with
+            # the site name stripped out.
+            #
+            # Example: for site name 'commercial', the files:
+            #     * index-commercial.html would end up as #{site_root}/index.html
+            #     * search-commercial.html would end up as #{site_root}/search.html
+            #     * index-community.html would be ignored
+            site_files = Dir.glob(File.join(docs_root_dir, '*-' + site.id + '.html'))
+            unless site_files.empty?
+              site_files.each do |fpath|
+                target_basename = File.basename(fpath).gsub(/-#{site.id}\.html$/, '.html')
+                FileUtils.cp(fpath,File.join(package_dir,site.id,target_basename))
+              end
+            else
+              FileUtils.cp(File.join(preview_dir,distro_id,'index.html'),File.join(package_dir,site.id,'index.html'))
+            end
+            ['_images','_stylesheets'].each do |support_dir|
+              FileUtils.cp_r(File.join(docs_root_dir,support_dir),File.join(package_dir,site.id,support_dir))
+            end
+
+            # Now build a sitemap
+            site_dir_path = Pathname.new(site_dir)
+            SitemapGenerator::Sitemap.create(
+              :default_host => site.url,
+              :public_path  => site_dir_path,
+              :compress     => false,
+              :filename     => File.join(site_dir,'sitemap')
+            ) do
+              file_list = Find.find(site_dir).select{ |path| not path.nil? and path =~ /.*\.html$/ }.map{ |path| '/' + Pathname.new(path).relative_path_from(site_dir_path).to_s }
+              file_list.each do |file|
+                add(file, :changefreq => 'daily')
+              end
+            end
+          end
+        end
+      end
+    end
+
+    def clean_up
+      if not system("rm -rf #{docs_root_dir}/_preview/* #{docs_root_dir}/_package/*")
+        puts "Nothing to clean."
+      end
+    end
+  end
+end