docubot 0.3.3 → 0.4

data/bin/docubot

@@ -99,8 +99,12 @@ if ARGS[:create]
 		end
 	end
 else
+	# require 'perftools'
 	start = Time.now
-	bundle = DocuBot::Bundle.new( ARGS[:directory] )
+	# bundle = nil
+	# PerfTools::CpuProfiler.start("/tmp/docubot") do
+		bundle = DocuBot::Bundle.new( ARGS[:directory] )
+	# end
 	lap = Time.now
 	puts "%.2fs to prepare the bundle..." % (lap-start)
 	bundle.write( ARGS[:writer], ARGS[:output] )

data/lib/docubot.rb

@@ -20,7 +20,7 @@ module FileUtils
 end
 
 module DocuBot
-	VERSION = '0.3.3'
+	VERSION = '0.4'
 	DIR     = File.expand_path( File.dirname( __FILE__ ) )
 	
 	TEMPLATE_DIR = DIR / 'docubot/templates'
@@ -28,10 +28,12 @@ module DocuBot
 	Dir.chdir( SHELL_DIR ){ SHELLS = Dir['*'] }
 	
 	def self.id_from_text( text )
-		text.strip.gsub(/[^\w.:-]+/,'-').gsub(/^-|-$/,'')
+		"#" << text.strip.gsub(/[^\w.:-]+/,'-').gsub(/^[^a-z]+|-+$/i,'')
 	end
 end
 
+require 'docubot/link_tree'
+require 'docubot/metasection'
 require 'docubot/snippet'
 require 'docubot/converter'
 require 'docubot/writer'

data/lib/docubot/bundle.rb

@@ -1,64 +1,49 @@
 # encoding: UTF-8
 require 'pathname'
 class DocuBot::Bundle
-	attr_reader :toc, :extras, :glossary, :index, :source
+	attr_reader :toc, :extras, :glossary, :index, :source, :global
 	attr_reader :internal_links, :external_links, :file_links, :broken_links
+	attr_reader :pages, :pages_by_title, :page_by_file_path, :page_by_html_path
 	def initialize( source_directory )
 		@source = File.expand_path( source_directory )
 		raise "DocuBot cannot find directory #{@source}. Exiting." unless File.exists?( @source )
+		@pages  = []
 		@extras = []
+		@pages_by_title    = Hash.new{ |h,k| h[k]=[] }
+		@page_by_file_path = {}
+		@page_by_html_path = {}
+
 		@glossary = DocuBot::Glossary.new( self, @source/'_glossary' )
 		@index    = DocuBot::Index.new( self )
+		@toc      = DocuBot::LinkTree::Root.new( self )
+
 		Dir.chdir( @source ) do
-			@toc = DocuBot::Page.new( self, ".", "Table of Contents" )
-			@toc.meta['glossary'] = @glossary
-			@toc.meta['index']    = @index
-			pages_by_path = { '.'=>@toc }
-		
+			# This might be nil; MetaSection.new is OK with that.
+			index_file = Dir[ *DocuBot::Converter.types.map{|t| "index.#{t}"} ][ 0 ]
+			@global = DocuBot::MetaSection.new( {}, index_file )
+			@global.glossary = @glossary
+			@global.index    = @index
+			@global.toc      = @toc
+
 			files_and_folders = Dir[ '**/*' ]
-			files_and_folders.reject!{ |f| File.basename(f) =~ /^index\.[^.]+$/ || File.basename(f) == '_static' || File.basename(f) == '_glossary' }
-			files_and_folders.reject!{ |f| f =~ /\b_templates\b/ }
-			files_and_folders.each do |item|
-				extension = File.extname( item )[ 1..-1 ]
-				item_is_page = File.directory?(item) || DocuBot::Converter.by_type[extension]
-				if item_is_page
-					parent = pages_by_path[ File.dirname( item ) ]
-					page = DocuBot::Page.new( self, item )
-					pages_by_path[ item ] = page
-					parent << page if parent
-					if item =~ /\b_glossary\b/
-						@glossary << page
-					end
-					@index.process_page( page )
-					
-					# TODO: Move this bloat elsewhere.
-					if page.toc?
-						ndoc = page.nokodoc
-						toc = page.toc
-						ids = if toc[','] # Comma-delimited toc interpreted as generated ids
-							toc.split(/,\s*/).map{ |title| DocuBot.id_from_text(title) }
-						else
-							toc.scan /[a-z][\w.:-]*/i
-						end
- 						ids.each do |id|
-							if ele = ndoc.at_css("##{id}")
-								page << DocuBot::SubLink.new( page, ele.inner_text, id )
-							else
-								warn "Could not find requested toc anchor '##{id}' on #{page.html_path}"
-							end
-						end
-					end
-					
-				else
-					# TODO: Anything better needed?
-					@extras << item
-				end
+			
+			# index files are handled by Page.new for a directory; no sections for special folders (but process contents)
+			files_and_folders.reject!{ |path| name = File.basename( path ); name =~ /^(?:index\.[^.]+|_static|_glossary)$/ }
+			
+			# All files in the _templates directory should be ignored
+			files_and_folders.reject!{ |f| f =~ /^_templates\b/ }
+
+			@global.ignore.as_list.each do |glob|
+				files_and_folders = files_and_folders - Dir[glob]
 			end
+			
+			create_pages( files_and_folders )			
 		end
+		# puts @toc.to_txt
 		
 		# Regenerate pages whose templates require full scaning to have completed
 		# TODO: make this based off of a metasection attribute.
-		@toc.every_page.select do |page|
+		@pages.select do |page|
 			%w[ glossary ].include?( page.template )
 		end.each do |page|
 			page.dirty_template
@@ -66,20 +51,48 @@ class DocuBot::Bundle
 		
 		# TODO: make this optional via global variable
 		validate_links
-		@broken_links.each do |page,links|
-			links.each do |link|
-				warn "Broken link on #{page.file}: '#{link}'"
-			end
-		end
+		warn_for_broken_links
 		
 		# TODO: make this optional via global variable
-		@glossary.missing_terms.each do |term,referrers|
-			warn "Glossary term '#{term}' never defined."
-			referrers.each do |referring_page|
-				warn "...seen on #{referring_page.file}."
-			end
-		end
+		warn_for_missing_glossary_terms
 		
+		find_page_collisions
+	end
+	
+	def create_pages( files_and_folders )
+		files_and_folders.each do |path|
+			extension = File.extname( path )[ 1..-1 ]
+			item_is_page = File.directory?(path) || DocuBot::Converter.by_type[extension]
+			if !item_is_page
+				@extras << path
+			else
+				page = DocuBot::Page.new( self, path )
+
+				if path =~ %r{^_glossary/}
+					@glossary << page
+				else
+					@pages                            << page
+					@page_by_file_path[path]           = page
+					@page_by_html_path[page.html_path] = page
+					@pages_by_title[page.title]       << page
+					@index.process_page( page )
+
+					# Add the page (and any sub-links) to the toc
+					unless page.hide
+						@toc.add_to_link_hierarchy( page.title, page.html_path, page )
+						page.toc.as_list.each do |id_or_text|
+							id = id_or_text[0..0] == '#' ? id_or_text : DocuBot.id_from_text(id_or_text)
+							if ele = page.nokodoc.at_css(id)
+								@toc.add_to_link_hierarchy( ele.inner_text, page.html_path + id, page )
+							else
+								warn "Could not find requested toc anchor #{id.inspect} based on #{id_or_text.inspect} on #{page.html_path}"
+							end
+						end
+					end
+					
+				end
+			end
+		end		
 	end
 
 	def validate_links
@@ -90,26 +103,26 @@ class DocuBot::Bundle
 
 		page_by_html_path = {}
 		page_by_orig_path = {}
-		@toc.every_page.each do |page|
+		@pages.each do |page|
 			page_by_html_path[page.html_path] = page
 			page_by_orig_path[page.file]      = page if page.file
 		end
 
 		Dir.chdir( @source ) do 
-			@toc.every_page.each do |page|
-				next unless page.nokodoc # Sub-links don't have documents
-				page.nokodoc.xpath('//a/@href').each do |href|
+			@pages.each do |page|
+				page.nokodoc.xpath('.//a/@href').each do |href|
 					href=href.content
 					if href=~%r{^[a-z]+://}i
 						@external_links[page] << href
 					else
-						id   = href[/(#[a-z][\w.:-]*)/i,1]
+						id   = href[/#[a-z][\w.:-]*/i]
 						file = href.sub(/#.+/,'')
 						path = file.empty? ? page.html_path : Pathname.new( File.dirname(page.html_path) / file ).cleanpath.to_s
 						if target=page_by_html_path[path]
 							if !id || target.nokodoc.at_css(id)
 								@internal_links[page] << href
 							else
+								warn "Could not find internal link for #{id.inspect} on #{page.html_path.inspect}" if id 
 								@broken_links[page] << href
 							end
 						else
@@ -124,6 +137,38 @@ class DocuBot::Bundle
 			end
 		end
 	end
+	
+	def warn_for_broken_links
+		@broken_links.each do |page,links|
+			links.each do |link|
+				warn "Broken link on #{page.file}: '#{link}'"
+			end
+		end
+	end
+	
+	def warn_for_missing_glossary_terms
+		@glossary.missing_terms.each do |term,referrers|
+			warn "Glossary term '#{term}' never defined."
+			referrers.each do |referring_page|
+				warn "...seen on #{referring_page.file}."
+			end
+		end		
+	end
+	
+	def find_page_collisions
+		# Find any and all pages that would collide
+		pages_by_html_path = Hash.new{ |h,k| h[k] = [] }
+		@pages.each do |page|
+			pages_by_html_path[page.html_path] << page
+		end
+		collisions = pages_by_html_path.select{ |path,pages| pages.length>1 }
+		unless collisions.empty?
+			message = collisions.map do |path,pages|
+				"#{path}: #{pages.map{ |page| "'#{page.title}' (#{page.file})" }.join(', ')}"
+			end.join("\n")
+			raise PageCollision.new, message
+		end		
+	end
 
 	def write( writer_type, destination=nil )
 		writer = DocuBot::Writer.by_type[ writer_type.to_s.downcase ]
@@ -134,4 +179,7 @@ class DocuBot::Bundle
 		end
 	end
 	
-end
+end
+
+class DocuBot::Bundle::PageCollision < RuntimeError; end
+

data/lib/docubot/converter.rb

@@ -8,6 +8,9 @@ module DocuBot
 		def self.by_type
 			@by_type
 		end
+		def self.types
+			@by_type.keys
+		end
 	end
 
 	def self.convert_to_html( page, source, type )
@@ -22,4 +25,4 @@ end
 
 Dir[ DocuBot::DIR/'docubot/converters/*.rb' ].each do |converter|
 	require converter
-end
+end

data/lib/docubot/converters/haml.rb

@@ -5,5 +5,5 @@ options = { :format=>:html4, :ugly=>true }
 options.merge!( :encoding=>'utf-8' ) if Object.const_defined? "Encoding"
 
 DocuBot::Converter.to_convert :haml do |page, source|
-	Haml::Engine.new( source, options ).render( page, :page=>page, :global=>page.bundle.toc, :root=>page.root )
+	Haml::Engine.new( source, options ).render( page, :page=>page, :global=>page.bundle.global, :root=>page.root )
 end

data/lib/docubot/glossary.rb

@@ -46,4 +46,7 @@ class DocuBot::Glossary
 			end.join(",\n")
 		}};"
 	end
+	def inspect
+		"<#{self.class} terms='#{@entries.keys.join ', '}'>"
+	end
 end

data/lib/docubot/link_tree.rb

@@ -0,0 +1,109 @@
+# encoding: UTF-8
+module DocuBot::LinkTree; end
+
+class DocuBot::LinkTree::Node
+	attr_accessor :title, :link, :page, :parent
+	
+	def initialize( title=nil, link=nil, page=nil )
+		@title,@link,@page = title,link,page
+		@children = []
+	end
+	
+	def anchor
+		@link[/#(.+)/,1]
+	end
+	
+	def file
+		@link.sub(/#.+/,'')
+	end
+	
+	def leaf?
+		!@children.any?{ |node| node.page != @page }
+	end
+
+	# Add a new link underneath a link to its logical parent
+	def add_to_link_hierarchy( title, link, page=nil )
+		node = DocuBot::LinkTree::Node.new( title, link, page )
+		parent_link = if node.anchor
+			node.file
+		elsif File.basename(link)=='index.html'
+			File.dirname(File.dirname(link))/'index.html'
+		else
+			(File.dirname(link) / 'index.html')
+		end
+		#puts "Adding #{title.inspect} (#{link}) to hierarchy under #{parent_link}"
+		parent = descendants.find{ |node| node.link==parent_link } || self
+		parent << node
+	end
+
+	def <<( node )
+		node.parent = self
+		@children << node
+	end
+	
+	def children( parent_link=nil, &block )
+		if parent_link
+			root = find( parent_link )
+			root ? root.children( &block ) : []
+		else
+			@children
+		end
+	end
+	
+	def descendants
+		( @children + @children.map{ |child| child.descendants } ).flatten
+	end
+	
+	def find( link )
+		# TODO: this is eminently cachable
+		descendants.find{ |node| node.link==link }
+	end
+	
+	def depth
+		# Assuming no one is going to shuffle the nodes after placement
+		@depth ||= ancestors.length
+	end
+	
+	def root
+		@root ||= "../" * (depth + ( leaf? ? 0 : 1 ))
+	end
+	
+	def ancestors
+		ancestors = []
+		node = self
+		ancestors << node while node = node.parent
+		ancestors.reverse!
+	end
+	
+	def to_s
+		"#{@title} (#{@link}) - #{@page && @page.title}"
+	end
+	
+	def to_txt( depth=0 )
+		indent = "  "*depth
+		[
+			indent+to_s,
+			children.map{|kid|kid.to_txt(depth+1)}
+		].flatten.join("\n")
+	end
+end
+
+class DocuBot::LinkTree::Root < DocuBot::LinkTree::Node
+	undef_method :title
+	undef_method :link
+	undef_method :page
+	attr_reader :bundle
+	def initialize( bundle )
+		@bundle   = bundle
+		@children = []
+	end
+
+	def <<( node )
+		node.parent = nil
+		@children << node
+	end
+	
+	def to_s
+		"(Table of Contents)"
+	end
+end

data/lib/docubot/metasection.rb

@@ -0,0 +1,61 @@
+# encoding: UTF-8
+class DocuBot::MetaSection; end
+module DocuBot::MetaSection::Castable
+	def as_boolean
+		self == 'true'
+	end
+	def as_list
+		return [] if self.nil?
+		# Replace commas inside quoted strings with unlikely-to-be-used Unicode
+		# FIXME: This doesnt work for the sadistic case of "hello """, "world"
+		csv = self.gsub( /("(?:[^",]|"")+),/, '\\1⥸' )
+		csv.split(/\s*,\s*/).map do |str|
+			# Put real commas back, unquote, undouble internal quotes
+			str[/^".*"$/] ? str[1..-2].gsub('⥸',',').gsub('""','"') : str
+		end
+	end
+end
+
+class DocuBot::MetaSection
+	META_SEPARATOR = /^\+\+\+\s*$/ # Sort of like +++ATH0
+	NIL_CASTABLE   = nil.extend( Castable )
+	attr_reader :__contents__	
+	
+	def initialize( attrs={}, file_path=nil )
+		@attrs = {}
+		attrs.each{ |key,value| self[key]=value }
+		if file_path && File.exists?( file_path )
+			parts = IO.read( file_path ).split( META_SEPARATOR, 2 )
+			if parts.length > 1
+				parts.first.scan(/.+/) do |line|
+					next if line =~ /^\s*#/
+					next unless line.include?(':')
+					attribute, value = line.split(':',2).map{ |str| str.strip }
+					self[attribute] = value
+				end
+			end
+			@__contents__ = parts.last && parts.last.strip
+		end
+	end
+	
+	def has_key?( key )
+		@attrs.has_key?( key )
+	end
+	
+	def []( attribute )
+		@attrs.has_key?( attribute ) ? @attrs[attribute] : NIL_CASTABLE
+	end
+	
+	def []=( attribute, value )
+		@attrs[attribute.to_s] = value.extend(Castable)
+	end
+	
+	def method_missing( method, *args )
+		key=method.to_s
+		case key[-1..-1] # the last character of the method name
+			when '=' then self[key[0..-2]] = args.first
+			else self[key]
+		end
+	end
+	
+end