thewoolleyman-webgen 0.5.8.20090419

data/lib/webgen/sourcehandler/base.rb

@@ -0,0 +1,248 @@
+# -*- encoding: utf-8 -*-
+
+require 'facets/symbol/to_proc'
+require 'webgen/websiteaccess'
+require 'webgen/loggable'
+require 'webgen/page'
+
+module Webgen::SourceHandler
+
+  # This module should be included in every source handler as it provides the default methods for
+  # creating nodes.
+  #
+  # == Implementing Source Handlers
+  #
+  # A source handler is a webgen extension that processes source paths to create nodes and that
+  # provides the rendered content of these nodes. The nodes are later written to the output
+  # location. This can range from simply copying a path from the source to the output location to
+  # generating a whole set of nodes from one input path!
+  #
+  # The paths that are handled by a source handler are specified via path patterns (see
+  # below). During a webgen run the #create_node method for each source paths that matches a
+  # specified path pattern is called. And when it is time to write out the node, the the #content
+  # method is called to retrieve the rendered content.
+  #
+  # A source handler must not take any parameters on initialization and when this module is not
+  # mixed in, the methods #create_node and #content need to be defined. Also, a source handler does
+  # not need to reside under the Webgen::SourceHandler namespace but all shipped ones do.
+  #
+  # This base class provides useful default implementations of methods that are used by nearly all
+  # source handler classes:
+  # * #create_node
+  # * #output_path
+  # * #node_exists?
+  #
+  # It also provides other utility methods:
+  # * #page_from_path
+  # * #content
+  #
+  # == Nodes Created for Paths
+  #
+  # The main functions of a source handler class are to create one or more nodes for a source path
+  # and to provide the content of these nodes. To achieve this, certain information needs to be set
+  # on a created node. If you use the +create_node+ method provided by this base class, you don't
+  # need to set them explicitly because this is done by the method:
+  #
+  # [<tt>node_info[:processor]</tt>] Has to be set to the class name of the source handler. This is
+  #                                  used by the Node class: all unknown method calls are forwarded
+  #                                  to the node processor.
+  # [<tt>node_info[:src]</tt>] Has to be set to the string version of the path that lead to the
+  #                            creation of the node.
+  # [<tt>node_info[:creation_path]</tt>] Has to be set to the string version of the path that is
+  #                                      used to create the path.
+  # [<tt>meta_info['no_output']</tt>] Has to be set to +true+ on nodes that are used during a
+  #                                   webgen run but do not produce an output file.
+  # [<tt>meta_info['modified_at']</tt>] Has to be set to the current time if not already set
+  #                                     correctly (ie. if not a Time object).
+  #
+  # If <tt>meta_info['draft']</tt> is set on a path, then no node should be created in +create_node+
+  # and +nil+ has to be returned.
+  #
+  # Note: The difference between +:src+ and +:creation_path+ is that a creation path
+  # need not have an existing source path representation. For example, fragments created from a page
+  # source path have a different +:creation_path+ which includes the fragment part.
+  #
+  # Additional information that is used only for processing purposes should be stored in the
+  # #node_info hash of a node as the #meta_info hash is reserved for real node meta information and
+  # should not be changed once the node is created.
+  #
+  # == Output Path Names
+  #
+  # The method for creating an output path name for a source path is stored in the meta information
+  # +output_path+. If you don't use the provided method +output_path+, have a look at its
+  # implementation to see how to an output path gets created. Individual output path creation
+  # methods are stored as methods in the OutputPathHelpers module.
+  #
+  # == Path Patterns and Invocation order
+  #
+  # Path patterns define which paths are handled by a specific source handler. These patterns are
+  # specified in the <tt>sourcehandler.patterns</tt> configuration hash as a mapping from the source
+  # handler class name to an array of path patterns. The patterns need to have a format that
+  # <tt>Dir.glob</tt> can handle. You can use the configuration helper +patterns+ to set this (is
+  # shown in the example below).
+  #
+  # Specifying a path pattern does not mean that webgen uses the source handler. One also needs to
+  # provide an entry in the configuration value <tt>sourcehandler.invoke</tt>. This is a hash that
+  # maps the invocation rank (a number) to an array of source handler class names. The lower the
+  # invocation rank the earlier the specified source handlers are used.
+  #
+  # The default invocation ranks are:
+  # [1] Early. Normally there is no need to use this rank.
+  # [5] Standard. This is the rank the normal source handler should use.
+  # [9] Late. This rank should be used by source handlers that operate on/use already created nodes
+  #     and need to ensure that these nodes are available.
+  #
+  # == Default Meta Information
+  #
+  # Each source handler can define default meta information that gets automatically set on the
+  # source paths that are passed to the #create_node method.
+  #
+  # The default meta information is specified in the <tt>sourcehandler.default_meta_info</tt>
+  # configuration hash as a mapping from the source handler class name to the meta information
+  # hash.
+  #
+  # == Sample Source Handler Class
+  #
+  # Following is a simple source handler class example which copies paths from the source to
+  # the output location modifying the extension:
+  #
+  #   class SimpleCopy
+  #
+  #     include Webgen::SourceHandler::Base
+  #
+  #     def create_node(parent, path)
+  #       path.ext += '.copied'
+  #       super(parent, path)
+  #     end
+  #
+  #     def content(node)
+  #       website.blackboard.invoke(:source_paths)[node.node_info[:src]].io
+  #     end
+  #
+  #   end
+  #
+  #   WebsiteAccess.website.config.patterns('SimpleCopy' => ['**/*.jpg', '**/*.png'])
+  #   WebsiteAccess.website.config.sourcehandler.invoke[5] << 'SimpleCopy'
+  #
+  module Base
+
+    # This module is used for defining all methods that can be used for creating output paths.
+    #
+    # All public methods of this module are considered to be output path creation methods and must
+    # have the following parameters:
+    #
+    # [+parent+] the parent node
+    # [+path+] the path for which the output name should be created
+    # [+use_lang_part+] controls whether the output path name has to include the language part
+    module OutputPathHelpers
+
+      # Default method for creating an output path for +parent+ and source +path+.
+      #
+      # The automatically set parameter +style+ (which uses the meta information +output_path_style+
+      # from the path's meta information hash) defines how the output name should be built (more
+      # information about this in the user documentation).
+      def standard_output_path(parent, path, use_lang_part, style = path.meta_info['output_path_style'])
+        result = style.collect do |part|
+          case part
+          when String  then part
+          when :lang   then use_lang_part ? path.meta_info['lang'] : ''
+          when :ext    then path.ext.empty? ? '' : '.' + path.ext
+          when :parent then temp = parent; temp = temp.parent while temp.is_fragment?; temp.path
+          when :year, :month, :day
+            ctime = path.meta_info['created_at']
+            if !ctime.kind_of?(Time)
+              raise "Invalid meta info 'created_at' for #{path}, needed because of used output path style"
+            end
+            ctime.send(part).to_s.rjust(2, '0')
+          when Symbol  then path.send(part)
+          when Array   then part.include?(:lang) && !use_lang_part ? '' : standard_output_path(parent, path, use_lang_part, part)
+          else ''
+          end
+        end
+        result.join('')
+      end
+
+    end
+
+    include Webgen::Loggable
+    include OutputPathHelpers
+
+    # Construct the output name for the given +path+ and +parent+. First it is checked if a node
+    # with the constructed output name already exists. If it exists, the language part is forced to
+    # be in the output name and the resulting output name is returned.
+    def output_path(parent, path)
+      method = path.meta_info['output_path'] + '_output_path'
+      use_lang_part = if path.meta_info['lang'].nil? # unlocalized files never get a lang in the filename!
+                        false
+                      else
+                        Webgen::WebsiteAccess.website.config['sourcehandler.default_lang_in_output_path'] ||
+                          Webgen::WebsiteAccess.website.config['website.lang'] != path.meta_info['lang']
+                      end
+      if OutputPathHelpers.public_instance_methods(false).map(&:to_s).include?(method)
+        name = send(method, parent, path, use_lang_part)
+        name += '/'  if path.path =~ /\/$/ && name !~ /\/$/
+        if (node = node_exists?(parent, path, name)) && node.lang == path.meta_info['lang']
+          name = node.path
+        elsif node
+          name = send(method, parent, path, (path.meta_info['lang'].nil? ? false : true))
+          name += '/'  if path.path =~ /\/$/ && name !~ /\/$/
+        end
+        name
+      else
+        raise "Unknown method for creating output path: #{method}"
+      end
+    end
+
+    # Check if the node alcn and output path which would be created by #create_node exists. The
+    # +output_path+ to check for can individually be set.
+    def node_exists?(parent, path, output_path = self.output_path(parent, path))
+      parent.tree[Webgen::Node.absolute_name(parent, path.lcn, :alcn)] || (!path.meta_info['no_output'] && parent.tree[output_path, :path])
+    end
+
+    # Create a node under +parent+ from +path+ if it does not already exists or needs to be
+    # re-initialized. The found node or the newly created node is returned afterwards. +nil+ is
+    # returned if no node can be created (e.g. when <tt>path.meta_info['draft']</tt> is set). Some
+    # additional node information like <tt>:src</tt> and <tt>:processor</tt> is set and the meta
+    # information is checked for validness. The created/re-initialized node is yielded if a block is
+    # given.
+    def create_node(parent, path, output_path = self.output_path(parent, path))
+      return nil if path.meta_info['draft']
+      node = node_exists?(parent, path, output_path)
+      if node && (node.node_info[:src] != path.source_path || node.node_info[:processor] != self.class.name)
+        log(:warn) { "Node already exists: source = #{path.source_path} | path = #{node.path} | alcn = #{node.absolute_lcn}"}
+        return node
+      elsif !node
+        node = Webgen::Node.new(parent, output_path, path.cn, path.meta_info)
+      elsif node.flagged(:reinit)
+        node.reinit(output_path, path.meta_info)
+      else
+        return node
+      end
+      node['modified_at'] = Time.now unless node['modified_at'].kind_of?(Time)
+      node.node_info[:src] = path.source_path
+      node.node_info[:creation_path] = path.path
+      node.node_info[:processor] = self.class.name
+      yield(node) if block_given?
+      node
+    end
+
+    # Return the content of the given +node+. This default +content+ method just returns +nil+.
+    def content(node)
+      nil
+    end
+
+    # Utility method for creating a Webgen::Page object from the +path+. Also updates
+    # <tt>path.meta_info</tt> with the meta info from the page.
+    def page_from_path(path)
+      begin
+        page = Webgen::Page.from_data(path.io.data, path.meta_info)
+      rescue Webgen::WebgenPageFormatError => e
+        raise "Error reading source path <#{path}>: #{e.message}"
+      end
+      path.meta_info = page.meta_info
+      page
+    end
+
+  end
+
+end

data/lib/webgen/sourcehandler/copy.rb

@@ -0,0 +1,43 @@
+# -*- encoding: utf-8 -*-
+
+module Webgen::SourceHandler
+
+  # Simple source handler for copying files from the source tree, either verbatim or by applying a
+  # content processor.
+  class Copy
+
+    include Webgen::WebsiteAccess
+    include Base
+
+    # Create the node for +parent+ and +path+. If the +path+ has the name of a content processor as
+    # the first part in the extension, it is preprocessed.
+    def create_node(parent, path)
+      if path.ext.index('.')
+        processor, *rest = path.ext.split('.')
+        if website.blackboard.invoke(:content_processor_names).include?(processor)
+          path.ext = rest.join('.')
+        else
+          processor = nil
+        end
+      end
+      super(parent, path) do |node|
+        node.node_info[:preprocessor] = processor
+      end
+    end
+
+    # Return either the preprocessed content of the +node+ or the IO object for the node's source
+    # path depending on the node type.
+    def content(node)
+      io = website.blackboard.invoke(:source_paths)[node.node_info[:src]].io
+      if node.node_info[:preprocessor]
+        context = Webgen::ContentProcessor::Context.new(:content => io.data, :chain => [node])
+        website.blackboard.invoke(:content_processor, node.node_info[:preprocessor]).call(context)
+        context.content
+      else
+        io
+      end
+    end
+
+  end
+
+end

data/lib/webgen/sourcehandler/directory.rb

@@ -0,0 +1,36 @@
+# -*- encoding: utf-8 -*-
+
+module Webgen::SourceHandler
+
+  # Handles directory source paths.
+  class Directory
+
+    include Base
+    include Webgen::WebsiteAccess
+
+    def initialize # :nodoc:
+      website.blackboard.add_service(:create_directories, method(:create_directories))
+    end
+
+    # Recursively create the directories specified in +dirname+ under +parent+ (a leading slash is
+    # ignored). The path +path+ is the path that lead to the creation of these directories.
+    def create_directories(parent, dirname, path)
+      dirname.sub(/^\//, '').split('/').each do |dir|
+        dir_path = Webgen::Path.new(File.join(parent.absolute_lcn, dir, '/'), path)
+        nodes = website.blackboard.invoke(:create_nodes, parent.tree, parent.absolute_lcn,
+                                           dir_path, self) do |dir_parent, dir_path|
+          node_exists?(dir_parent, dir_path) || create_node(dir_parent, dir_path)
+        end
+        parent = nodes.first
+      end
+      parent
+    end
+
+    # Return an empty string to signal that the directory should be written to the output.
+    def content(node)
+      ''
+    end
+
+  end
+
+end

data/lib/webgen/sourcehandler/feed.rb

@@ -0,0 +1,117 @@
+# -*- encoding: utf-8 -*-
+
+module Webgen::SourceHandler
+
+  # Source handler for creating atom and/or rss feeds.
+  class Feed
+
+    include Webgen::WebsiteAccess
+    include Base
+
+    # The mandatory keys that need to be set in a feed file.
+    MANDATORY_INFOS = %W[site_url author entries]
+
+    def initialize # :nodoc:
+      website.blackboard.add_listener(:node_changed?, method(:node_changed?))
+    end
+
+    # Create atom and/or rss feed files from +parent+ and +path+.
+    def create_node(parent, path)
+      page = page_from_path(path)
+      path.meta_info['link'] ||= parent.absolute_lcn
+
+      if MANDATORY_INFOS.any? {|t| path.meta_info[t].nil?}
+        raise "One of #{MANDATORY_INFOS.join('/')} information missing for feed <#{path}>"
+      end
+
+      create_feed_node = lambda do |type|
+        path.ext = type
+        super(parent, path) do |node|
+          node.node_info[:feed] = page
+          node.node_info[:feed_type] = type
+        end
+      end
+
+      nodes = []
+      nodes << create_feed_node['atom'] if path.meta_info['atom']
+      nodes << create_feed_node['rss'] if path.meta_info['rss']
+
+      nodes
+    end
+
+    # Return the rendered feed represented by +node+.
+    def content(node)
+      website.cache[[:sourcehandler_feed, node.node_info[:src]]] = feed_entries(node).map {|n| n.absolute_lcn}
+      block_name = node.node_info[:feed_type] + '_template'
+      if node.node_info[:feed].blocks.has_key?(block_name)
+        node.node_info[:feed].blocks[block_name].
+          render(Webgen::ContentProcessor::Context.new(:chain => [node])).content
+      else
+        feed = (website.cache.volatile[:sourcehandler_feed] ||= {})[node.node_info[:src]] ||= build_feed_for(node)
+        feed.build_xml(node.node_info[:feed_type], (node.node_info[:feed_type] == 'rss' ? node['rss_version'] || 2.0 : nil))
+      end
+    end
+
+    # Helper method for returning the entries for the feed node +node+.
+    def feed_entries(node)
+      nr_items = (node['number_of_entries'].to_i == 0 ? 10 : node['number_of_entries'].to_i)
+      patterns = [node['entries']].flatten.map {|pat| Webgen::Common.absolute_path(pat, node.parent.absolute_lcn)}
+
+      node.tree.node_access[:alcn].values.
+        select {|node| patterns.any? {|pat| node =~ pat} && node.node_info[:page]}.
+        sort {|a,b| a['modified_at'] <=> b['modified_at']}[0, nr_items]
+    end
+
+    #######
+    private
+    #######
+
+    # Return the populated FeedTools::Feed object for +node+.
+    def build_feed_for(node)
+      require 'feed_tools'
+      require 'time'
+
+      site_url = node['site_url']
+
+      feed = FeedTools::Feed.new
+      feed.title = node['title']
+      feed.description = node['description']
+      feed.author = node['author']
+      feed.author.url = node['author_url']
+      feed.base_uri = site_url
+      feed.link = File.join(site_url, node.tree[node['link']].path)
+      feed.id = feed.link
+
+      feed.published = (node['created_at'].kind_of?(Time) ? node['created_at'] : Time.now)
+      feed.updated = Time.now
+      feed.generator = 'webgen - Webgen::SourceHandler::Feed'
+
+      node.feed_entries.each do |entry|
+        item = FeedTools::FeedItem.new
+        item.title = entry['title']
+        item.link = File.join(site_url, entry.path)
+        item.content = entry.node_info[:page].blocks[node['content_block_name'] || 'content'].render(Webgen::ContentProcessor::Context.new(:chain => [entry])).content
+        item.updated = entry['modified_at']
+        item.published = entry['created_at'] if entry['created_at'].kind_of?(Time)
+        if entry['author']
+          item.author = entry['author']
+          item.author.url = entry['author_url']
+        end
+        item.id = item.link
+        feed << item
+      end
+      feed
+    end
+
+    # Check if the any of the nodes used by this feed +node+ have changed and then mark the node as
+    # dirty.
+    def node_changed?(node)
+      return if node.node_info[:processor] != self.class.name
+      entries = node.feed_entries
+      node.flag(:dirty) if entries.map {|n| n.absolute_lcn } != website.cache[[:sourcehandler_feed, node.node_info[:src]]] ||
+        entries.any? {|n| n.changed?}
+    end
+
+  end
+
+end

data/lib/webgen/sourcehandler/fragment.rb

@@ -0,0 +1,68 @@
+# -*- encoding: utf-8 -*-
+
+module Webgen::SourceHandler
+
+  # Handles page fragment nodes and provides utility methods for parsing HTML headers and generating
+  # fragment nodes from them.
+  class Fragment
+
+    include Base
+    include Webgen::WebsiteAccess
+
+    HTML_HEADER_REGEXP = /<h([123456])(?:>|\s([^>]*)>)(.*?)<\/h\1\s*>/i
+    HTML_ATTR_REGEXP = /\s*(\w+)\s*=\s*('|")([^\2]+)\2\s*/
+
+    # Parse the string +content+ for headers +h1+, ..., +h6+ and return the found, nested sections.
+    #
+    # Only those headers are used which have an +id+ attribute set. The method returns a list of
+    # arrays with entries <tt>level, id, title, sub sections</tt> where <tt>sub sections</tt> is
+    # such a list again.
+    def parse_html_headers(content)
+      sections = []
+      stack = []
+      content.scan(HTML_HEADER_REGEXP).each do |level,attrs,title|
+        next if attrs.nil?
+        id_attr = attrs.scan(HTML_ATTR_REGEXP).find {|name,sep,value| name == 'id'}
+        next if id_attr.nil?
+        id = id_attr[2]
+
+        section = [level.to_i, id, title, []]
+        success = false
+        while !success
+          if stack.empty?
+            sections << section
+            stack << section
+            success = true
+          elsif stack.last.first < section.first
+            stack.last.last << section
+            stack << section
+            success = true
+          else
+            stack.pop
+          end
+        end
+      end
+      sections
+    end
+
+    # Create nested fragment nodes under +parent+ from +sections+ (which can be created using
+    # +parse_html_headers+). +path+ is the source path that defines the fragments. The meta
+    # information +in_menu+ of the fragment nodes is set to the parameter +in_menu+ and the meta
+    # info +sort_info+ is calculated from the base +si+ value.
+    def create_fragment_nodes(sections, parent, path, in_menu, si = 1000 )
+      sections.each do |level, id, title, sub_sections|
+        node = website.blackboard.invoke(:create_nodes, parent.tree, parent.absolute_lcn,
+                                         Webgen::Path.new('#' + id, path.source_path),
+                                         self) do |cn_parent, cn_path|
+          cn_path.meta_info['title'] = title
+          cn_path.meta_info['in_menu'] = in_menu
+          cn_path.meta_info['sort_info'] = si = si.succ
+          create_node(cn_parent, cn_path)
+        end.first
+        create_fragment_nodes(sub_sections, node, path, in_menu, si.succ)
+      end
+    end
+
+  end
+
+end