webgen 0.4.7 → 0.5.0

data/lib/webgen/source/resource.rb

@@ -0,0 +1,33 @@
+require 'webgen/websiteaccess'
+require 'webgen/source'
+
+module Webgen::Source
+
+  # This class is used to provide access to source provided by resources.
+  class Resource
+
+    include Webgen::WebsiteAccess
+
+    # The glob specifying the resources.
+    attr_reader :glob
+
+    # Create a new resource source for the the +glob+.
+    def initialize(glob)
+      @glob = glob
+    end
+
+    # Return all paths associated with the resources identified by #glob.
+    def paths
+      if !defined?(@paths)
+        stack = Stacked.new
+        website.config['resources'].select {|name, infos| File.fnmatch(@glob, name)}.sort.each do |name, infos|
+          stack.add([['/', constant(infos.first).new(*infos[1..-1])]])
+        end
+        @paths = stack.paths
+      end
+      @paths
+    end
+
+  end
+
+end

data/lib/webgen/source/stacked.rb

@@ -0,0 +1,53 @@
+module Webgen::Source
+
+  # This source class is used to stack several sources together.
+  #
+  # It serves two purposes:
+  #
+  # * First, it can be used to access more than one source. This is useful when your website
+  #   consists of more than one source directory and you want to use all of them.
+  #
+  # * Second, sources can mounted on specific directories. For example, a folder with images that
+  #   you don't want to copy to the website source directory can be mounted under <tt>/images</tt>
+  #   sothat they are available nonetheless.
+  #
+  # Also be aware that when a path is returned by a source that has already be returned by a prior
+  # source, it is discarded and not used.
+  class Stacked
+
+    # Return the stack of Webgen::Source objects.
+    attr_reader :stack
+
+    # Create a new stack. The optional +map+ parameter can be used to provide initial mappings of
+    # mount points to source objects (see #add for details).
+    def initialize(map = {})
+      @stack = []
+      add(map)
+    end
+
+    # Add all mappings found in +maps+ to the stack. The parameter +maps+ should be an array of
+    # two-element arrays which contain an absolute directoriy (ie. starting with a slash) and a
+    # source object.
+    def add(maps)
+      maps.each do |mp, source|
+        raise "Invalid mount point specified: #{mp}" unless mp =~ /^\//
+        @stack << [mp, source]
+      end
+    end
+
+    # Return all paths returned by the sources in the stack. Since the stack is ordered, paths
+    # returned by later source objects are not used if a prior source object has returned the same
+    # path.
+    def paths
+      @paths = Set.new
+      @stack.each do |mp, source|
+        source.paths.each do |path|
+          @paths.add?(path.mount_at(mp))
+        end
+      end
+      @paths
+    end
+
+  end
+
+end

data/lib/webgen/sourcehandler.rb

@@ -0,0 +1,175 @@
+require 'webgen/loggable'
+require 'benchmark'
+
+module Webgen
+
+  # Namespace for all classes that handle source paths.
+  #
+  # Have a look at Webgen::SourceHandler::Base for details on how to implement a source handler
+  # class.
+  module SourceHandler
+
+    autoload :Base, 'webgen/sourcehandler/base'
+    autoload :Copy, 'webgen/sourcehandler/copy'
+    autoload :Directory, 'webgen/sourcehandler/directory'
+    autoload :Metainfo, 'webgen/sourcehandler/metainfo'
+    autoload :Template, 'webgen/sourcehandler/template'
+    autoload :Page, 'webgen/sourcehandler/page'
+    autoload :Fragment, 'webgen/sourcehandler/fragment'
+    autoload :Virtual, 'webgen/sourcehandler/virtual'
+
+    # This class is used by Website to do the actual rendering of the website. It
+    #
+    # * collects all source paths using the source classes
+    # * creates nodes using the source handler classes
+    # * writes changed nodes out using an output class
+    class Main
+
+      include WebsiteAccess
+      include Loggable
+
+      def initialize #:nodoc:
+        website.blackboard.add_service(:create_nodes, method(:create_nodes))
+        website.blackboard.add_service(:source_paths, method(:find_all_source_paths))
+      end
+
+      # Render the nodes provided in the +tree+. Before the actual rendering is done, the sources
+      # are checked (nodes for deleted sources are deleted, nodes for new and changed sources).
+      def render(tree)
+        # Add new and changed nodes, remove nodes of deleted paths
+        puts "Generating tree..."
+        time = Benchmark.measure do
+          used_paths = Set.new
+          paths = Set.new([nil])
+          while paths.length > 0
+            used_paths += (paths = Set.new(find_all_source_paths.keys) - used_paths - clean(tree))
+            create_nodes_from_paths(tree, paths)
+          end
+        end
+        puts "...done in " + ('%2.4f' % time.real) + ' seconds'
+
+        output = website.blackboard.invoke(:output_instance)
+
+        # Enable permanent (till end of run) storing of volatile information
+        website.cache.enable_volatile_cache
+
+        puts "Writing changed nodes..."
+        time = Benchmark.measure do
+          tree.node_access[:alcn].sort.each do |name, node|
+            next if node == tree.dummy_root || !node.dirty
+            node.dirty = node.created = false
+            if !node['no_output'] && (content = node.content)
+              puts " "*4 + name, :verbose
+              type = if node.is_directory?
+                       :directory
+                     elsif node.is_fragment?
+                       :fragment
+                     else
+                       :file
+                     end
+              output.write(node.path, content, type)
+            end
+          end
+        end
+        puts "...done in " + ('%2.4f' % time.real) + ' seconds'
+      end
+
+      #######
+      private
+      #######
+
+      # Return a hash with all source paths.
+      def find_all_source_paths
+        if !@paths
+          source = Webgen::Source::Stacked.new(website.config['sources'].collect do |mp, name, *args|
+                                                 [mp, constant(name).new(*args)]
+                                               end)
+          @paths = {}
+          source.paths.each do |path|
+            if !(website.config['sourcehandler.ignore'].any? {|pat| File.fnmatch(pat, path, File::FNM_CASEFOLD|File::FNM_DOTMATCH)})
+              @paths[path.path] = path
+            end
+          end
+        end
+        @paths
+      end
+
+      # Return only the subset of +paths+ which are handled by the source handler +name+.
+      def paths_for_handler(name, paths)
+        patterns = website.config['sourcehandler.patterns'][name]
+        return [] if patterns.nil?
+
+        options = (website.config['sourcehandler.casefold'] ? File::FNM_CASEFOLD : 0) |
+          (website.config['sourcehandler.use_hidden_files'] ? File::FNM_DOTMATCH : 0)
+        find_all_source_paths.values_at(*paths).select do |path|
+          patterns.any? {|pat| File.fnmatch(pat, path, options)}
+        end
+      end
+
+      # Use the source handlers to create nodes for the +paths+ in the +tree+.
+      def create_nodes_from_paths(tree, paths)
+        website.config['sourcehandler.invoke'].sort.each do |priority, shns|
+          shns.each do |shn|
+            sh = website.cache.instance(shn)
+            paths_for_handler(shn, paths).sort.each do |path|
+              parent_dir = path.directory.split('/').collect {|p| Path.new(p).cn}.join('/')
+              parent_dir += '/' if path != '/' && parent_dir == ''
+              create_nodes(tree, parent_dir, path, sh)
+            end
+          end
+        end
+      end
+
+      # Prepare everything to create nodes under the absolute lcn path +parent_path_name+ in the
+      # +tree from the +path+ using the +source_handler+. If a block is given, the actual creation
+      # of the nodes is deferred to it. After the nodes are created, it is also checked if they have
+      # all needed properties.
+      def create_nodes(tree, parent_path_name, path, source_handler) #:yields: parent, path
+        if !(parent = tree[parent_path_name])
+          raise "The specified parent path <#{parent_path_name}> does not exist"
+        end
+        path = path.dup
+        path.meta_info.update(website.config['sourcehandler.default_meta_info'][:all])
+        path.meta_info.update(website.config['sourcehandler.default_meta_info'][source_handler.class.name] || {})
+        website.blackboard.dispatch_msg(:before_node_created, parent, path)
+        *nodes = if block_given?
+                   yield(parent, path)
+                 else
+                   source_handler.create_node(parent, path.dup)
+                 end
+        nodes.compact.each do |node|
+          website.blackboard.dispatch_msg(:after_node_created, node)
+        end
+        nodes
+      end
+
+
+      # Clean the +tree+ by deleting nodes which have changed or which don't have an associated
+      # source anymore.
+      def clean(tree)
+        paths_to_delete = Set.new
+        paths_not_to_delete = Set.new
+        tree.node_access[:alcn].each do |alcn, node|
+          next if node == tree.dummy_root || tree[alcn].nil?
+
+          deleted = !find_all_source_paths.include?(node.node_info[:src])
+          if !node.created && (deleted ||
+                               find_all_source_paths[node.node_info[:src]].changed? ||
+                               node.changed?)
+            paths_not_to_delete << node.node_info[:src]
+            tree.delete_node(node, deleted)
+            #TODO: delete output path
+          else
+            paths_to_delete << node.node_info[:src]
+          end
+        end
+
+        # source paths that should be used
+        paths_to_delete - paths_not_to_delete
+      end
+
+    end
+
+  end
+
+end

data/lib/webgen/sourcehandler/base.rb

@@ -0,0 +1,188 @@
+require 'webgen/websiteaccess'
+require 'webgen/loggable'
+require 'webgen/page'
+
+module Webgen::SourceHandler
+
+  # This module should be included in every source handler as it provides the default methods for
+  # creating nodes.
+  #
+  # = Implementing Source Handlers
+  #
+  # A source handler is a webgen extension that processes source paths to create nodes and that
+  # provides the rendered content of these nodes. The nodes are later written to the output
+  # location. This can range from simply copying a path from the source to the output location to
+  # generating a whole set of nodes from one input path!
+  #
+  # The paths that are handled by a source handler are specified via path patterns (see
+  # below). During a webgen run the #create_node method for each source paths that matches a
+  # specified path pattern is called. And when it is time to write out the node, the the #content
+  # method is called to retrieve the rendered content.
+  #
+  # A source handler must not take any parameters on initialization and when this module is not
+  # mixed in, the methods #create_node and #content need to be defined. Also, a source handler does
+  # not need to reside under the Webgen::SourceHandler namespace but all shipped ones do.
+  #
+  # This base class provides useful default implementations of methods that are used by nearly all
+  # source handler class:
+  # * #create_node
+  # * #node_exists?
+  #
+  # It also provides other utility methods:
+  # * #page_from_path
+  # * #content
+  #
+  # = Nodes Created for Paths
+  #
+  # The main functions of a source handler class are to create one or more nodes for a source path
+  # and to provide the content of these nodes. To achieve this, certain information needs to be set
+  # on a node. If you use the methods provided by this base class, you don't need to set them
+  # explicitly because this is done by the provided methods.
+  #
+  # <tt>node_info[:processor]</tt>:: Has to be set to the class name of the source handler. This is
+  #                                  used by the Node class: all unknown method calls are forwarded
+  #                                  to the node processor.
+  # <tt>node_info[:src]</tt>:: Has to be set to the string version of the source path.
+  # <tt>meta_info['no_output']</tt>:: Has to be set to +true+ on nodes that are used during a
+  #                                   webgen run but do not produce an output file.
+  #
+  # Additional information that is used only for processing purposes should be stored in the
+  # #node_info hash of a node as the #meta_info hash is reserved for real node meta information and
+  # should not be changed once the node is created.
+  #
+  # = Path Patterns and Invocation order
+  #
+  # Path patterns define which paths are handled by a specific source handler. These patterns are
+  # specified in the <tt>sourcehandler.patterns</tt> configuration hash as a mapping from the source
+  # handler class name to an array of path patterns. The patterns need to have a format that
+  # <tt>Dir.glob</tt> can handle.
+  #
+  # Specifying a path pattern does not mean that webgen uses the source handler. One also needs to
+  # provide an entry in the configuration value <tt>sourcehandler.invoke</tt>. This is a hash that
+  # maps the invocation rank (a number) to an array of source handler class names. The lower the
+  # invocation rank the earlier the specified source handlers are used.
+  #
+  # The default invocation ranks are:
+  # 1:: Early. Normally there is no need to use this rank.
+  # 5:: Standard. This is the rank the normal source handler should use.
+  # 9:: Late. This rank should be used by source handlers that operate on/use already created nodes
+  #     and need to ensure that these nodes are available.
+  #
+  # = Default Meta Information
+  #
+  # Each source handler can define default meta information that gets automatically set on the
+  # source paths that are passed to the #create_node method.
+  #
+  # The default meta information is specified in the <tt>sourcehandler.default_meta_info</tt>
+  # configuration hash as a mapping from the source handler class name to the meta information
+  # hash.
+  #
+  # = Sample Source Handler Class
+  #
+  # Following is a simple source handler class example which copies paths from the source to
+  # the output location modifying the extension:
+  #
+  #   class SimpleCopy
+  #
+  #     include Webgen::SourceHandler::Base
+  #
+  #     def create_node(parent, path)
+  #       path.ext += '.copied'
+  #       super(parent, path)
+  #     end
+  #
+  #     def content(node)
+  #       website.blackboard.invoke(:source_paths)[node.node_info[:src]].io
+  #     end
+  #
+  #   end
+  #
+  #   WebsiteAccess.website.config.sourcehandler.patterns['SimpleCopy'] = ['**/*.jpg', '**/*.png']
+  #   WebsiteAccess.website.config.sourcehandler.invoke[5] << 'SimpleCopy'
+  #
+  module Base
+
+    include Webgen::Loggable
+
+    # Construct the output name for the given +path+. First it is checked if a node with the
+    # constructed output name already exists. If it exists, the language part is forced to be in the
+    # output name and the resulting output name is returned.
+    #
+    # The parameter +style+ (which uses the meta information +output_path_style+ from the path's
+    # meta information hash) defines how the output name should be built (more information about
+    # this in the user documentation).
+    def output_path(parent, path, style = path.meta_info['output_path_style'])
+      name = construct_output_path(parent, path, style)
+      name += '/'  if path.path =~ /\/$/ && name !~ /\/$/
+      if node_exists?(parent, path, name)
+        name = construct_output_path(parent, path, style, true)
+        name += '/'  if path.path =~ /\/$/ && name !~ /\/$/
+      end
+      name
+    end
+
+    # Utility method for constructing the output name.
+    def construct_output_path(parent, path, style, use_lang_part = nil)
+      use_lang_part = if path.meta_info['lang'].nil? # unlocalized files never get a lang in the filename!
+                        false
+                      elsif use_lang_part.nil?
+                        Webgen::WebsiteAccess.website.config['sourcehandler.default_lang_in_output_path'] ||
+                          Webgen::WebsiteAccess.website.config['website.lang'] != path.meta_info['lang']
+                      else
+                        use_lang_part
+                      end
+      result = style.collect do |part|
+        case part
+        when String  then part
+        when :lang   then use_lang_part ? path.meta_info['lang'] : ''
+        when :ext    then path.ext.empty? ? '' : '.' + path.ext
+        when :parent then temp = parent; temp = temp.parent while temp.is_fragment?; temp.path
+        when Symbol  then path.send(part)
+        when Array   then part.include?(:lang) && !use_lang_part ? '' : construct_output_path(parent, path, part, use_lang_part)
+        else ''
+        end
+      end
+      result.join('')
+    end
+    private :construct_output_path
+
+    # Check if the node alcn and output path which would be created by #create_node exists. The
+    # +output_path+ to check for can individually be set.
+    def node_exists?(parent, path, output_path = self.output_path(parent, path))
+      parent.tree[Webgen::Node.absolute_name(parent, path.lcn, :alcn)] || (!path.meta_info['no_output'] && parent.tree[output_path, :path])
+    end
+
+    # Create and return a node under +parent+ from +path+ if it does not already exists. The created
+    # node is yielded if a block is given.
+    def create_node(parent, path, output_path = self.output_path(parent, path))
+      if !node_exists?(parent, path, output_path)
+        node = Webgen::Node.new(parent, output_path, path.cn, path.meta_info)
+        node.node_info[:src] = path.path
+        node.node_info[:processor] = self.class.name
+        yield(node) if block_given?
+        node
+      else
+        log(:warn) { "Node already exists: output path = #{output_path} | alcn = #{Webgen::Node.absolute_name(parent, path.lcn, :alcn)}"}
+      end
+    end
+
+    # Return the content of the given +node+. This default +content+ method just returns +nil+.
+    def content(node)
+      nil
+    end
+
+    # Utility method for creating a Webgen::Page object from the +path+. Also updates
+    # <tt>path.meta_info</tt> with the meta info from the page.
+    def page_from_path(path)
+      begin
+        page = Webgen::Page.from_data(path.io.data, path.meta_info)
+      rescue Webgen::WebgenPageFormatError => e
+        raise "Error reading source path <#{path}>: #{e.message}"
+      end
+      path.meta_info = page.meta_info
+      page
+    end
+
+  end
+
+end