webgen 1.0.0.beta3 → 1.0.0

data/lib/webgen/node.rb

@@ -216,7 +216,7 @@ module Webgen
     # Return all versions of this node.
     def versions
       tree.node_access[:alcn].select {|alcn, n| n.node_info[:path] == node_info[:path]}.
-        each_with_object({}) {|(k, v), h| h[v['version']] = v}
+        each_with_object({}) {|(_, v), h| h[v['version']] = v}
     end
 
   end

data/lib/webgen/node_finder.rb

@@ -21,116 +21,22 @@ module Webgen
   #
   # == Finder options
   #
-  # Following is the list of all finder options. Note that there may also be other 3rd party node
-  # filters available if you are using extension bundles!
-  #
-  # === Non-filter options
-  #
-  # These options are not used for filtering out nodes but provide additional functionality.
-  #
-  # [:limit]
-  #   Value: an integer. Specifies the maximum number of nodes that should be returned. Implies
-  #   'flatten = true'.
-  #
-  #   Note that fewer nodes may be returned if fewer nodes match the filter criterias.
-  #
-  # [:offset]
-  #   Value: an integer. Specifies how many nodes from the front of the list should *not* be
-  #   returned. Implies 'flatten = true'.
-  #
-  # [:levels]
-  #   Value: one integer (is used as start and end level) or an array with two integers (the start
-  #   and end levels). All nodes whose hierarchy level in the returned node hierarchy is greater
-  #   than or equal to the start level and lower than or equal to the end level are used.
-  #
-  #   Only used when the node hierarchy is not flattened.
-  #
-  # [:flatten]
-  #   Value: anything except +nil+ or +false+. A flat list of nodes is returned if this option is
-  #   set, otherwise the nodes are returned in their correct hierarchical order using nested lists.
-  #
-  #   Note that any missing nodes in the hierarchy are automatically added so that traversing the
-  #   hierarchy is always possible. For example, if we have the tree '/a/b/c' and only nodes +a+ and
-  #   +c+ are found, node +b+ is automatically added.
-  #
-  # [:sort]
-  #   Value: +nil+/+false+, +true+ or a meta information key. If +nil+ or +false+ is specified, no
-  #   sorting is performed. If +true+ is specified, the meta information +sort_info+ (or if absent,
-  #   the meta information +title+) is used for sorting. If the compared values are both integers, a
-  #   numeric comparison is done, else a string comparison. If a meta information key is specified,
-  #   the value of this meta information is used for comparison of nodes (again, if both compared
-  #   values are integers, a numeric comparison is done, else a string comparison).
-  #
-  # [:reverse]
-  #   Value: +true+ of +false+/+nil+. If this option is set to +true+, the sort order is reversed.
-  #
-  # === Filter options
-  #
-  # These options are used for filtering the nodes. All nodes are used by default if no filter
-  # options are specified.
-  #
-  # [:alcn]
-  #   Value: an alcn pattern or an array of alcn patterns. Nodes that match any of the patterns are
-  #   used.
-  #
-  # [:lang]
-  #   Value: a language code/+nil+/the special value +node+ or an array of these values. Nodes that
-  #   have one of the specified language codes, are language independent (in case of the value
-  #   +nil+) or have the same language as the reference node (in case of the value +node+) are
-  #   used.
-  #
-  # [:mi]
-  #   Value: a hash with meta information key to value mappings. Only nodes that have the same
-  #   values for all meta information keys are used.
-  #
-  # [:or]
-  #   Value: a finder option set or an array of finder options sets (specifying option set names is
-  #   also possible). Nodes that appear in any specified option set are additionally used.
-  #
-  # [:and]
-  #   Value: a finder option set or an array of finder options sets (specifying option set names is
-  #   also possible). Only nodes that appear in all specified option sets are used.
-  #
-  # [:not]
-  #   Value: a finder option set or an array of finder options sets (specifying option set names is
-  #   also possible). Only nodes that do not appear in any specified option set are used.
-  #
-  # [:absolute_levels]
-  #   Value: one integer (is used as start and end level) or an array with two integers (the start
-  #   and end levels). All nodes whose hierarchy level in the node tree are greater than or equal to
-  #   the start level and lower than or equal to the end level are used.
-  #
-  #   Negative numbers, where -1 stands for the reference node, -2 for its parent node and so on,
-  #   can also be used.
-  #
-  # [:ancestors]
-  #   Value: +true+ or +false+/+nil+. If this filter option is set to +true+, only nodes that are
-  #   ancestors of the reference node are used. The reference node itself is used as well.
-  #
-  # [:descendants]
-  #   Value: +true+ or +false+/+nil+. If this filter option is set to +true+, only nodes that are
-  #   descendants of the reference node are used. The reference node itself is used as well.
-  #
-  # [:siblings]
-  #   Value: +true+, +false+/+nil+, or an array with two integers. If this filter option is set to
-  #   +true+, only nodes that are sibling node of the reference node are used. The reference node
-  #   itself is used as well. If set to +false+ or +nil+, this filter is ignored.
-  #
-  #   If an array with two numbers is specified, all sibling nodes of the reference node or its
-  #   parent nodes the hierarchy level of which lies between these numbers are used. The parent
-  #   nodes and the reference node are used as well if their level lies between the numbers.
-  #   Counting starts at zero (the root node).
-  #
-  #   Negative numbers, where -1 stands for the reference node, -2 for its parent node and so on,
-  #   can also be used.
+  # A complete list of the supported finder options can be found in the user documentation! Note
+  # that there may also be other 3rd party node filters available if you are using extension
+  # bundles!
   #
   # == Implementing a filter module
   #
   # Implementing a filter module is very easy. Just create a module that contains your filter
   # methods and tell the NodeFinder object about it using the #add_filter_module method. A filter
-  # method needs to take three arguments: an array of nodes, the reference node and the filter
+  # method needs to take three arguments: the Result stucture, the reference node and the filter
   # value.
   #
+  # The +result.nodes+ accessor contains the array of nodes that should be manipulated in-place.
+  #
+  # If a filter uses the reference node in any way, it has to set +result.ref_node_used+ to +true+
+  # to allow proper caching!
+  #
   # Here is a sample filter module which provides the ability to filter nodes based on the meta
   # information key +category+. The +category+ key contains an array with one or more categories.
   # The value for this category filter is one or more strings and the filter returns those nodes
@@ -138,9 +44,9 @@ module Webgen
   #
   #   module CategoryFilter
   #
-  #     def filter_on_category(nodes, ref_node, categories)
+  #     def filter_on_category(result, ref_node, categories)
   #       categories = [categories].flatten # needed in case categories is a string
-  #       nodes.select {|n| categories.any? {|c| n['category'].include?(c)}}
+  #       result.nodes.select! {|n| categories.any? {|c| n['category'].include?(c)}}
   #     end
   #
   #   end
@@ -149,6 +55,11 @@ module Webgen
   #
   class NodeFinder
 
+    # Result class used when filtering the nodes.
+    #
+    # The attribute +ref_node_used+ must not be set to +false+ once it is +true+!
+    Result = Struct.new(:nodes, :ref_node_used)
+
     # Create a new NodeFinder object for the given website.
     def initialize(website)
       @website = website
@@ -196,7 +107,7 @@ module Webgen
       flatten = true if limit || offset
       levels = [levels || [1, 1_000_000]].flatten.map {|i| i.to_i}
 
-      nodes = filter_nodes(opts, ref_node)
+      nodes, ref_node_used = filter_nodes(opts, ref_node)
 
       if flatten
         sort_nodes(nodes, sort, reverse)
@@ -226,7 +137,7 @@ module Webgen
         sort_nodes(nodes, sort, reverse, false)
       end
 
-      cache_result(opts_or_name, ref_node, nodes)
+      cache_result(opts_or_name, ref_node, nodes, ref_node_used)
     end
 
     #######
@@ -234,11 +145,19 @@ module Webgen
     #######
 
     def cached_result(opts, ref_node)
-      (@website.cache.volatile[:node_finder] ||= {})[[opts, ref_node.alcn]]
+      result_cache[opts] || result_cache[[opts, ref_node.alcn]]
     end
 
-    def cache_result(opts, ref_node, result)
-      (@website.cache.volatile[:node_finder] ||= {})[[opts, ref_node.alcn]] = result
+    def cache_result(opts, ref_node, result, ref_node_used)
+      if ref_node_used
+        result_cache[[opts, ref_node.alcn]] = result
+      else
+        result_cache[opts] = result
+      end
+    end
+
+    def result_cache
+      @website.cache.volatile[:node_finder] ||= {}
     end
 
     def prepare_options_hash(opts_or_name)
@@ -260,15 +179,17 @@ module Webgen
       nodes = @website.tree.node_access[:alcn].values
       nodes.delete(@website.tree.dummy_root)
 
+      result = Result.new(nodes, false)
+
       opts.each do |filter, value|
         if @mapping.has_key?(filter)
-          nodes = send(@mapping[filter], nodes, ref_node, value)
+          send(@mapping[filter], result, ref_node, value)
         else
           @website.logger.warn { "Ignoring unknown node finder filter '#{filter}'" }
         end
       end
 
-      nodes
+      [result.nodes, result.ref_node_used]
     end
 
     def sort_nodes(nodes, sort, reverse, flat_mode = true)
@@ -292,78 +213,102 @@ module Webgen
 
     # :section: Filter methods
 
-    def filter_and(nodes, ref_node, opts)
+    def filter_and(result, ref_node, opts)
       [opts].flatten.each do |cur_opts|
         cur_opts = prepare_options_hash(cur_opts)
         remove_non_filter_options(cur_opts)
-        nodes &= filter_nodes(cur_opts, ref_node)
+        nodes, ref_node_used = filter_nodes(cur_opts, ref_node)
+        result.nodes &= nodes
+        result.ref_node_used |= ref_node_used
       end
-      nodes
     end
 
-    def filter_or(nodes, ref_node, opts)
+    def filter_or(result, ref_node, opts)
       [opts].flatten.each do |cur_opts|
         cur_opts = prepare_options_hash(cur_opts)
         remove_non_filter_options(cur_opts)
-        nodes |= filter_nodes(cur_opts, ref_node)
+        nodes, ref_node_used = filter_nodes(cur_opts, ref_node)
+        result.nodes |= nodes
+        result.ref_node_used |= ref_node_used
       end
-      nodes
     end
 
-    def filter_not(nodes, ref_node, opts)
+    def filter_not(result, ref_node, opts)
       [opts].flatten.each do |cur_opts|
         cur_opts = prepare_options_hash(cur_opts)
         remove_non_filter_options(cur_opts)
-        nodes -= filter_nodes(cur_opts, ref_node)
+        nodes, ref_node_used = filter_nodes(cur_opts, ref_node)
+        result.nodes -= nodes
+        result.ref_node_used |= ref_node_used
       end
-      nodes
     end
 
-    def filter_meta_info(nodes, ref_node, mi)
-      nodes.keep_if {|n| mi.all? {|key, val| n[key] == val}}
+    def filter_meta_info(result, ref_node, mi)
+      result.nodes.keep_if {|n| mi.all? {|key, val| n[key] == val}}
     end
 
-    def filter_alcn(nodes, ref_node, alcn)
+    def filter_alcn(result, ref_node, alcn)
+      result.ref_node_used = true
       alcn = [alcn].flatten.map {|a| Webgen::Path.append(ref_node.alcn, a.to_s)}
-      nodes.keep_if {|n| alcn.any? {|a| n =~ a}}
+      result.nodes.keep_if {|n| alcn.any? {|a| n =~ a}}
     end
 
-    def filter_absolute_levels(nodes, ref_node, range)
-      range = [range].flatten.map {|i| (i = i.to_i) < 0 ? ref_node.level + 1 + i : i}
-      nodes.keep_if {|n| n.level >= range.first && n.level <= range.last}
+    def filter_absolute_levels(result, ref_node, range)
+      range = [range].flatten.map do |i|
+        if (i = i.to_i) < 0
+          result.ref_node_used = true
+          ref_node.level + 1 + i
+        else
+          i
+        end
+      end
+      result.nodes.keep_if {|n| n.level >= range.first && n.level <= range.last}
     end
 
-    def filter_lang(nodes, ref_node, langs)
-      langs = [langs].flatten.map {|l| l == 'node' ? ref_node.lang : l}.uniq
-      nodes.keep_if {|n| langs.any? {|l| n.lang == l}}
+    def filter_lang(result, ref_node, langs)
+      langs = [langs].flatten.map do |l|
+        if l == 'node'
+          result.ref_node_used = true
+          ref_node.lang
+        else
+          l
+        end
+      end.uniq
+      result.nodes.keep_if {|n| langs.any? {|l| n.lang == l}}
     end
 
-    def filter_ancestors(nodes, ref_node, enabled)
-      return nodes unless enabled
+    def filter_ancestors(result, ref_node, enabled)
+      return unless enabled
+      result.ref_node_used = true
+
       nodes = []
       node = ref_node
       until node == node.tree.dummy_root
         nodes.unshift(node)
         node = node.parent
       end
-      nodes
+      result.nodes = nodes
     end
 
-    def filter_descendants(nodes, ref_node, enabled)
-      return nodes unless enabled
-      nodes.keep_if do |n|
+    def filter_descendants(result, ref_node, enabled)
+      return unless enabled
+      result.ref_node_used = true
+
+      result.nodes.keep_if do |n|
         n = n.parent while n != n.tree.dummy_root && n != ref_node
         n == ref_node
       end
     end
 
-    def filter_siblings(nodes, ref_node, value)
-      return nodes unless value
+    def filter_siblings(result, ref_node, value)
+      return unless value
+      result.ref_node_used = true
+
       if value == true
-        nodes.keep_if {|n| n.parent == ref_node.parent}
+        result.nodes.keep_if {|n| n.parent == ref_node.parent}
       else
         value = [value].flatten.map {|i| (i = i.to_i) < 0 ? ref_node.level + 1 + i : i}
-        nodes.keep_if do |n|
+        result.nodes.keep_if do |n|
           n.level >= value.first && n.level <= value.last && (n.parent.is_ancestor_of?(ref_node) || n.is_root?)
         end
       end

data/lib/webgen/page.rb

@@ -18,11 +18,11 @@ module Webgen
 
     # :stopdoc:
     RE_NEWLINE = /\r?\n/
-    RE_META_INFO_START = /\A---\s*#{RE_NEWLINE}/
+    RE_META_INFO_START = /\A---[ \t]*#{RE_NEWLINE}/
     RE_META_INFO = /#{RE_META_INFO_START}.*?#{RE_NEWLINE}(?=---.*?#{RE_NEWLINE}|\Z)/m
-    RE_BLOCKS_START_SIMPLE = /^--- (\w+)(?:\s*| -+\s*)$|^$/
-    RE_BLOCKS_START_COMPLEX = /^--- *?(?: *((?:\w+:[^\s]* *)*))?-*\s*$|^$/
-    RE_BLOCKS_START = /^---(?: .*?|)(?=#{RE_NEWLINE})/
+    RE_BLOCKS_START_SIMPLE = /^---[ \t]*$|^---[ \t]+(\w+)[ \t]*(?:[ \t]+-+[ \t]*)?$|^$/
+    RE_BLOCKS_START_COMPLEX = /^---[ \t]+?(?:[ \t]*((?:\w+:\S*[ \t]*)*))?(?:[ \t]+-+[ \t]*)?$/
+    RE_BLOCKS_START = /^---(?:[ \t]+.*?|)(?=#{RE_NEWLINE})/
     RE_BLOCKS = /(?:(#{RE_BLOCKS_START})|\A)#{RE_NEWLINE}?(.*?)(?:(?=#{RE_BLOCKS_START})|\z)/m
     RE_PAGE = /(#{RE_META_INFO})?(.*)/m
     # :startdoc:
@@ -53,7 +53,7 @@ module Webgen
             unless meta_info.kind_of?(Hash)
               raise FormatError, "Invalid structure of meta information block: expected YAML hash but found #{meta_info.class}"
             end
-          rescue ArgumentError, SyntaxError => e
+          rescue ArgumentError, SyntaxError, YAML::SyntaxError => e
             raise FormatError, "Invalid YAML syntax in meta information block: #{e.message}"
           end
           meta_info

data/lib/webgen/path.rb

@@ -64,11 +64,14 @@ module Webgen
 
     # Return +true+ if the given path string matches the given path pattern.
     #
+    # If a fragment path (i.e. one which has a hash character somewhere) should be matched, the
+    # pattern needs to have a hash character as well.
+    #
     # For information on which patterns are supported, have a look at the API documentation of
     # File.fnmatch.
     def self.matches_pattern?(path, pattern, options = File::FNM_DOTMATCH|File::FNM_CASEFOLD|File::FNM_PATHNAME)
       pattern += '/' if path =~ /\/$/ && pattern !~ /\/$|^$/
-      File.fnmatch(pattern, path, options)
+      (path.to_s.include?('#') ? pattern.include?('#') : true) && File.fnmatch(pattern, path, options)
     end
 
     # Construct a localized canonical name from a given canonical name and a language.

data/lib/webgen/path_handler.rb

@@ -17,14 +17,10 @@ module Webgen
   # source to the destination, or a complex things, like generating a whole set of nodes from one
   # input path (e.g. generating a whole image gallery)!
   #
-  # The paths that are handled by a path handler are specified via path patterns (see below). The
-  # #create_nodes method of a path handler is called for each source path that matches a specified
-  # path pattern. And when it is time to write out a node, the #content method on the associated
-  # path handler is called to retrieve the rendered content of the node.
-  #
-  # Also note that any method invoked on a Node object that is not defined in the Node class itself
-  # is forwarded to the associated path handler, adding the node as first parameter to the parameter
-  # list.
+  # The paths that are handled by a path handler are generally specified via path patterns. The
+  # #create_nodes method of a path handler is called for each source path that should be handled.
+  # And when it is time to write out a node, the #content method on the path handler associated with
+  # the node is called to retrieve the rendered content of the node.
   #
   # === Tree creation
   #
@@ -40,10 +36,11 @@ module Webgen
   #    created before their file nodes.
   #
   # 2. When a path handler is used for creating nodes, all source paths (retrieved by using
-  #    Webgen::Source#paths method) that match one of the associated patterns are used.
+  #    Webgen::Source#paths method) that match one of the associated patterns and/or all path with
+  #    the 'handler' meta information set to the path handler are used.
   #
-  # 3. The meta information of a used source path is then updated with the meta information from the
-  #    'path_handler.default_meta_info' configuration option key.
+  # 3. The meta information of a used source path is then updated with the meta information applied
+  #    by methods registered for the :apply_meta_info_to_path blackboard message.
   #
   #    After that the source path is given to the #parse_meta_info! method of the path handler so
   #    that meta information of the path can be updated with meta information stored in the content
@@ -53,7 +50,7 @@ module Webgen
   #    should be used for creating nodes and each path version is then given to the #create_nodes
   #    method of the path handler so that it can create one or more nodes.
   #
-  # 4. Nodes returned by the #creates_nodes of a path handler are assumed to have the Node#node_info
+  # 4. Nodes returned by #creates_nodes of a path handler are assumed to have the Node#node_info
   #    keys :path and :path_handler and the meta info key 'modified_at' correctly set (this is
   #    automatically done if the Webgen::PathHandler::Base#create_node method is used).
   #
@@ -61,7 +58,8 @@ module Webgen
   #
   # Path patterns define which paths are handled by a specific path handler. These patterns are
   # specified when a path handler is registered using #register method. The patterns need to have a
-  # format that Dir.glob can handle.
+  # format that Dir.glob can handle. Note that a user can always associate any path with a path
+  # handler through a meta information path and the 'handler' meta information key.
   #
   # In addition to specifying the patterns a path handler uses, one can also specify the place in
   # the invocation list which the path handler should use. The invocation list is used from the
@@ -141,23 +139,23 @@ module Webgen
       @instances = {}
       @secondary_nodes = {}
 
-      @website.blackboard.add_listener(:website_generated, self) do
+      @website.blackboard.add_listener(:website_generated, 'path_handler') do
         @website.cache[:path_handler_secondary_nodes] = @secondary_nodes
       end
 
       used_secondary_paths = {}
       written_nodes = Set.new
-      @website.blackboard.add_listener(:before_secondary_nodes_created, self) do |path, source_alcn|
+      @website.blackboard.add_listener(:before_secondary_nodes_created, 'path_handler') do |path, source_alcn|
         (used_secondary_paths[source_alcn] ||= Set.new) << path if source_alcn
       end
-      @website.blackboard.add_listener(:before_all_nodes_written, self) do |node|
+      @website.blackboard.add_listener(:before_all_nodes_written, 'path_handler') do |node|
         used_secondary_paths = {}
         written_nodes = Set.new
       end
-      @website.blackboard.add_listener(:after_node_written, self) do |node|
+      @website.blackboard.add_listener(:after_node_written, 'path_handler') do |node|
         written_nodes << node.alcn
       end
-      @website.blackboard.add_listener(:after_all_nodes_written, self) do |node|
+      @website.blackboard.add_listener(:after_all_nodes_written, 'path_handler') do
         @secondary_nodes.delete_if do |path, data|
           if written_nodes.include?(data[1]) && (!used_secondary_paths[data[1]] ||
                                                  !used_secondary_paths[data[1]].include?(path))
@@ -225,10 +223,16 @@ module Webgen
 
       time = Benchmark.measure do
         meta_info, rest = @website.ext.source.paths.partition {|path| path.path =~ /[\/.]metainfo$/}
+
+        used_paths = []
+
+        @website.blackboard.add_listener(:before_node_created, 'path_handler (temp_populate_tree)') do |path|
+          used_paths << path
+        end
         create_nodes(meta_info, [:meta_info])
         create_nodes(rest)
+        @website.blackboard.remove_listener(:before_node_created, 'path_handler (temp_populate_tree)')
 
-        used_paths = @website.tree.node_access[:alcn].values.map {|n| n.node_info[:path]}
         unused_paths = rest - used_paths
         @website.logger.vinfo do
           "The following source paths have not been used: #{unused_paths.join(', ')}"
@@ -258,11 +262,11 @@ module Webgen
         at_least_one_node_written = false
         @website.cache.reset_volatile_cache
         @website.blackboard.dispatch_msg(:before_all_nodes_written)
-        @website.tree.node_access[:alcn].sort.each do |name, node|
+        @website.tree.node_access[:alcn].sort_by {|a, n| [n['write_order'].to_s, a]}.each do |name, node|
           begin
             next if node == @website.tree.dummy_root ||
               (node['passive'] && !node['no_output'] && !@website.ext.item_tracker.node_referenced?(node)) ||
-              ((@website.config['website.dry_run'] || @website.ext.destination.exists?(node.dest_path)) &&
+              ((@website.config['website.dry_run'] || node['no_output'] || @website.ext.destination.exists?(node.dest_path)) &&
                !@website.ext.item_tracker.node_changed?(node))
 
             @website.blackboard.dispatch_msg(:before_node_written, node)