webgen 0.5.8 → 0.5.9

data/lib/webgen/source/stacked.rb

@@ -17,20 +17,27 @@ module Webgen::Source
   # source, it is discarded and not used.
   class Stacked
 
-    # Return the stack of Webgen::Source objects.
+    # Return the stack of mount point to Webgen::Source object maps.
     attr_reader :stack
 
+    # Specifies whether the result of #paths calls should be cached (default: +false+). If caching
+    # is activated, new maps cannot be added to the stacked source anymore!
+    attr_accessor :cache_paths
+
     # Create a new stack. The optional +map+ parameter can be used to provide initial mappings of
-    # mount points to source objects (see #add for details).
-    def initialize(map = {})
+    # mount points to source objects (see #add for details). You cannot add other maps after a call
+    # to #paths if +cache_paths+ is +true+
+    def initialize(map = {}, cache_paths = false)
       @stack = []
+      @cache_paths = cache_paths
       add(map)
     end
 
     # Add all mappings found in +maps+ to the stack. The parameter +maps+ should be an array of
-    # two-element arrays which contain an absolute directoriy (ie. starting with a slash) and a
-    # source object.
+    # two-element arrays which contain an absolute directory (ie. starting and ending with a slash)
+    # and a source object.
     def add(maps)
+      raise "Cannot add new maps since caching is activated for this source" if defined?(@paths) && @cache_paths
       maps.each do |mp, source|
         raise "Invalid mount point specified: #{mp}" unless mp =~ /^\//
         @stack << [mp, source]
@@ -41,6 +48,7 @@ module Webgen::Source
     # returned by later source objects are not used if a prior source object has returned the same
     # path.
     def paths
+      return @paths if defined?(@paths) && @cache_paths
       @paths = Set.new
       @stack.each do |mp, source|
         source.paths.each do |path|

data/lib/webgen/source/tararchive.rb

@@ -5,7 +5,11 @@ require 'webgen/websiteaccess'
 require 'webgen/path'
 require 'open-uri'
 require 'zlib'
-require 'archive/tar/minitar'
+begin
+  require 'archive/tar/minitar'
+rescue LoadError
+  raise Webgen::LoadError.new('archive/tar/minitar', self.class.name, context.dest_node.alcn, 'archive-tar-minitar')
+end
 
 module Webgen
 
@@ -24,7 +28,7 @@ module Webgen
 
       # Create a new tar archive path object for the entry +entry+.
       def initialize(path, data, mtime, uri)
-        super(path) { StringIO.new(data.to_s) }
+        super(path) {|mode| StringIO.new(data.to_s, mode) }
         @uri = uri
         @mtime = mtime
         WebsiteAccess.website.cache[[:tararchive_path, @uri, path]] = @mtime if WebsiteAccess.website

data/lib/webgen/sourcehandler.rb

@@ -28,6 +28,7 @@ module Webgen
     # * collects all source paths using the source classes
     # * creates nodes using the source handler classes
     # * writes changed nodes out using an output class
+    # * deletes old nodes
     class Main
 
       include WebsiteAccess
@@ -35,88 +36,121 @@ module Webgen
 
       def initialize #:nodoc:
         website.blackboard.add_service(:create_nodes, method(:create_nodes))
+        website.blackboard.add_service(:create_nodes_from_paths, method(:create_nodes_from_paths))
         website.blackboard.add_service(:source_paths, method(:find_all_source_paths))
         website.blackboard.add_listener(:node_meta_info_changed?, method(:meta_info_changed?))
+
+        website.blackboard.add_listener(:before_node_deleted) do |node|
+          website.blackboard.invoke(:output_instance).delete(node.path)
+        end if website.config['output.do_deletion']
       end
 
-      # Render the nodes provided in the +tree+. Before the actual rendering is done, the sources
-      # are checked (nodes for deleted sources are deleted, nodes for new and changed sources).
-      def render(tree)
+      # Render the current website. Before the actual rendering is done, the sources are checked for
+      # changes, i.e. nodes for deleted sources are deleted, nodes for new and changed sources are
+      # updated.
+      def render
         begin
           website.logger.mark_new_cycle if website.logger
 
           puts "Updating tree..."
           time = Benchmark.measure do
             website.cache.reset_volatile_cache
-            update_tree(tree)
+            update_tree
           end
           puts "...done in " + ('%2.4f' % time.real) + ' seconds'
 
-          if !tree.root
+          if !website.tree.root
             puts 'No source files found - maybe not a webgen website?'
             return nil
           end
 
           puts "Writing changed nodes..."
           time = Benchmark.measure do
-            write_tree(tree)
+            write_tree
           end
           puts "...done in " + ('%2.4f' % time.real) + ' seconds'
-        end while tree.node_access[:alcn].any? {|name,node| node.flagged?(:created) || node.flagged?(:reinit)}
+        end while website.tree.node_access[:alcn].any? {|name,node| node.flagged?(:created) || node.flagged?(:reinit)}
         :success
+      rescue Webgen::Error
+        raise
+      rescue Exception => e
+        raise Webgen::Error.new(e)
       end
 
       #######
       private
       #######
 
-      # Update the +tree+ by creating/reinitializing all needed nodes.
-      def update_tree(tree)
+      # Update the <tt>website.tree</tt> by creating/reinitializing all needed nodes.
+      def update_tree
         unused_paths = Set.new
+        referenced_nodes = Set.new
+        all_but_passive_paths = Set.new(find_all_source_paths.select {|name, path| !path.passive?}.collect {|name, path| name})
         begin
-          used_paths = Set.new(find_all_source_paths.keys) - unused_paths
+          used_paths = all_but_passive_paths - unused_paths
           paths_to_use = Set.new
           nodes_to_delete = Set.new
+          passive_nodes = Set.new
+
+          website.tree.node_access[:alcn].each do |alcn, node|
+            next if node == website.tree.dummy_root
+
+            begin
+              used_paths.delete(node.node_info[:src])
 
-          tree.node_access[:alcn].each do |alcn, node|
-            next if node == tree.dummy_root
-            used_paths.delete(node.node_info[:src])
-
-            deleted = !find_all_source_paths.include?(node.node_info[:src])
-            if deleted
-              nodes_to_delete << node
-              #TODO: delete output path
-            elsif (!node.flagged?(:created) && find_all_source_paths[node.node_info[:src]].changed?) || node.meta_info_changed?
-              node.flag(:reinit)
-              paths_to_use << node.node_info[:src]
-            elsif node.changed?
-              # nothing to be done here
+              src_path = find_all_source_paths[node.node_info[:src]]
+              if !src_path
+                nodes_to_delete << node
+              elsif (!node.flagged?(:created) && src_path.changed?) || node.meta_info_changed?
+                node.flag(:reinit)
+                paths_to_use << node.node_info[:src]
+              elsif node.changed?
+                # nothing to be done here but method node.changed? has to be called
+              end
+
+              if src_path && src_path.passive?
+                passive_nodes << node
+              elsif src_path
+                referenced_nodes += node.node_info[:used_meta_info_nodes] + node.node_info[:used_nodes]
+              end
+            rescue Webgen::Error => e
+              e.alcn = node.alcn unless e.alcn
+              raise
+            rescue Exception => e
+              raise Webgen::Error.new(e, nil, node.alcn)
             end
           end
 
-          nodes_to_delete.each {|node| tree.delete_node(node)}
+          # add unused passive nodes to node_to_delete set
+          unreferenced_passive_nodes, other_passive_nodes = passive_nodes.partition do |pnode|
+            !referenced_nodes.include?(pnode.alcn)
+          end
+          refs = other_passive_nodes.collect {|n| (n.node_info[:used_meta_info_nodes] + n.node_info[:used_nodes]).to_a}.flatten
+          unreferenced_passive_nodes.each {|n| nodes_to_delete << n if !refs.include?(n.alcn)}
+
+          nodes_to_delete.each {|node| website.tree.delete_node(node)}
           used_paths.merge(paths_to_use)
-          paths = create_nodes_from_paths(tree, used_paths.to_a.sort)
+          paths = create_nodes_from_paths(used_paths.to_a.sort)
           unused_paths.merge(used_paths - paths)
-          tree.node_access[:alcn].each {|name, node| tree.delete_node(node) if node.flagged?(:reinit)}
+          website.tree.node_access[:alcn].each {|name, node| website.tree.delete_node(node) if node.flagged?(:reinit)}
           website.cache.reset_volatile_cache
         end until used_paths.empty?
       end
 
-      # Write out all changed nodes of the +tree+.
-      def write_tree(tree)
+      # Write out all changed nodes of the <tt>website.tree</tt>.
+      def write_tree
         output = website.blackboard.invoke(:output_instance)
 
-        tree.node_access[:alcn].select do |name, node|
-          use_node = (node != tree.dummy_root && node.flagged?(:dirty))
+        website.tree.node_access[:alcn].select do |name, node|
+          use_node = (node != website.tree.dummy_root && node.flagged?(:dirty))
           node.unflag(:dirty_meta_info)
           node.unflag(:created)
           node.unflag(:dirty)
           use_node
         end.sort.each do |name, node|
-          next if node['no_output'] || !(content = node.content)
-
           begin
+            next if node['no_output'] || !(content = node.content)
+
             puts " "*4 + name, :verbose
             type = if node.is_directory?
                      :directory
@@ -126,8 +160,11 @@ module Webgen
                      :file
                    end
             output.write(node.path, content, type)
-          rescue
-            raise RuntimeError, "Error while processing <#{node.absolute_lcn}>: #{$!.message}", $!.backtrace
+          rescue Webgen::Error => e
+            e.alcn = node.alcn unless e.alcn
+            raise
+          rescue Exception => e
+            raise Webgen::RenderError.new(e, nil, node.alcn)
           end
         end
       end
@@ -135,9 +172,15 @@ module Webgen
       # Return a hash with all source paths.
       def find_all_source_paths
         if !defined?(@paths)
-          source = Webgen::Source::Stacked.new(website.config['sources'].collect do |mp, name, *args|
-                                                 [mp, constant(name).new(*args)]
-                                               end)
+          active_source = Webgen::Source::Stacked.new(website.config['sources'].collect do |mp, name, *args|
+                                                        [mp, constant(name).new(*args)]
+                                                      end)
+          passive_source = Webgen::Source::Stacked.new(website.config['passive_sources'].collect do |mp, name, *args|
+                                                         [mp, constant(name).new(*args)]
+                                                       end, true)
+          passive_source.paths.each {|path| path.passive = true }
+          source = Webgen::Source::Stacked.new([['/', active_source], ['/', passive_source]])
+
           @paths = {}
           source.paths.each do |path|
             if !(website.config['sourcehandler.ignore'].any? {|pat| File.fnmatch(pat, path, File::FNM_CASEFOLD|File::FNM_DOTMATCH)})
@@ -155,13 +198,13 @@ module Webgen
 
         options = (website.config['sourcehandler.casefold'] ? File::FNM_CASEFOLD : 0) |
           (website.config['sourcehandler.use_hidden_files'] ? File::FNM_DOTMATCH : 0)
-        find_all_source_paths.values_at(*paths).select do |path|
+        find_all_source_paths.values_at(*paths).compact.select do |path|
           patterns.any? {|pat| File.fnmatch(pat, path, options)}
         end
       end
 
-      # Use the source handlers to create nodes for the +paths+ in the +tree+.
-      def create_nodes_from_paths(tree, paths)
+      # Use the source handlers to create nodes for the +paths+ in the <tt>website.tree</tt>.
+      def create_nodes_from_paths(paths)
         used_paths = Set.new
         website.config['sourcehandler.invoke'].sort.each do |priority, shns|
           shns.each do |shn|
@@ -169,36 +212,39 @@ module Webgen
             handler_paths = paths_for_handler(shn, paths)
             used_paths.merge(handler_paths)
             handler_paths.sort {|a,b| a.path.length <=> b.path.length}.each do |path|
-              parent_dir = path.directory.split('/').collect {|p| Path.new(p).cn}.join('/')
-              parent_dir += '/' if path != '/' && parent_dir == ''
-              create_nodes(tree, parent_dir, path, sh)
+              if !website.tree[path.parent_path]
+                used_paths.merge(create_nodes_from_paths([path.parent_path]))
+              end
+              create_nodes(path, sh)
             end
           end
         end
         used_paths
       end
 
-      # Prepare everything to create nodes under the absolute lcn path +parent_path_name+ in the
-      # +tree from the +path+ using the +source_handler+. If a block is given, the actual creation
-      # of the nodes is deferred to it. After the nodes are created, it is also checked if they have
-      # all needed properties.
-      def create_nodes(tree, parent_path_name, path, source_handler) #:yields: parent, path
-        if !(parent = tree[parent_path_name])
-          raise "The specified parent path <#{parent_path_name}> does not exist"
-        end
+      # Prepare everything to create from the +path+ using the +source_handler+. If a block is
+      # given, the actual creation of the nodes is deferred to it. Otherwise the #create_node method
+      # of the +source_handler+ is used. After the nodes are created, it is also checked if they
+      # have all needed properties.
+      def create_nodes(path, source_handler) #:yields: path
         path = path.dup
         path.meta_info = default_meta_info(path, source_handler.class.name)
         (website.cache[:sourcehandler_path_mi] ||= {})[[path.path, source_handler.class.name]] = path.meta_info.dup
-        website.blackboard.dispatch_msg(:before_node_created, parent, path)
+        website.blackboard.dispatch_msg(:before_node_created, path)
         *nodes = if block_given?
-                   yield(parent, path)
+                   yield(path)
                  else
-                   source_handler.create_node(parent, path)
+                   source_handler.create_node(path)
                  end
         nodes.flatten.compact.each do |node|
           website.blackboard.dispatch_msg(:after_node_created, node)
         end
         nodes
+      rescue Webgen::Error => e
+        e.alcn = path unless e.alcn
+        raise
+      rescue Exception => e
+        raise Webgen::NodeCreationError.new(e, source_handler.class.name, path)
       end
 
       # Return the default meta info for the pair of +path+ and +sh_name+.

data/lib/webgen/sourcehandler/base.rb

@@ -19,7 +19,7 @@ module Webgen::SourceHandler
   #
   # The paths that are handled by a source handler are specified via path patterns (see
   # below). During a webgen run the #create_node method for each source paths that matches a
-  # specified path pattern is called. And when it is time to write out the node, the the #content
+  # specified path pattern is called. And when it is time to write out the node, the #content
   # method is called to retrieve the rendered content.
   #
   # A source handler must not take any parameters on initialization and when this module is not
@@ -35,6 +35,7 @@ module Webgen::SourceHandler
   # It also provides other utility methods:
   # * #page_from_path
   # * #content
+  # * #parent_node
   #
   # == Nodes Created for Paths
   #
@@ -109,10 +110,11 @@ module Webgen::SourceHandler
   #   class SimpleCopy
   #
   #     include Webgen::SourceHandler::Base
+  #     include Webgen::WebsiteAccess
   #
-  #     def create_node(parent, path)
+  #     def create_node(path)
   #       path.ext += '.copied'
-  #       super(parent, path)
+  #       super(path)
   #     end
   #
   #     def content(node)
@@ -151,7 +153,8 @@ module Webgen::SourceHandler
           when :year, :month, :day
             ctime = path.meta_info['created_at']
             if !ctime.kind_of?(Time)
-              raise "Invalid meta info 'created_at' for #{path}, needed because of used output path style"
+              raise Webgen::NodeCreationError.new("Invalid meta info 'created_at', needed because of used output path style",
+                                                  self.class.name, path)
             end
             ctime.send(part).to_s.rjust(2, '0')
           when Symbol  then path.send(part)
@@ -181,36 +184,49 @@ module Webgen::SourceHandler
       if OutputPathHelpers.public_instance_methods(false).map(&:to_s).include?(method)
         name = send(method, parent, path, use_lang_part)
         name += '/'  if path.path =~ /\/$/ && name !~ /\/$/
-        if (node = node_exists?(parent, path, name)) && node.lang == path.meta_info['lang']
-          name = node.path
-        elsif node
+        if (node = node_exists?(path, name)) && node.lang != path.meta_info['lang']
           name = send(method, parent, path, (path.meta_info['lang'].nil? ? false : true))
           name += '/'  if path.path =~ /\/$/ && name !~ /\/$/
         end
         name
       else
-        raise "Unknown method for creating output path: #{method}"
+        raise Webgen::NodeCreationError.new("Unknown method for creating output path: #{path.meta_info['output_path']}",
+                                            self.class.name, path)
       end
     end
 
-    # Check if the node alcn and output path which would be created by #create_node exists. The
+    # Check if the node alcn and output path which would be created by #create_node exist. The
     # +output_path+ to check for can individually be set.
-    def node_exists?(parent, path, output_path = self.output_path(parent, path))
-      parent.tree[Webgen::Node.absolute_name(parent, path.lcn, :alcn)] || (!path.meta_info['no_output'] && parent.tree[output_path, :path])
+    def node_exists?(path, output_path = self.output_path(parent_node(path), path))
+      Webgen::WebsiteAccess.website.tree[path.alcn] || (!path.meta_info['no_output'] && Webgen::WebsiteAccess.website.tree[output_path, :path])
     end
 
-    # Create a node under +parent+ from +path+ if it does not already exists or needs to be
-    # re-initialized. The found node or the newly created node is returned afterwards. +nil+ is
-    # returned if no node can be created (e.g. when <tt>path.meta_info['draft']</tt> is set). Some
-    # additional node information like <tt>:src</tt> and <tt>:processor</tt> is set and the meta
-    # information is checked for validness. The created/re-initialized node is yielded if a block is
-    # given.
-    def create_node(parent, path, output_path = self.output_path(parent, path))
+    # Create a node from +path+ if it does not already exists or re-initalize an already existing
+    # node. The found node or the newly created node is returned afterwards. +nil+ is returned if no
+    # node can be created (e.g. when <tt>path.meta_info['draft']</tt> is set).
+    #
+    # The +options+ parameter can be used for providing the optional parameters:
+    #
+    # [<tt>:parent</tt>] The parent node under which the new node should be created. If this is not
+    #                    specified (the usual case), the parent node is determined by the
+    #                    #parent_node method.
+    #
+    # [<tt>:output_path</tt>] The output path that should be used for the node. If this is not
+    #                         specified (the usual case), the output path is determined via the
+    #                         #output_path method.
+    #
+    # Some additional node information like <tt>:src</tt> and <tt>:processor</tt> is set and the
+    # meta information is checked for validness. The created/re-initialized node is yielded if a
+    # block is given.
+    def create_node(path, options = {})
       return nil if path.meta_info['draft']
-      node = node_exists?(parent, path, output_path)
+      parent = options[:parent] || parent_node(path)
+      output_path = options[:output_path] || self.output_path(parent, path)
+      node = node_exists?(path, output_path)
+
       if node && (node.node_info[:src] != path.source_path || node.node_info[:processor] != self.class.name)
-        log(:warn) { "Node already exists: source = #{path.source_path} | path = #{node.path} | alcn = #{node.absolute_lcn}"}
-        return node
+        log(:warn) { "Node already exists: source = #{path.source_path} | path = #{node.path} | alcn = #{node.alcn}"}
+        return node #TODO: think! should nil be returned?
       elsif !node
         node = Webgen::Node.new(parent, output_path, path.cn, path.meta_info)
       elsif node.flagged?(:reinit)
@@ -218,7 +234,11 @@ module Webgen::SourceHandler
       else
         return node
       end
-      node['modified_at'] = Time.now unless node['modified_at'].kind_of?(Time)
+
+      if !node['modified_at'].kind_of?(Time)
+        log(:warn) { "Meta information 'modified_at' set to current time in <#{node.alcn}> since its value '#{node['modified_at']}' was of type #{node['modified_at'].class}" } unless node['modified_at'].nil?
+        node['modified_at'] = Time.now
+      end
       node.node_info[:src] = path.source_path
       node.node_info[:creation_path] = path.path
       node.node_info[:processor] = self.class.name
@@ -226,7 +246,10 @@ module Webgen::SourceHandler
       node
     end
 
-    # Return the content of the given +node+. This default +content+ method just returns +nil+.
+    # Return the content of the given +node+. If the return value is not +nil+ then the node gets
+    # written, otherwise it is ignored.
+    #
+    # This default +content+ method just returns +nil+.
     def content(node)
       nil
     end
@@ -237,12 +260,23 @@ module Webgen::SourceHandler
       begin
         page = Webgen::Page.from_data(path.io.data, path.meta_info)
       rescue Webgen::Page::FormatError => e
-        raise "Error reading source path <#{path}>: #{e.message}"
+        raise Webgen::NodeCreationError.new("Error reading source path: #{e.message}",
+                                            self.class.name, path)
       end
       path.meta_info = page.meta_info
       page
     end
 
+    # Return the parent node for the given +path+.
+    def parent_node(path)
+      parent_dir = (path.parent_path == '' ? '' : Webgen::Path.new(path.parent_path).alcn)
+      if !(parent = Webgen::WebsiteAccess.website.tree[parent_dir])
+        raise Webgen::NodeCreationError.new("The needed parent path <#{parent_dir}> does not exist",
+                                            self.class.name, path)
+      end
+      parent
+    end
+
   end
 
 end