webgen 0.5.5 → 0.5.6

data/lib/webgen/languages.rb

@@ -50,6 +50,15 @@ module Webgen
       self.to_s <=> other.to_s
     end
 
+    def eql?(other) #:nodoc:
+      (other.is_a?(self.class) && other.to_s == self.to_s) ||
+        (other.is_a?(String) && self.to_s == other)
+    end
+
+    def hash #:nodoc:
+      self.to_s.hash
+    end
+
   end
 
 

data/lib/webgen/logger.rb

@@ -27,11 +27,28 @@ module Webgen
       end
       self.level = ::Logger::WARN
       self.verbosity = :normal
+      @marks = []
     end
 
     # Returns the output of the logger when #sync is +false+. Otherwise an empty string is returned.
     def log_output
-      @sync ? '' : @logio.string
+      if @sync
+        ''
+      else
+        out = @logio.string.dup
+        @marks.reverse.each_with_index do |mark, index|
+          out.insert(mark, " INFO -- Log messages for run #{@marks.length - index} are following\n")
+        end if out.length > 0
+        out
+      end
+    end
+
+    # Only used when #sync is +false: Mark the location in the log stream where a new update/write
+    # run begins.
+    def mark_new_cycle
+      if !@sync
+        @marks << @logio.string.length
+      end
     end
 
     # The severity threshold level.

data/lib/webgen/node.rb

@@ -46,17 +46,6 @@ module Webgen
     # Meta information associated with the node.
     attr_reader :meta_info
 
-    # Set by other objects to +true+ if they think the object has changed since the last run. Must
-    # not be set to +false+ once it is +true+!
-    attr_accessor :dirty
-
-    # Set by other objects to +true+ if the meta information of the node has changed since the last
-    # run. Must not be set to +false+ once it is +true+!
-    attr_accessor :dirty_meta_info
-
-    # Has the node been created or has it been read from the cache?
-    attr_accessor :created
-
     # Create a new Node instance.
     #
     # +parent+ (immutable)::
@@ -78,17 +67,29 @@ module Webgen
     # found, the node is language neutral.
     def initialize(parent, path, cn, meta_info = {})
       @parent = parent
-      @path = path.freeze
       @cn = cn.chomp('/').freeze
-      @lang = meta_info.delete('lang').freeze
-      @lang = nil unless is_file?
-      @meta_info = meta_info
       @children = []
-      @dirty = true
-      @created = true
+      reinit(path, meta_info)
       init_rest
     end
 
+    # Re-initializes an already initialized node and resets it to its pristine state.
+    def reinit(path, meta_info = {})
+      old_path = @path
+      @path = path.freeze
+      @lang = Webgen::LanguageManager.language_for_code(meta_info.delete('lang'))
+      @lang = nil unless is_file?
+      @meta_info = meta_info
+      @flags = Set.new([:dirty, :created])
+      if @tree
+        @tree.node_access[:path].delete(old_path) if old_path
+        @tree.register_path(self)
+        self.node_info.clear
+        self.node_info[:used_nodes] = Set.new
+        self.node_info[:used_meta_info_nodes] = Set.new
+      end
+    end
+
     # Return the meta information item for +key+.
     def [](key)
       @meta_info[key]
@@ -116,26 +117,50 @@ module Webgen
     # Check if the node is the root node.
     def is_root?; self == tree.root;  end
 
+    # Check if the node is flagged with one of the following:
+    #
+    # :created:: Has the node been created or has it been read from the cache?
+    # :reinit:: Does the node need to be reinitialized?
+    # :dirty:: Set by other objects to +true+ if they think the object has changed since the last
+    #          run. Must not be set to +false+ once it is +true+!
+    # :dirty_meta_info:: Set by other objects to +true+ if the meta information of the node has
+    #                    changed since the last run. Must not be set to +false+ once it is +true+!
+    def flagged(key)
+      @flags.include?(key)
+    end
+
+    # Flag the node with the +keys+. See #flagged for valid keys.
+    def flag(*keys)
+      @flags += keys
+      website.blackboard.dispatch_msg(:node_flagged, self, keys)
+    end
+
+    # Remove the flags +keys+ from the node.
+    def unflag(*keys)
+      @flags.subtract(keys)
+      website.blackboard.dispatch_msg(:node_unflagged, self, keys)
+    end
+
     # Return +true+ if the node has changed since the last webgen run. If it has changed, +dirty+ is
     # set to +true+.
     def changed?
       if_not_checked(:node) do
-        @dirty = @dirty || meta_info_changed?
-        @dirty = node_info[:used_nodes].any? {|n| n != @absolute_lcn && (!tree[n] || tree[n].changed?)} unless @dirty
-        website.blackboard.dispatch_msg(:node_changed?, self) unless @dirty
+        flag(:dirty) if meta_info_changed? ||
+          node_info[:used_nodes].any? {|n| n != @absolute_lcn && (!tree[n] || tree[n].changed?)}
+        website.blackboard.dispatch_msg(:node_changed?, self) unless flagged(:dirty)
       end
-      @dirty
+      flagged(:dirty)
     end
 
     # Return +true+ if the meta information of the node has changed.
     def meta_info_changed?
       if_not_checked(:meta_info) do
-        @dirty_meta_info = node_info[:used_meta_info_nodes].any? do |n|
+        flag(:dirty_meta_info) if node_info[:used_meta_info_nodes].any? do |n|
           n != @absolute_lcn && (!tree[n] || tree[n].meta_info_changed?)
-        end unless @dirty_meta_info
-        website.blackboard.dispatch_msg(:node_meta_info_changed?, self) unless @dirty_meta_info
+        end
+        website.blackboard.dispatch_msg(:node_meta_info_changed?, self) unless flagged(:dirty_meta_info)
       end
-      @dirty_meta_info
+      flagged(:dirty_meta_info)
     end
 
     # Return an informative representation of the node.

data/lib/webgen/page.rb

@@ -55,14 +55,17 @@ module Webgen
       # the information. The +meta_info+ parameter can be used to provide default meta information.
       def from_data(data, meta_info = {})
         md = /(#{RE_META_INFO})?(.*)/m.match(normalize_eol(data))
-        if md[1].nil? && data =~ RE_META_INFO_START
-          raise WebgenPageFormatError, 'Found start line for meta information block but no valid meta information block'
-        end
-        meta_info = meta_info.merge(md[1].nil? ? {} : parse_meta_info(md[1]))
+        meta_info = meta_info.merge(parse_meta_info(md[1], data))
         blocks = parse_blocks(md[2] || '', meta_info)
         new(meta_info, blocks)
       end
 
+      # Parse the given string +data+ in Webgen Page Format and return the found meta information.
+      def meta_info_from_data(data)
+        md = /(#{RE_META_INFO})?/m.match(normalize_eol(data))
+        parse_meta_info(md[1], data)
+      end
+
       #######
       private
       #######
@@ -72,17 +75,24 @@ module Webgen
         data.gsub(/\r\n?/, "\n")
       end
 
-      # Parse the meta info string +data+ and return the hash with the meta information.
-      def parse_meta_info(data)
-        begin
-          meta_info = YAML::load(data)
-          unless meta_info.kind_of?(Hash)
-            raise WebgenPageFormatError, "Invalid structure of meta information block: expected YAML hash but found #{meta_info.class}"
+      # Parse the meta info string in +mi_data+ and return the hash with the meta information. The
+      # original +data+ is used for checking the validness of the meta information block.
+      def parse_meta_info(mi_data, data)
+        if mi_data.nil? && data =~ RE_META_INFO_START
+          raise WebgenPageFormatError, 'Found start line for meta information block but no valid meta information block'
+        elsif mi_data.nil?
+          {}
+        else
+          begin
+            meta_info = YAML::load(mi_data.to_s)
+            unless meta_info.kind_of?(Hash)
+              raise WebgenPageFormatError, "Invalid structure of meta information block: expected YAML hash but found #{meta_info.class}"
+            end
+          rescue ArgumentError => e
+            raise WebgenPageFormatError, e.message
           end
-        rescue ArgumentError => e
-          raise WebgenPageFormatError, e.message
+          meta_info
         end
-        meta_info
       end
 
       # Parse all blocks in +data+ and return them. Meta information can be provided in +meta_info+
@@ -96,7 +106,7 @@ module Webgen
           options, content = *block_data
           md = RE_BLOCKS_OPTIONS.match(options.to_s)
           raise(WebgenPageFormatError, "Found invalid blocks starting line for block #{index+1}: #{options}") if content =~ /\A---/ || md.nil?
-          options = Hash[*md[1].to_s.scan(/(\w+):([^\s]*)/).map {|k,v| [k, YAML::load(v)]}.flatten]
+          options = Hash[*md[1].to_s.scan(/(\w+):([^\s]*)/).map {|k,v| [k, (v == '' ? nil : YAML::load(v))]}.flatten]
           options = (meta_info['blocks']['default'] || {} rescue {}).
             merge((meta_info['blocks'][index+1] || {} rescue {})).
             merge(options)
@@ -118,12 +128,12 @@ module Webgen
     # The contents of the meta information block.
     attr_reader :meta_info
 
-    # The array of blocks for the page.
+    # The hash of blocks for the page.
     attr_reader :blocks
 
     # Create a new Page object with the meta information provided in +meta_info+ and the given
     # +blocks+.
-    def initialize(meta_info = {}, blocks = nil)
+    def initialize(meta_info = {}, blocks = {})
       @meta_info = meta_info
       @blocks = blocks
     end

data/lib/webgen/path.rb

@@ -35,11 +35,24 @@ module Webgen
     end
 
 
+    # Return +true+ if the given +path+ matches the given +pattern+ (trailing slashes of directories
+    # are not respected). For information on which patterns are supported, have a look at the
+    # documentation of File.fnmatch.
+    def self.match(path, pattern)
+      path = path.to_s.chomp('/') unless path == '/'
+      pattern = pattern.to_s.chomp('/') unless pattern == '/'
+      File.fnmatch(pattern, path, File::FNM_DOTMATCH|File::FNM_CASEFOLD|File::FNM_PATHNAME)
+    end
+
+
     include Comparable
 
-    # The full source path.
+    # The full path.
     attr_accessor :path
 
+    # The source path that lead to the creation of this path.
+    attr_accessor :source_path
+
     # The basename part of the path.
     attr_accessor :basename
 
@@ -55,11 +68,13 @@ module Webgen
     # Extracted meta information for the path.
     attr_accessor :meta_info
 
-    # Create a new Path object for +path+. The optional block needs to return an IO object for the
-    # content of the path.
-    def initialize(path, &ioblock)
+    # Create a new Path object for +path+. The optional +source_path+ parameter specifies the path
+    # that lead to the creation of this path. The optional block needs to return an IO object for
+    # the content of the path.
+    def initialize(path, source_path = path, &ioblock)
       @meta_info = {}
       @io = SourceIO.new(&ioblock) if block_given?
+      @source_path = source_path
       analyse(path)
     end
 
@@ -70,6 +85,7 @@ module Webgen
       temp.path = temp.path.sub(/^#{Regexp.escape(prefix.chomp("/"))}/, '') if prefix     #"
       reanalyse = (@path == '/' || temp.path == '/')
       temp.path = File.join(mp, temp.path)
+      temp.source_path = temp.path if @path == @source_path
       if reanalyse
         temp.send(:analyse, temp.path)
       else
@@ -131,12 +147,6 @@ module Webgen
     end
     alias_method(:eql?, :==)
 
-    # Return +true+ if the localized path matches the given +pattern+. For information on which
-    # patterns are supported, have a look at the documentation of File.fnmatch.
-    def =~(pattern)
-      File.fnmatch(pattern, File.join(@directory, lcn), File::FNM_DOTMATCH|File::FNM_CASEFOLD|File::FNM_PATHNAME)
-    end
-
     # Implemented sothat a Path looks like a String when used as key in a hash.
     def <=>(other)
       @path <=> other.to_str

data/lib/webgen/sourcehandler.rb

@@ -19,6 +19,7 @@ module Webgen
     autoload :Virtual, 'webgen/sourcehandler/virtual'
     autoload :Feed, 'webgen/sourcehandler/feed'
     autoload :Sitemap, 'webgen/sourcehandler/sitemap'
+    autoload :Memory, 'webgen/sourcehandler/memory'
 
     # This class is used by Website to do the actual rendering of the website. It
     #
@@ -39,52 +40,90 @@ module Webgen
       # Render the nodes provided in the +tree+. Before the actual rendering is done, the sources
       # are checked (nodes for deleted sources are deleted, nodes for new and changed sources).
       def render(tree)
-        # Add new and changed nodes, remove nodes of deleted paths
-        puts "Generating tree..."
-        time = Benchmark.measure do
-          used_paths = Set.new
-          paths = Set.new([nil])
-          while paths.length > 0
-            used_paths += (paths = Set.new(find_all_source_paths.keys) - used_paths - clean(tree))
-            create_nodes_from_paths(tree, paths)
+        begin
+          website.logger.mark_new_cycle if website.logger
+
+          puts "Updating tree..."
+          time = Benchmark.measure do
             website.cache.reset_volatile_cache
+            update_tree(tree)
           end
-        end
-        puts "...done in " + ('%2.4f' % time.real) + ' seconds'
-
-        output = website.blackboard.invoke(:output_instance)
+          puts "...done in " + ('%2.4f' % time.real) + ' seconds'
 
-        puts "Writing changed nodes..."
-        time = Benchmark.measure do
-          tree.node_access[:alcn].sort.each do |name, node|
-            node.dirty_meta_info = node.created = false
-            next if node == tree.dummy_root || !node.dirty
-            node.dirty = false
-
-            begin
-              if !node['no_output'] && (content = node.content)
-                puts " "*4 + name, :verbose
-                type = if node.is_directory?
-                         :directory
-                       elsif node.is_fragment?
-                         :fragment
-                       else
-                         :file
-                       end
-                output.write(node.path, content, type)
-              end
-            rescue
-              raise RuntimeError, "Error while processing <#{node.absolute_lcn}>: #{$!.message}", $!.backtrace
-            end
+          puts "Writing changed nodes..."
+          time = Benchmark.measure do
+            write_tree(tree)
           end
-        end
-        puts "...done in " + ('%2.4f' % time.real) + ' seconds'
+          puts "...done in " + ('%2.4f' % time.real) + ' seconds'
+        end while tree.node_access[:alcn].any? {|name,node| node.flagged(:created) || node.flagged(:reinit)}
       end
 
       #######
       private
       #######
 
+      # Update the +tree+ by creating/reinitializing all needed nodes.
+      def update_tree(tree)
+        unused_paths = Set.new
+        begin
+          used_paths = Set.new(find_all_source_paths.keys) - unused_paths
+          paths_to_use = Set.new
+          nodes_to_delete = Set.new
+
+          tree.node_access[:alcn].each do |alcn, node|
+            next if node == tree.dummy_root
+            used_paths.delete(node.node_info[:src])
+
+            deleted = !find_all_source_paths.include?(node.node_info[:src])
+            if deleted
+              nodes_to_delete << node
+              #TODO: delete output path
+            elsif (!node.flagged(:created) && find_all_source_paths[node.node_info[:src]].changed?) || node.meta_info_changed?
+              node.flag(:reinit)
+              paths_to_use << node.node_info[:src]
+            elsif node.changed?
+              # nothing to be done here
+            end
+          end
+
+          nodes_to_delete.each {|node| tree.delete_node(node)}
+          used_paths.merge(paths_to_use)
+          paths = create_nodes_from_paths(tree, used_paths.to_a.sort)
+          unused_paths.merge(used_paths - paths)
+          tree.node_access[:alcn].each {|name, node| tree.delete_node(node) if node.flagged(:reinit)}
+          website.cache.reset_volatile_cache
+        end until used_paths.empty?
+      end
+
+      # Write out all changed nodes of the +tree+.
+      def write_tree(tree)
+        output = website.blackboard.invoke(:output_instance)
+
+        tree.node_access[:alcn].select do |name, node|
+          use_node = (node != tree.dummy_root && node.flagged(:dirty))
+          node.unflag(:dirty_meta_info)
+          node.unflag(:created)
+          node.unflag(:dirty)
+          use_node
+        end.sort.each do |name, node|
+          next if node['no_output'] || !(content = node.content)
+
+          begin
+            puts " "*4 + name, :verbose
+            type = if node.is_directory?
+                     :directory
+                   elsif node.is_fragment?
+                     :fragment
+                   else
+                     :file
+                   end
+            output.write(node.path, content, type)
+          rescue
+            raise RuntimeError, "Error while processing <#{node.absolute_lcn}>: #{$!.message}", $!.backtrace
+          end
+        end
+      end
+
       # Return a hash with all source paths.
       def find_all_source_paths
         if !defined?(@paths)
@@ -94,7 +133,7 @@ module Webgen
           @paths = {}
           source.paths.each do |path|
             if !(website.config['sourcehandler.ignore'].any? {|pat| File.fnmatch(pat, path, File::FNM_CASEFOLD|File::FNM_DOTMATCH)})
-              @paths[path.path] = path
+              @paths[path.source_path] = path
             end
           end
         end
@@ -115,16 +154,20 @@ module Webgen
 
       # Use the source handlers to create nodes for the +paths+ in the +tree+.
       def create_nodes_from_paths(tree, paths)
+        used_paths = Set.new
         website.config['sourcehandler.invoke'].sort.each do |priority, shns|
           shns.each do |shn|
             sh = website.cache.instance(shn)
-            paths_for_handler(shn, paths).sort.each do |path|
+            handler_paths = paths_for_handler(shn, paths)
+            used_paths.merge(handler_paths)
+            handler_paths.sort {|a,b| a.path.length <=> b.path.length}.each do |path|
               parent_dir = path.directory.split('/').collect {|p| Path.new(p).cn}.join('/')
               parent_dir += '/' if path != '/' && parent_dir == ''
               create_nodes(tree, parent_dir, path, sh)
             end
           end
         end
+        used_paths
       end
 
       # Prepare everything to create nodes under the absolute lcn path +parent_path_name+ in the
@@ -142,7 +185,7 @@ module Webgen
         *nodes = if block_given?
                    yield(parent, path)
                  else
-                   source_handler.create_node(parent, path.dup)
+                   source_handler.create_node(parent, path)
                  end
         nodes.flatten.compact.each do |node|
           website.blackboard.dispatch_msg(:after_node_created, node)
@@ -160,39 +203,12 @@ module Webgen
       # take the node's path's +modified_at+ meta information into account since that changes on
       # every path change.
       def meta_info_changed?(node)
-        path = node.node_info[:src]
+        path = node.node_info[:creation_path]
         old_mi = website.cache[:sourcehandler_path_mi][[path, node.node_info[:processor]]]
         old_mi.delete('modified_at')
-        new_mi = default_meta_info(@paths[path], node.node_info[:processor])
+        new_mi = default_meta_info(@paths[path] || Webgen::Path.new(path), node.node_info[:processor])
         new_mi.delete('modified_at')
-        node.dirty_meta_info = true if !old_mi || old_mi != new_mi
-      end
-
-      # Clean the +tree+ by deleting nodes which have changed or which don't have an associated
-      # source anymore. Return all paths for which nodes need to be created.
-      def clean(tree)
-        paths_to_delete = Set.new
-        paths_not_to_delete = Set.new
-        nodes_to_be_deleted = Set.new
-        tree.node_access[:alcn].each do |alcn, node|
-          next if node == tree.dummy_root || tree[alcn].nil?
-
-          deleted = !find_all_source_paths.include?(node.node_info[:src])
-          if !node.created && (deleted ||
-                               find_all_source_paths[node.node_info[:src]].changed? ||
-                               node.changed?)
-            paths_not_to_delete << node.node_info[:src]
-            nodes_to_be_deleted << [node, deleted]
-          else
-            paths_to_delete << node.node_info[:src]
-          end
-        end
-
-        nodes_to_be_deleted.each {|node, deleted| tree.delete_node(node, deleted)}
-        #TODO: delete output path
-
-        # source paths that should be used
-        paths_to_delete - paths_not_to_delete
+        node.flag(:dirty_meta_info) if !old_mi || old_mi != new_mi
       end
 
     end