gettalong-webgen 0.5.4.20080929

data/lib/webgen/source/filesystem.rb

@@ -0,0 +1,58 @@
+require 'pathname'
+require 'webgen/websiteaccess'
+require 'webgen/path'
+
+module Webgen
+
+  # This class is used to read source paths from a directory in the file system.
+  class Source::FileSystem
+
+    # A special Webgen::Path class for handling with file system paths.
+    class Path < Webgen::Path
+
+      # Create a new object with absolute path +path+ for the file system path +fs_path+.
+      def initialize(path, fs_path)
+        super(path) { File.open(fs_path, 'rb') }
+        @fs_path = fs_path
+        WebsiteAccess.website.cache[[:fs_path, @fs_path]] = File.mtime(@fs_path)
+        @meta_info['modified_at'] = File.mtime(@fs_path)
+      end
+
+      # Return +true+ if the file system path used by the object has been modified.
+      def changed?
+        data = WebsiteAccess.website.cache[[:fs_path, @fs_path]]
+        File.mtime(@fs_path) > data
+      end
+
+    end
+
+    # The root path from which paths read.
+    attr_reader :root
+
+    # The glob (see Dir.glob for details) that is used to specify which paths under the root path
+    # should be returned by #paths.
+    attr_reader :glob
+
+    # Create a new file system source for the root path +root+ using the provided +glob+.
+    def initialize(root, glob = '**/*')
+      if root =~ /^([a-zA-Z]:|\/)/
+        @root = root
+      else
+        @root = File.join(WebsiteAccess.website.directory, root)
+      end
+      @glob = glob
+    end
+
+    # Return all paths under #root which match #glob.
+    def paths
+      @paths ||= Dir.glob(File.join(@root, @glob), File::FNM_DOTMATCH|File::FNM_CASEFOLD).to_set.collect! do |f|
+        temp = Pathname.new(f.sub(/^#{Regexp.escape(@root)}\/?/, '/')).cleanpath.to_s
+        temp += '/' if File.directory?(f) && temp[-1] != ?/
+        path = Path.new(temp, f)
+        path
+      end
+    end
+
+  end
+
+end

data/lib/webgen/source/resource.rb

@@ -0,0 +1,42 @@
+require 'webgen/websiteaccess'
+require 'webgen/source'
+
+module Webgen::Source
+
+  # This class is used to provide access to source provided by resources.
+  class Resource
+
+    include Webgen::WebsiteAccess
+
+    # The glob specifying the resources.
+    attr_reader :glob
+
+    # The glob specifying the paths that should be used from the resources.
+    attr_reader :paths_glob
+
+    # The prefix that should optionally be stripped from the paths.
+    attr_reader :strip_prefix
+
+    # Create a new resource source for the the +glob+ and use only those paths matching +paths_glob+
+    # while stripping +strip_prefix+ off the path.
+    def initialize(glob, paths_glob = nil, strip_prefix = nil)
+      @glob, @paths_glob, @strip_prefix = glob, paths_glob, strip_prefix
+    end
+
+    # Return all paths associated with the resources identified by #glob.
+    def paths
+      if !defined?(@paths)
+        stack = Stacked.new
+        website.config['resources'].select {|name, infos| File.fnmatch(@glob, name)}.sort.each do |name, infos|
+          stack.add([['/', constant(infos.first).new(*infos[1..-1])]])
+        end
+        @paths = stack.paths
+        @paths = @paths.select {|p| File.fnmatch(@paths_glob, p)} if @paths_glob
+        @paths.collect! {|p| p.mount_at('/', @strip_prefix)} if @strip_prefix
+      end
+      @paths
+    end
+
+  end
+
+end

data/lib/webgen/source/stacked.rb

@@ -0,0 +1,53 @@
+module Webgen::Source
+
+  # This source class is used to stack several sources together.
+  #
+  # It serves two purposes:
+  #
+  # * First, it can be used to access more than one source. This is useful when your website
+  #   consists of more than one source directory and you want to use all of them.
+  #
+  # * Second, sources can mounted on specific directories. For example, a folder with images that
+  #   you don't want to copy to the website source directory can be mounted under <tt>/images</tt>
+  #   sothat they are available nonetheless.
+  #
+  # Also be aware that when a path is returned by a source that has already be returned by a prior
+  # source, it is discarded and not used.
+  class Stacked
+
+    # Return the stack of Webgen::Source objects.
+    attr_reader :stack
+
+    # Create a new stack. The optional +map+ parameter can be used to provide initial mappings of
+    # mount points to source objects (see #add for details).
+    def initialize(map = {})
+      @stack = []
+      add(map)
+    end
+
+    # Add all mappings found in +maps+ to the stack. The parameter +maps+ should be an array of
+    # two-element arrays which contain an absolute directoriy (ie. starting with a slash) and a
+    # source object.
+    def add(maps)
+      maps.each do |mp, source|
+        raise "Invalid mount point specified: #{mp}" unless mp =~ /^\//
+        @stack << [mp, source]
+      end
+    end
+
+    # Return all paths returned by the sources in the stack. Since the stack is ordered, paths
+    # returned by later source objects are not used if a prior source object has returned the same
+    # path.
+    def paths
+      @paths = Set.new
+      @stack.each do |mp, source|
+        source.paths.each do |path|
+          @paths.add?(path.mount_at(mp))
+        end
+      end
+      @paths
+    end
+
+  end
+
+end

data/lib/webgen/sourcehandler.rb

@@ -0,0 +1,202 @@
+require 'webgen/loggable'
+require 'benchmark'
+
+module Webgen
+
+  # Namespace for all classes that handle source paths.
+  #
+  # Have a look at Webgen::SourceHandler::Base for details on how to implement a source handler
+  # class.
+  module SourceHandler
+
+    autoload :Base, 'webgen/sourcehandler/base'
+    autoload :Copy, 'webgen/sourcehandler/copy'
+    autoload :Directory, 'webgen/sourcehandler/directory'
+    autoload :Metainfo, 'webgen/sourcehandler/metainfo'
+    autoload :Template, 'webgen/sourcehandler/template'
+    autoload :Page, 'webgen/sourcehandler/page'
+    autoload :Fragment, 'webgen/sourcehandler/fragment'
+    autoload :Virtual, 'webgen/sourcehandler/virtual'
+    autoload :Feed, 'webgen/sourcehandler/feed'
+    autoload :Sitemap, 'webgen/sourcehandler/sitemap'
+
+    # This class is used by Website to do the actual rendering of the website. It
+    #
+    # * collects all source paths using the source classes
+    # * creates nodes using the source handler classes
+    # * writes changed nodes out using an output class
+    class Main
+
+      include WebsiteAccess
+      include Loggable
+
+      def initialize #:nodoc:
+        website.blackboard.add_service(:create_nodes, method(:create_nodes))
+        website.blackboard.add_service(:source_paths, method(:find_all_source_paths))
+        website.blackboard.add_listener(:node_meta_info_changed?, method(:meta_info_changed?))
+      end
+
+      # Render the nodes provided in the +tree+. Before the actual rendering is done, the sources
+      # are checked (nodes for deleted sources are deleted, nodes for new and changed sources).
+      def render(tree)
+        # Add new and changed nodes, remove nodes of deleted paths
+        puts "Generating tree..."
+        time = Benchmark.measure do
+          used_paths = Set.new
+          paths = Set.new([nil])
+          while paths.length > 0
+            used_paths += (paths = Set.new(find_all_source_paths.keys) - used_paths - clean(tree))
+            create_nodes_from_paths(tree, paths)
+            website.cache.reset_volatile_cache
+          end
+        end
+        puts "...done in " + ('%2.4f' % time.real) + ' seconds'
+
+        output = website.blackboard.invoke(:output_instance)
+
+        puts "Writing changed nodes..."
+        time = Benchmark.measure do
+          tree.node_access[:alcn].sort.each do |name, node|
+            node.dirty_meta_info = node.created = false
+            next if node == tree.dummy_root || !node.dirty
+            node.dirty = false
+
+            begin
+              if !node['no_output'] && (content = node.content)
+                puts " "*4 + name, :verbose
+                type = if node.is_directory?
+                         :directory
+                       elsif node.is_fragment?
+                         :fragment
+                       else
+                         :file
+                       end
+                output.write(node.path, content, type)
+              end
+            rescue
+              raise RuntimeError, "Error while processing <#{node.absolute_lcn}>: #{$!.message}", $!.backtrace
+            end
+          end
+        end
+        puts "...done in " + ('%2.4f' % time.real) + ' seconds'
+      end
+
+      #######
+      private
+      #######
+
+      # Return a hash with all source paths.
+      def find_all_source_paths
+        if !defined?(@paths)
+          source = Webgen::Source::Stacked.new(website.config['sources'].collect do |mp, name, *args|
+                                                 [mp, constant(name).new(*args)]
+                                               end)
+          @paths = {}
+          source.paths.each do |path|
+            if !(website.config['sourcehandler.ignore'].any? {|pat| File.fnmatch(pat, path, File::FNM_CASEFOLD|File::FNM_DOTMATCH)})
+              @paths[path.path] = path
+            end
+          end
+        end
+        @paths
+      end
+
+      # Return only the subset of +paths+ which are handled by the source handler +name+.
+      def paths_for_handler(name, paths)
+        patterns = website.config['sourcehandler.patterns'][name]
+        return [] if patterns.nil?
+
+        options = (website.config['sourcehandler.casefold'] ? File::FNM_CASEFOLD : 0) |
+          (website.config['sourcehandler.use_hidden_files'] ? File::FNM_DOTMATCH : 0)
+        find_all_source_paths.values_at(*paths).select do |path|
+          patterns.any? {|pat| File.fnmatch(pat, path, options)}
+        end
+      end
+
+      # Use the source handlers to create nodes for the +paths+ in the +tree+.
+      def create_nodes_from_paths(tree, paths)
+        website.config['sourcehandler.invoke'].sort.each do |priority, shns|
+          shns.each do |shn|
+            sh = website.cache.instance(shn)
+            paths_for_handler(shn, paths).sort.each do |path|
+              parent_dir = path.directory.split('/').collect {|p| Path.new(p).cn}.join('/')
+              parent_dir += '/' if path != '/' && parent_dir == ''
+              create_nodes(tree, parent_dir, path, sh)
+            end
+          end
+        end
+      end
+
+      # Prepare everything to create nodes under the absolute lcn path +parent_path_name+ in the
+      # +tree from the +path+ using the +source_handler+. If a block is given, the actual creation
+      # of the nodes is deferred to it. After the nodes are created, it is also checked if they have
+      # all needed properties.
+      def create_nodes(tree, parent_path_name, path, source_handler) #:yields: parent, path
+        if !(parent = tree[parent_path_name])
+          raise "The specified parent path <#{parent_path_name}> does not exist"
+        end
+        path = path.dup
+        path.meta_info = default_meta_info(path, source_handler.class.name)
+        (website.cache[:sourcehandler_path_mi] ||= {})[[path.path, source_handler.class.name]] = path.meta_info.dup
+        website.blackboard.dispatch_msg(:before_node_created, parent, path)
+        *nodes = if block_given?
+                   yield(parent, path)
+                 else
+                   source_handler.create_node(parent, path.dup)
+                 end
+        nodes.flatten.compact.each do |node|
+          website.blackboard.dispatch_msg(:after_node_created, node)
+        end
+        nodes
+      end
+
+      # Return the default meta info for the pair of +path+ and +sh_name+.
+      def default_meta_info(path, sh_name)
+        path.meta_info.merge(website.config['sourcehandler.default_meta_info'][:all]).
+          merge(website.config['sourcehandler.default_meta_info'][sh_name] || {})
+      end
+
+      # Check if the default meta information for +node+ has changed since the last run. But don't
+      # take the node's path's +modified_at+ meta information into account since that changes on
+      # every path change.
+      def meta_info_changed?(node)
+        path = node.node_info[:src]
+        old_mi = website.cache[:sourcehandler_path_mi][[path, node.node_info[:processor]]]
+        old_mi.delete('modified_at')
+        new_mi = default_meta_info(@paths[path], node.node_info[:processor])
+        new_mi.delete('modified_at')
+        node.dirty_meta_info = true if !old_mi || old_mi != new_mi
+      end
+
+      # Clean the +tree+ by deleting nodes which have changed or which don't have an associated
+      # source anymore. Return all paths for which nodes need to be created.
+      def clean(tree)
+        paths_to_delete = Set.new
+        paths_not_to_delete = Set.new
+        nodes_to_be_deleted = Set.new
+        tree.node_access[:alcn].each do |alcn, node|
+          next if node == tree.dummy_root || tree[alcn].nil?
+
+          deleted = !find_all_source_paths.include?(node.node_info[:src])
+          if !node.created && (deleted ||
+                               find_all_source_paths[node.node_info[:src]].changed? ||
+                               node.changed?)
+            paths_not_to_delete << node.node_info[:src]
+            nodes_to_be_deleted << [node, deleted]
+          else
+            paths_to_delete << node.node_info[:src]
+          end
+        end
+
+        nodes_to_be_deleted.each {|node, deleted| tree.delete_node(node, deleted)}
+        #TODO: delete output path
+
+        # source paths that should be used
+        paths_to_delete - paths_not_to_delete
+      end
+
+    end
+
+  end
+
+end

data/lib/webgen/sourcehandler/base.rb

@@ -0,0 +1,211 @@
+require 'webgen/websiteaccess'
+require 'webgen/loggable'
+require 'webgen/page'
+
+module Webgen::SourceHandler
+
+  # This module should be included in every source handler as it provides the default methods for
+  # creating nodes.
+  #
+  # = Implementing Source Handlers
+  #
+  # A source handler is a webgen extension that processes source paths to create nodes and that
+  # provides the rendered content of these nodes. The nodes are later written to the output
+  # location. This can range from simply copying a path from the source to the output location to
+  # generating a whole set of nodes from one input path!
+  #
+  # The paths that are handled by a source handler are specified via path patterns (see
+  # below). During a webgen run the #create_node method for each source paths that matches a
+  # specified path pattern is called. And when it is time to write out the node, the the #content
+  # method is called to retrieve the rendered content.
+  #
+  # A source handler must not take any parameters on initialization and when this module is not
+  # mixed in, the methods #create_node and #content need to be defined. Also, a source handler does
+  # not need to reside under the Webgen::SourceHandler namespace but all shipped ones do.
+  #
+  # This base class provides useful default implementations of methods that are used by nearly all
+  # source handler class:
+  # * #create_node
+  # * #node_exists?
+  #
+  # It also provides other utility methods:
+  # * #page_from_path
+  # * #content
+  #
+  # = Nodes Created for Paths
+  #
+  # The main functions of a source handler class are to create one or more nodes for a source path
+  # and to provide the content of these nodes. To achieve this, certain information needs to be set
+  # on a node. If you use the methods provided by this base class, you don't need to set them
+  # explicitly because this is done by the provided methods.
+  #
+  # <tt>node_info[:processor]</tt>:: Has to be set to the class name of the source handler. This is
+  #                                  used by the Node class: all unknown method calls are forwarded
+  #                                  to the node processor.
+  # <tt>node_info[:src]</tt>:: Has to be set to the string version of the source path.
+  # <tt>meta_info['no_output']</tt>:: Has to be set to +true+ on nodes that are used during a
+  #                                   webgen run but do not produce an output file.
+  #
+  # Additional information that is used only for processing purposes should be stored in the
+  # #node_info hash of a node as the #meta_info hash is reserved for real node meta information and
+  # should not be changed once the node is created.
+  #
+  # = Path Patterns and Invocation order
+  #
+  # Path patterns define which paths are handled by a specific source handler. These patterns are
+  # specified in the <tt>sourcehandler.patterns</tt> configuration hash as a mapping from the source
+  # handler class name to an array of path patterns. The patterns need to have a format that
+  # <tt>Dir.glob</tt> can handle.
+  #
+  # Specifying a path pattern does not mean that webgen uses the source handler. One also needs to
+  # provide an entry in the configuration value <tt>sourcehandler.invoke</tt>. This is a hash that
+  # maps the invocation rank (a number) to an array of source handler class names. The lower the
+  # invocation rank the earlier the specified source handlers are used.
+  #
+  # The default invocation ranks are:
+  # 1:: Early. Normally there is no need to use this rank.
+  # 5:: Standard. This is the rank the normal source handler should use.
+  # 9:: Late. This rank should be used by source handlers that operate on/use already created nodes
+  #     and need to ensure that these nodes are available.
+  #
+  # = Default Meta Information
+  #
+  # Each source handler can define default meta information that gets automatically set on the
+  # source paths that are passed to the #create_node method.
+  #
+  # The default meta information is specified in the <tt>sourcehandler.default_meta_info</tt>
+  # configuration hash as a mapping from the source handler class name to the meta information
+  # hash.
+  #
+  # = Sample Source Handler Class
+  #
+  # Following is a simple source handler class example which copies paths from the source to
+  # the output location modifying the extension:
+  #
+  #   class SimpleCopy
+  #
+  #     include Webgen::SourceHandler::Base
+  #
+  #     def create_node(parent, path)
+  #       path.ext += '.copied'
+  #       super(parent, path)
+  #     end
+  #
+  #     def content(node)
+  #       website.blackboard.invoke(:source_paths)[node.node_info[:src]].io
+  #     end
+  #
+  #   end
+  #
+  #   WebsiteAccess.website.config.sourcehandler.patterns['SimpleCopy'] = ['**/*.jpg', '**/*.png']
+  #   WebsiteAccess.website.config.sourcehandler.invoke[5] << 'SimpleCopy'
+  #
+  module Base
+
+    # This module is used for defining all methods that can be used for creating output paths.
+    #
+    # All public methods of this module are considered to be output path creation methods and must
+    # have the following parameters:
+    # +parent+:: the parent node
+    # +path+:: the path for which the output name should be created
+    # +use_lang_part+:: controls whether the output path name has to include the language part
+    module OutputPathHelpers
+
+      # Default method for creating an output path for +parent+ and source +path+.
+      #
+      # The automatically set parameter +style+ (which uses the meta information +output_path_style+
+      # from the path's meta information hash) defines how the output name should be built (more
+      # information about this in the user documentation).
+      def standard_output_path(parent, path, use_lang_part, style = path.meta_info['output_path_style'])
+        result = style.collect do |part|
+          case part
+          when String  then part
+          when :lang   then use_lang_part ? path.meta_info['lang'] : ''
+          when :ext    then path.ext.empty? ? '' : '.' + path.ext
+          when :parent then temp = parent; temp = temp.parent while temp.is_fragment?; temp.path
+          when :year, :month, :day
+            ctime = path.meta_info['created_at']
+            if !ctime.kind_of?(Time)
+              raise "Invalid meta info 'created_at' for #{path}, needed because of used output path style"
+            end
+            ctime.send(part).to_s.rjust(2, '0')
+          when Symbol  then path.send(part)
+          when Array   then part.include?(:lang) && !use_lang_part ? '' : standard_output_path(parent, path, use_lang_part, part)
+          else ''
+          end
+        end
+        result.join('')
+      end
+
+    end
+
+    include Webgen::Loggable
+    include OutputPathHelpers
+
+    # Construct the output name for the given +path+ and +parent+. First it is checked if a node
+    # with the constructed output name already exists. If it exists, the language part is forced to
+    # be in the output name and the resulting output name is returned.
+    def output_path(parent, path)
+      method = path.meta_info['output_path'] + '_output_path'
+      use_lang_part = if path.meta_info['lang'].nil? # unlocalized files never get a lang in the filename!
+                        false
+                      else
+                        Webgen::WebsiteAccess.website.config['sourcehandler.default_lang_in_output_path'] ||
+                          Webgen::WebsiteAccess.website.config['website.lang'] != path.meta_info['lang']
+                      end
+      if OutputPathHelpers.public_instance_methods(false).include?(method)
+        name = send(method, parent, path, use_lang_part)
+        name += '/'  if path.path =~ /\/$/ && name !~ /\/$/
+        if node_exists?(parent, path, name)
+          name = send(method, parent, path, (path.meta_info['lang'].nil? ? false : true))
+          name += '/'  if path.path =~ /\/$/ && name !~ /\/$/
+        end
+        name
+      else
+        raise "Unknown method for creating output path: #{method}"
+      end
+    end
+
+    # Check if the node alcn and output path which would be created by #create_node exists. The
+    # +output_path+ to check for can individually be set.
+    def node_exists?(parent, path, output_path = self.output_path(parent, path))
+      parent.tree[Webgen::Node.absolute_name(parent, path.lcn, :alcn)] || (!path.meta_info['no_output'] && parent.tree[output_path, :path])
+    end
+
+    # Create and return a node under +parent+ from +path+ if it does not already exists. Some
+    # additional node information like <tt>:src</tt> and <tt>:processor</tt> is set and the meta
+    # information is checked for validness. The created node is yielded if a block is given.
+    def create_node(parent, path, output_path = self.output_path(parent, path))
+      return if path.meta_info['draft']
+      if !node_exists?(parent, path, output_path)
+        node = Webgen::Node.new(parent, output_path, path.cn, path.meta_info)
+        node['modified_at'] = Time.now unless node['modified_at'].kind_of?(Time)
+        node.node_info[:src] = path.path
+        node.node_info[:processor] = self.class.name
+        yield(node) if block_given?
+        node
+      else
+        log(:warn) { "Node already exists: output path = #{output_path} | alcn = #{Webgen::Node.absolute_name(parent, path.lcn, :alcn)}"}
+      end
+    end
+
+    # Return the content of the given +node+. This default +content+ method just returns +nil+.
+    def content(node)
+      nil
+    end
+
+    # Utility method for creating a Webgen::Page object from the +path+. Also updates
+    # <tt>path.meta_info</tt> with the meta info from the page.
+    def page_from_path(path)
+      begin
+        page = Webgen::Page.from_data(path.io.data, path.meta_info)
+      rescue Webgen::WebgenPageFormatError => e
+        raise "Error reading source path <#{path}>: #{e.message}"
+      end
+      path.meta_info = page.meta_info
+      page
+    end
+
+  end
+
+end