thewoolleyman-webgen 0.5.8.20090419

data/lib/webgen/source.rb

@@ -0,0 +1,54 @@
+# -*- encoding: utf-8 -*-
+
+module Webgen
+
+  # Namespace for all classes that provide source paths.
+  #
+  # == Implementing a source class
+  #
+  # Source classes provide access to the source paths on which the source handlers act.
+  #
+  # A source class only needs to respond to the method +paths+ which needs to return a set of paths
+  # for the source. The returned paths must respond to the method <tt>changed?</tt> (has to return
+  # +true+ if the paths has changed since the last webgen run). The default implementation in the
+  # Path class just returns +true+. One can either derive a specialized path class or define
+  # singleton methods on each path object.
+  #
+  # == Sample Source Class
+  #
+  # Following is a simple source class which has stored the paths and their contents in a hash:
+  #
+  #   require 'stringio'
+  #
+  #   class MemorySource
+  #
+  #     CONTENT = {
+  #       '/directory/' => nil,
+  #       '/directory/file.page' => "This is the content of the file"
+  #     }
+  #
+  #     def paths
+  #       CONTENT.collect do |path, content|
+  #         Webgen::Path.new(path) { StringIO.new(content.to_s) }
+  #       end.to_set
+  #     end
+  #
+  #   end
+  #
+  # You can use this source class in your website (after placing the code in, for example,
+  # <tt>ext/init.rb</tt>) by updating the <tt>sources</tt> configuration option (the following code
+  # has to be placed after the definition of the +MemorySource+ class):
+  #
+  #   WebsiteAccess.website.config['sources'] << ['/', MemorySource]
+  #
+  module Source
+
+    autoload :Base, 'webgen/source/base'
+    autoload :FileSystem, 'webgen/source/filesystem'
+    autoload :Stacked, 'webgen/source/stacked'
+    autoload :Resource, 'webgen/source/resource'
+    autoload :TarArchive, 'webgen/source/tararchive'
+
+  end
+
+end

data/lib/webgen/source/filesystem.rb

@@ -0,0 +1,61 @@
+# -*- encoding: utf-8 -*-
+
+require 'pathname'
+require 'webgen/websiteaccess'
+require 'webgen/path'
+
+module Webgen
+
+  # This class is used to read source paths from a directory in the file system.
+  class Source::FileSystem
+
+    # A special Webgen::Path class for handling with file system paths.
+    class Path < Webgen::Path
+
+      # Create a new object with absolute path +path+ for the file system path +fs_path+.
+      def initialize(path, fs_path)
+        super(path) { File.open(fs_path, 'rb') }
+        @fs_path = fs_path
+        WebsiteAccess.website.cache[[:fs_path, @fs_path]] = File.mtime(@fs_path)
+        @meta_info['modified_at'] = File.mtime(@fs_path)
+      end
+
+      # Return +true+ if the file system path used by the object has been modified.
+      def changed?
+        data = WebsiteAccess.website.cache[[:fs_path, @fs_path]]
+        File.mtime(@fs_path) > data
+      end
+
+    end
+
+    # The root path from which paths read.
+    attr_reader :root
+
+    # The glob (see Dir.glob for details) that is used to specify which paths under the root path
+    # should be returned by #paths.
+    attr_reader :glob
+
+    # Create a new file system source for the root path +root+ using the provided +glob+.
+    def initialize(root, glob = '**/*')
+      if root =~ /^([a-zA-Z]:|\/)/
+        @root = root
+      else
+        @root = File.join(WebsiteAccess.website.directory, root)
+      end
+      @glob = glob
+    end
+
+    # Return all paths under #root which match #glob.
+    def paths
+      @paths ||= Dir.glob(File.join(@root, @glob), File::FNM_DOTMATCH|File::FNM_CASEFOLD).to_set.collect do |f|
+        next unless File.exists?(f) # handle invalid links
+        temp = Pathname.new(f.sub(/^#{Regexp.escape(@root)}\/?/, '/')).cleanpath.to_s
+        temp += '/' if File.directory?(f) && temp[-1] != ?/
+        path = Path.new(temp, f)
+        path
+      end.compact
+    end
+
+  end
+
+end

data/lib/webgen/source/resource.rb

@@ -0,0 +1,44 @@
+# -*- encoding: utf-8 -*-
+
+require 'webgen/websiteaccess'
+require 'webgen/source'
+
+module Webgen::Source
+
+  # This class is used to provide access to sources provided by resources.
+  class Resource
+
+    include Webgen::WebsiteAccess
+
+    # The glob (see File.fnmatch) specifying the resources.
+    attr_reader :glob
+
+    # The glob (see File.fnmatch) specifying the paths that should be used from the resources.
+    attr_reader :paths_glob
+
+    # The prefix that should optionally be stripped from the paths.
+    attr_reader :strip_prefix
+
+    # Create a new resource source for the the +glob+ and use only those paths matching +paths_glob+
+    # while stripping +strip_prefix+ off the path.
+    def initialize(glob, paths_glob = nil, strip_prefix = nil)
+      @glob, @paths_glob, @strip_prefix = glob, paths_glob, strip_prefix
+    end
+
+    # Return all paths associated with the resources identified by #glob.
+    def paths
+      if !defined?(@paths)
+        stack = Stacked.new
+        website.config['resources'].select {|name, infos| File.fnmatch(@glob, name)}.sort.each do |name, infos|
+          stack.add([['/', constant(infos.first).new(*infos[1..-1])]])
+        end
+        @paths = stack.paths
+        @paths = @paths.select {|p| File.fnmatch(@paths_glob, p)} if @paths_glob
+        @paths.collect! {|p| p.mount_at('/', @strip_prefix)} if @strip_prefix
+      end
+      @paths
+    end
+
+  end
+
+end

data/lib/webgen/source/stacked.rb

@@ -0,0 +1,55 @@
+# -*- encoding: utf-8 -*-
+
+module Webgen::Source
+
+  # This source class is used to stack several sources together.
+  #
+  # It serves two purposes:
+  #
+  # * First, it can be used to access more than one source. This is useful when your website
+  #   consists of more than one source directory and you want to use all of them.
+  #
+  # * Second, sources can be mounted on specific directories. For example, a folder with images that
+  #   you don't want to copy to the website source directory can be mounted under <tt>/images</tt>
+  #   sothat they are available nonetheless.
+  #
+  # Also be aware that when a path is returned by a source that has already be returned by a prior
+  # source, it is discarded and not used.
+  class Stacked
+
+    # Return the stack of Webgen::Source objects.
+    attr_reader :stack
+
+    # Create a new stack. The optional +map+ parameter can be used to provide initial mappings of
+    # mount points to source objects (see #add for details).
+    def initialize(map = {})
+      @stack = []
+      add(map)
+    end
+
+    # Add all mappings found in +maps+ to the stack. The parameter +maps+ should be an array of
+    # two-element arrays which contain an absolute directoriy (ie. starting with a slash) and a
+    # source object.
+    def add(maps)
+      maps.each do |mp, source|
+        raise "Invalid mount point specified: #{mp}" unless mp =~ /^\//
+        @stack << [mp, source]
+      end
+    end
+
+    # Return all paths returned by the sources in the stack. Since the stack is ordered, paths
+    # returned by later source objects are not used if a prior source object has returned the same
+    # path.
+    def paths
+      @paths = Set.new
+      @stack.each do |mp, source|
+        source.paths.each do |path|
+          @paths.add?(path.mount_at(mp))
+        end
+      end
+      @paths
+    end
+
+  end
+
+end

data/lib/webgen/source/tararchive.rb

@@ -0,0 +1,73 @@
+# -*- encoding: utf-8 -*-
+
+require 'pathname'
+require 'webgen/websiteaccess'
+require 'webgen/path'
+require 'open-uri'
+require 'zlib'
+require 'archive/tar/minitar'
+
+module Webgen
+
+  # This class is used to read source paths from a (gzipped) tar archive. The archive can be remote
+  # (http(s) or ftp) or local.
+  #
+  # For example, the following are all valid URIs:
+  #   http://example.com/directory/file.tgz
+  #   /home/test/my.tar.gz
+  #   ftp://ftp.example.com/archives/archive.tar
+  #
+  class Source::TarArchive
+
+    # A special Webgen::Path class for handling paths from a tar archive.
+    class Path < Webgen::Path
+
+      # Create a new tar archive path object for the entry +entry+.
+      def initialize(path, data, mtime, uri)
+        super(path) { StringIO.new(data.to_s) }
+        @uri = uri
+        @mtime = mtime
+        WebsiteAccess.website.cache[[:tararchive_path, @uri, path]] = @mtime if WebsiteAccess.website
+        @meta_info['modified_at'] = @mtime
+      end
+
+      # Return +true+ if the tar archive path used by the object has been modified.
+      def changed?
+        !WebsiteAccess.website || @mtime > WebsiteAccess.website.cache[[:tararchive_path, @uri, path]]
+      end
+
+    end
+
+    # The URI of the tar archive.
+    attr_reader :uri
+
+    # The glob (see File.fnmatch for details) that is used to specify which paths in the archive should
+    # be returned by #paths.
+    attr_reader :glob
+
+    # Create a new tar archive source for the URI string +uri+.
+    def initialize(uri, glob = '**/*')
+      @uri = uri
+      @glob = glob
+    end
+
+    # Return all paths in the tar archive available at #uri.
+    def paths
+      if !defined?(@paths)
+        stream = open(@uri)
+        stream = Zlib::GzipReader.new(stream) if @uri =~ /(\.tar\.gz|\.tgz)$/
+        Archive::Tar::Minitar::Input.open(stream) do |input|
+          @paths = input.collect do |entry|
+            path = entry.full_name
+            next unless File.fnmatch(@glob, path, File::FNM_DOTMATCH|File::FNM_CASEFOLD|File::FNM_PATHNAME)
+            path += '/' if entry.directory? && path[-1] != ?/
+            Path.new(path, entry.read, Time.at(entry.mtime), @uri)
+          end.compact.to_set
+        end
+      end
+      @paths
+    end
+
+  end
+
+end

data/lib/webgen/sourcehandler.rb

@@ -0,0 +1,226 @@
+# -*- encoding: utf-8 -*-
+
+require 'webgen/loggable'
+require 'benchmark'
+
+module Webgen
+
+  # Namespace for all classes that handle source paths.
+  #
+  # Have a look at Webgen::SourceHandler::Base for details on how to implement a source handler
+  # class.
+  module SourceHandler
+
+    autoload :Base, 'webgen/sourcehandler/base'
+    autoload :Copy, 'webgen/sourcehandler/copy'
+    autoload :Directory, 'webgen/sourcehandler/directory'
+    autoload :Metainfo, 'webgen/sourcehandler/metainfo'
+    autoload :Template, 'webgen/sourcehandler/template'
+    autoload :Page, 'webgen/sourcehandler/page'
+    autoload :Fragment, 'webgen/sourcehandler/fragment'
+    autoload :Virtual, 'webgen/sourcehandler/virtual'
+    autoload :Feed, 'webgen/sourcehandler/feed'
+    autoload :Sitemap, 'webgen/sourcehandler/sitemap'
+    autoload :Memory, 'webgen/sourcehandler/memory'
+
+    # This class is used by Website to do the actual rendering of the website. It
+    #
+    # * collects all source paths using the source classes
+    # * creates nodes using the source handler classes
+    # * writes changed nodes out using an output class
+    class Main
+
+      include WebsiteAccess
+      include Loggable
+
+      def initialize #:nodoc:
+        website.blackboard.add_service(:create_nodes, method(:create_nodes))
+        website.blackboard.add_service(:source_paths, method(:find_all_source_paths))
+        website.blackboard.add_listener(:node_meta_info_changed?, method(:meta_info_changed?))
+      end
+
+      # Render the nodes provided in the +tree+. Before the actual rendering is done, the sources
+      # are checked (nodes for deleted sources are deleted, nodes for new and changed sources).
+      def render(tree)
+        begin
+          website.logger.mark_new_cycle if website.logger
+
+          puts "Updating tree..."
+          time = Benchmark.measure do
+            website.cache.reset_volatile_cache
+            update_tree(tree)
+          end
+          puts "...done in " + ('%2.4f' % time.real) + ' seconds'
+
+          if !tree.root
+            puts 'No source files found - maybe not a webgen website?'
+            return nil
+          end
+
+          puts "Writing changed nodes..."
+          time = Benchmark.measure do
+            write_tree(tree)
+          end
+          puts "...done in " + ('%2.4f' % time.real) + ' seconds'
+        end while tree.node_access[:alcn].any? {|name,node| node.flagged(:created) || node.flagged(:reinit)}
+        :success
+      end
+
+      #######
+      private
+      #######
+
+      # Update the +tree+ by creating/reinitializing all needed nodes.
+      def update_tree(tree)
+        unused_paths = Set.new
+        begin
+          used_paths = Set.new(find_all_source_paths.keys) - unused_paths
+          paths_to_use = Set.new
+          nodes_to_delete = Set.new
+
+          tree.node_access[:alcn].each do |alcn, node|
+            next if node == tree.dummy_root
+            used_paths.delete(node.node_info[:src])
+
+            deleted = !find_all_source_paths.include?(node.node_info[:src])
+            if deleted
+              nodes_to_delete << node
+              #TODO: delete output path
+            elsif (!node.flagged(:created) && find_all_source_paths[node.node_info[:src]].changed?) || node.meta_info_changed?
+              node.flag(:reinit)
+              paths_to_use << node.node_info[:src]
+            elsif node.changed?
+              # nothing to be done here
+            end
+          end
+
+          nodes_to_delete.each {|node| tree.delete_node(node)}
+          used_paths.merge(paths_to_use)
+          paths = create_nodes_from_paths(tree, used_paths.to_a.sort)
+          unused_paths.merge(used_paths - paths)
+          tree.node_access[:alcn].each {|name, node| tree.delete_node(node) if node.flagged(:reinit)}
+          website.cache.reset_volatile_cache
+        end until used_paths.empty?
+      end
+
+      # Write out all changed nodes of the +tree+.
+      def write_tree(tree)
+        output = website.blackboard.invoke(:output_instance)
+
+        tree.node_access[:alcn].select do |name, node|
+          use_node = (node != tree.dummy_root && node.flagged(:dirty))
+          node.unflag(:dirty_meta_info)
+          node.unflag(:created)
+          node.unflag(:dirty)
+          use_node
+        end.sort.each do |name, node|
+          next if node['no_output'] || !(content = node.content)
+
+          begin
+            puts " "*4 + name, :verbose
+            type = if node.is_directory?
+                     :directory
+                   elsif node.is_fragment?
+                     :fragment
+                   else
+                     :file
+                   end
+            output.write(node.path, content, type)
+          rescue
+            raise RuntimeError, "Error while processing <#{node.absolute_lcn}>: #{$!.message}", $!.backtrace
+          end
+        end
+      end
+
+      # Return a hash with all source paths.
+      def find_all_source_paths
+        if !defined?(@paths)
+          source = Webgen::Source::Stacked.new(website.config['sources'].collect do |mp, name, *args|
+                                                 [mp, constant(name).new(*args)]
+                                               end)
+          @paths = {}
+          source.paths.each do |path|
+            if !(website.config['sourcehandler.ignore'].any? {|pat| File.fnmatch(pat, path, File::FNM_CASEFOLD|File::FNM_DOTMATCH)})
+              @paths[path.source_path] = path
+            end
+          end
+        end
+        @paths
+      end
+
+      # Return only the subset of +paths+ which are handled by the source handler +name+.
+      def paths_for_handler(name, paths)
+        patterns = website.config['sourcehandler.patterns'][name]
+        return [] if patterns.nil?
+
+        options = (website.config['sourcehandler.casefold'] ? File::FNM_CASEFOLD : 0) |
+          (website.config['sourcehandler.use_hidden_files'] ? File::FNM_DOTMATCH : 0)
+        find_all_source_paths.values_at(*paths).select do |path|
+          patterns.any? {|pat| File.fnmatch(pat, path, options)}
+        end
+      end
+
+      # Use the source handlers to create nodes for the +paths+ in the +tree+.
+      def create_nodes_from_paths(tree, paths)
+        used_paths = Set.new
+        website.config['sourcehandler.invoke'].sort.each do |priority, shns|
+          shns.each do |shn|
+            sh = website.cache.instance(shn)
+            handler_paths = paths_for_handler(shn, paths)
+            used_paths.merge(handler_paths)
+            handler_paths.sort {|a,b| a.path.length <=> b.path.length}.each do |path|
+              parent_dir = path.directory.split('/').collect {|p| Path.new(p).cn}.join('/')
+              parent_dir += '/' if path != '/' && parent_dir == ''
+              create_nodes(tree, parent_dir, path, sh)
+            end
+          end
+        end
+        used_paths
+      end
+
+      # Prepare everything to create nodes under the absolute lcn path +parent_path_name+ in the
+      # +tree from the +path+ using the +source_handler+. If a block is given, the actual creation
+      # of the nodes is deferred to it. After the nodes are created, it is also checked if they have
+      # all needed properties.
+      def create_nodes(tree, parent_path_name, path, source_handler) #:yields: parent, path
+        if !(parent = tree[parent_path_name])
+          raise "The specified parent path <#{parent_path_name}> does not exist"
+        end
+        path = path.dup
+        path.meta_info = default_meta_info(path, source_handler.class.name)
+        (website.cache[:sourcehandler_path_mi] ||= {})[[path.path, source_handler.class.name]] = path.meta_info.dup
+        website.blackboard.dispatch_msg(:before_node_created, parent, path)
+        *nodes = if block_given?
+                   yield(parent, path)
+                 else
+                   source_handler.create_node(parent, path)
+                 end
+        nodes.flatten.compact.each do |node|
+          website.blackboard.dispatch_msg(:after_node_created, node)
+        end
+        nodes
+      end
+
+      # Return the default meta info for the pair of +path+ and +sh_name+.
+      def default_meta_info(path, sh_name)
+        path.meta_info.merge(website.config['sourcehandler.default_meta_info'][:all]).
+          merge(website.config['sourcehandler.default_meta_info'][sh_name] || {})
+      end
+
+      # Check if the default meta information for +node+ has changed since the last run. But don't
+      # take the node's path's +modified_at+ meta information into account since that changes on
+      # every path change.
+      def meta_info_changed?(node)
+        path = node.node_info[:creation_path]
+        old_mi = website.cache[:sourcehandler_path_mi][[path, node.node_info[:processor]]]
+        old_mi.delete('modified_at')
+        new_mi = default_meta_info(@paths[path] || Webgen::Path.new(path), node.node_info[:processor])
+        new_mi.delete('modified_at')
+        node.flag(:dirty_meta_info) if !old_mi || old_mi != new_mi
+      end
+
+    end
+
+  end
+
+end