nanoc3 3.0.0

data/lib/nanoc3/base/compiler_dsl.rb

@@ -0,0 +1,110 @@
+# encoding: utf-8
+
+module Nanoc3
+
+  # Nanoc3::CompilerDSL contains methods that will be executed by the site's
+  # rules file.
+  class CompilerDSL
+
+    # Creates a new compiler DSL for the given compiler.
+    def initialize(site)
+      @site = site
+    end
+
+    # Creates a preprocessor block that will be executed after all data is
+    # loaded, but before the site is compiled.
+    def preprocess(&block)
+      @site.preprocessor = block
+    end
+
+    # Creates a compilation rule for all items whose identifier match the
+    # given identifier, which may either be a string containing the *
+    # wildcard, or a regular expression.
+    #
+    # This rule will be applicable to reps with a name equal to "default"
+    # unless an explicit :rep parameter is given.
+    #
+    # An item rep will be compiled by calling the given block and passing the
+    # rep as a block argument.
+    #
+    # Example:
+    #
+    #   compile '/foo/*' do
+    #     rep.filter :erb
+    #   end
+    #   
+    #   compile '/bar/*', :rep => 'raw' do
+    #     # do nothing
+    #   end
+    def compile(identifier, params={}, &block)
+      # Require block
+      raise ArgumentError.new("#compile requires a block") unless block_given?
+
+      # Get rep name
+      rep_name = params[:rep] || :default
+
+      # Create rule
+      rule = Rule.new(identifier_to_regex(identifier), rep_name, block)
+      @site.compiler.item_compilation_rules << rule
+    end
+
+    # Creates a routing rule for all items whose identifier match the
+    # given identifier, which may either be a string containing the *
+    # wildcard, or a regular expression.
+    #
+    # This rule will be applicable to reps with a name equal to "default";
+    # this can be changed by givign an explicit :rep parameter.
+    #
+    # The path of an item rep will be determined by calling the given block
+    # and passing the rep as a block argument.
+    #
+    # Example:
+    #
+    #   route '/foo/*' do
+    #     '/blahblah' + rep.item.identifier + 'index.html'
+    #   end
+    #   
+    #   route '/bar/*', :rep => 'raw' do
+    #     '/blahblah' + rep.item.identifier + 'index.txt'
+    #   end
+    def route(identifier, params={}, &block)
+      # Require block
+      raise ArgumentError.new("#route requires a block") unless block_given?
+
+      # Get rep name
+      rep_name = params[:rep] || :default
+
+      # Create rule
+      rule = Rule.new(identifier_to_regex(identifier), rep_name, block)
+      @site.compiler.item_routing_rules << rule
+    end
+
+    # Creates a layout rule for all layouts whose identifier match the given
+    # identifier, which may either be a string containing the * wildcard, or a
+    # regular expression. The layouts matching the identifier will be filtered
+    # using the filter specified in the second argument. The params hash
+    # contains filter arguments that will be passed to the filter.
+    #
+    # Example:
+    #
+    #   layout '/default/', :erb
+    #   layout '/custom/',  :haml, :format => :html5
+    def layout(identifier, filter_name, params={})
+      @site.compiler.layout_filter_mapping[identifier_to_regex(identifier)] = [ filter_name, params ]
+    end
+
+  private
+
+    # Converts the given identifier, which can contain the '*' wildcard, to a regex.
+    # For example, 'foo/*/bar' is transformed into /^foo\/(.*?)\/bar$/.
+    def identifier_to_regex(identifier)
+      if identifier.is_a? String
+        /^#{identifier.cleaned_identifier.gsub('*', '(.*?)')}?$/
+      else
+        identifier
+      end
+    end
+
+  end
+
+end

data/lib/nanoc3/base/core_ext/array.rb

@@ -0,0 +1,21 @@
+# encoding: utf-8
+
+module Nanoc3::ArrayExtensions
+
+  def symbolize_keys
+    inject([]) do |array, element|
+      array + [ element.respond_to?(:symbolize_keys) ? element.symbolize_keys : element ]
+    end
+  end
+
+  def stringify_keys
+    inject([]) do |array, element|
+      array + [ element.respond_to?(:stringify_keys) ? element.symbolize_keys : element ]
+    end
+  end
+
+end
+
+class Array
+  include Nanoc3::ArrayExtensions
+end

data/lib/nanoc3/base/core_ext/hash.rb

@@ -0,0 +1,23 @@
+# encoding: utf-8
+
+module Nanoc3::HashExtensions
+
+  # Returns a new hash where all keys are recursively converted into symbols.
+  def symbolize_keys
+    inject({}) do |hash, (key, value)|
+      hash.merge(key.to_sym => value.respond_to?(:symbolize_keys) ? value.symbolize_keys : value)
+    end
+  end
+
+  # Returns a new hash where all keys are recursively converted to strings.
+  def stringify_keys
+    inject({}) do |hash, (key, value)|
+      hash.merge(key.to_s => value.respond_to?(:stringify_keys) ? value.stringify_keys : value)
+    end
+  end
+
+end
+
+class Hash
+  include Nanoc3::HashExtensions
+end

data/lib/nanoc3/base/core_ext/string.rb

@@ -0,0 +1,14 @@
+# encoding: utf-8
+
+module Nanoc3::StringExtensions
+
+  # Transforms string into an actual identifier
+  def cleaned_identifier
+    "/#{self}/".gsub(/^\/+|\/+$/, '/')
+  end
+
+end
+
+class String
+  include Nanoc3::StringExtensions
+end

data/lib/nanoc3/base/core_ext.rb

@@ -0,0 +1,5 @@
+# encoding: utf-8
+
+require 'nanoc3/base/core_ext/array'
+require 'nanoc3/base/core_ext/hash'
+require 'nanoc3/base/core_ext/string'

data/lib/nanoc3/base/data_source.rb

@@ -0,0 +1,197 @@
+# encoding: utf-8
+
+module Nanoc3
+
+  # Nanoc3::DataSource is responsible for loading data. It is the (abstract)
+  # superclass for all data sources. Subclasses must at least implement the
+  # data reading methods (+items+, +layouts+, and +code_snippets+); all other
+  # methods involving data manipulation are optional.
+  #
+  # Apart from the methods for loading and storing data, there are the +up+
+  # and +down+ methods for bringing up and tearing down the connection to the
+  # data source. These should be overridden in subclasses. The +loading+
+  # method wraps +up+ and +down+.
+  #
+  # The +setup+ method is used for setting up a site's data source for the
+  # first time. This method should be overridden in subclasses.
+  class DataSource < Plugin
+
+    # A string containing the root where items returned by this data source
+    # should be mounted.
+    attr_reader :items_root
+
+    # A string containing the root where layouts returned by this data source
+    # should be mounted.
+    attr_reader :layouts_root
+
+    # A hash containing the configuration for this data source. For example,
+    # online data sources could contain authentication details.
+    attr_reader :config
+
+    # Creates a new data source for the given site.
+    #
+    # +site+:: The site this data source belongs to.
+    # +items_root+:: The prefix that should be given to all items returned by
+    #                the #items method (comparable to mount points for
+    #                filesystems in Unix-ish OSes).
+    # +layouts_root+:: The prefix that should be given to all layouts returned
+    #                  by the #layouts method (comparable to mount points for
+    #                  filesystems in Unix-ish OSes).
+    # +config+:: The configuration for this data source.
+    def initialize(site, items_root, layouts_root, config)
+      @site         = site
+      @items_root   = items_root
+      @layouts_root = layouts_root
+      @config       = config
+
+      @references = 0
+    end
+
+    # Sets the identifiers for this data source.
+    def self.identifiers(*identifiers)
+      Nanoc3::DataSource.register(self, *identifiers)
+    end
+
+    # Sets the identifier for this data source.
+    def self.identifier(identifier)
+      Nanoc3::DataSource.register(self, identifier)
+    end
+
+    # Registers the given class as a data source with the given identifier.
+    def self.register(class_or_name, *identifiers)
+      Nanoc3::Plugin.register(Nanoc3::DataSource, class_or_name, *identifiers)
+    end
+
+    # Loads the data source when necessary (calling +up+), yields, and unloads
+    # the data source when it is not being used elsewhere. All data source
+    # queries and data manipulations should be wrapped in a +loading+ block;
+    # it ensures that the data source is loaded when necessary and makes sure
+    # the data source does not get unloaded while it is still being used
+    # elsewhere.
+    def loading
+      use
+      yield
+    ensure
+      unuse
+    end
+
+    # Marks the data source as used by the caller.
+    #
+    # Calling this method increases the internal reference count. When the
+    # data source is used for the first time (first #use call), the data
+    # source will be loaded (#up will be called). Similarly, when the
+    # reference count reaches zero, the data source will be unloaded (#down
+    # will be called).
+    def use
+      up if @references == 0
+      @references += 1
+    end
+
+    # Marks the data source as unused by the caller.
+    #
+    # Calling this method increases the internal reference count. When the
+    # data source is used for the first time (first #use call), the data
+    # source will be loaded (#up will be called). Similarly, when the
+    # reference count reaches zero, the data source will be unloaded (#down
+    # will be called).
+    def unuse
+      @references -= 1
+      down if @references == 0
+    end
+
+    ########## Loading and unloading
+
+    # Brings up the connection to the data. This is an abstract method
+    # implemented by the subclass. Depending on the way data is stored, this
+    # may not be necessary. This is the ideal place to connect to the
+    # database, for example.
+    #
+    # Subclasses may implement this method.
+    def up
+    end
+
+    # Brings down the connection to the data. This is an abstract method
+    # implemented by the subclass. This method should undo the effects of
+    # +up+.
+    #
+    # Subclasses may implement this method.
+    def down
+    end
+
+    ########## Creating/updating
+
+    # Creates the bare minimum essentials for this data source to work. This
+    # action will likely be destructive. This method should not create sample
+    # data such as a default home page, a default layout, etc. For example, if
+    # you're using a database, this is where you should create the necessary
+    # tables for the data source to function properly.
+    #
+    # Subclasses must implement this method.
+    def setup
+      not_implemented('setup')
+    end
+
+    # Updated the content stored in this site to a newer version. A newer
+    # version of a data source may store content in a different format, and
+    # this method will update the stored content to this newer format.
+    #
+    # Subclasses may implement this method.
+    def update
+    end
+
+    ########## Loading data
+
+    # Returns the list of items (represented by Nanoc3::Item) in this site.
+    # The default implementation simply returns an empty array.
+    #
+    # Subclasses should not prepend items_root to the item's identifiers, as
+    # this will be done automatically.
+    #
+    # Subclasses may implement this method.
+    def items
+      []
+    end
+
+    # Returns the list of layouts (represented by Nanoc3::Layout) in this
+    # site. The default implementation simply returns an empty array.
+    #
+    # Subclasses should prepend layout_root to the layout's identifiers, since
+    # this is not done automatically.
+    #
+    # Subclasses may implement this method.
+    def layouts
+      []
+    end
+
+    # Returns the custom code snippets (represented by Nanoc3::CodeSnippet)
+    # for this site. The default implementation simply returns an empty array.
+    # This can be code for custom filters and more, but pretty much any code
+    # can be put in there (global helper functions are very useful).
+    #
+    # Subclasses may implement this method.
+    def code_snippets
+      []
+    end
+
+    ########## Creating data
+
+    # Creates a new item with the given content, attributes and identifier.
+    def create_item(content, attributes, identifier)
+      not_implemented('create_item')
+    end
+
+    # Creates a new layout with the given content, attributes and identifier.
+    def create_layout(content, attributes, identifier)
+      not_implemented('create_layout')
+    end
+
+  private
+
+    def not_implemented(name)
+      raise NotImplementedError.new(
+        "#{self.class} does not implement ##{name}"
+      )
+    end
+
+  end
+end

data/lib/nanoc3/base/dependency_tracker.rb

@@ -0,0 +1,291 @@
+# encoding: utf-8
+
+require 'pstore'
+
+module Nanoc3
+
+  # Nanoc3::DependencyTracker is responsible for remembering dependencies
+  # between items. It is used to speed up compilation by only letting an item
+  # be recompiled when it is outdated or any of its dependencies (or
+  # dependencies' dependencies, etc) is outdated.
+  #
+  # The dependencies tracked by the dependency tracker are not dependencies
+  # based on an item's content. When one item uses an attribute of another
+  # item, then this is also treated as a dependency. While dependencies based
+  # on an item's content (handled in Nanoc3::Compiler) cannot be mutually
+  # recursive, the more general dependencies in Nanoc3::DependencyTracker can
+  # (e.g. item A can use an attribute of item B and vice versa without
+  # problems).
+  class DependencyTracker
+
+    attr_accessor :filename
+
+    # FIXME The way the graph is stored is not exactly great. An adjacency
+    # matrix (wrapped in a Graph class, perhaps) would be a lot easier to work
+    # with, and it would not require an inverse graph to be maintained.
+
+    # Creates a new dependency tracker for the given items.
+    def initialize(items)
+      @items = items
+
+      @filename = 'tmp/dependencies'
+
+      @graph = {}
+      @inverse_graph = {}
+    end
+
+    # Starts listening for dependency messages (+:visit_started+ and
+    # +:visit_ended+) and start recording dependencies.
+    def start
+      # Initialize dependency stack. An item will be pushed onto this stack
+      # when it is visited. Therefore, an item on the stack always depends on
+      # all items pushed above it.
+      @stack = []
+
+      # Register start of visits
+      Nanoc3::NotificationCenter.on(:visit_started, self) do |item|
+        # Record possible dependency
+        unless @stack.empty?
+          self.record_dependency(@stack[-1], item)
+        end
+
+        @stack.push(item)
+      end
+
+      # Register end of visits
+      Nanoc3::NotificationCenter.on(:visit_ended, self) do |item|
+        @stack.pop
+      end
+    end
+
+    # Stop listening for dependency messages and stop recording dependencies.
+    def stop
+      # Unregister
+      Nanoc3::NotificationCenter.remove(:visit_started, self)
+      Nanoc3::NotificationCenter.remove(:visit_ended,   self)
+    end
+
+    # Returns the direct dependencies for +item+, i.e. the items that, when
+    # outdated, will cause +item+ to be marked as outdated. Indirect
+    # dependencies will not be returned (e.g. if A depends on B which depends
+    # on C, then the direct dependencies of A do not include C).
+    def direct_dependencies_for(item)
+      @graph[item] || []
+    end
+
+    # Returns all dependencies (direct and indirect) for +item+, i.e. the
+    # items that, when outdated, will cause +item+ to be marked as outdated.
+    def all_dependencies_for(item)
+      # FIXME can result in an infinite loop
+
+      direct_dependencies   = direct_dependencies_for(item)
+      indirect_dependencies = direct_dependencies.map { |i| all_dependencies_for(i) }
+
+      (direct_dependencies + indirect_dependencies).flatten
+    end
+
+    # Returns the direct inverse dependencies for +item+, i.e. the items that
+    # will be marked as outdated when +item+ is outdated. Indirect
+    # dependencies will not be returned (e.g. if A depends on B which depends
+    # on C, then the direct inverse dependencies of C do not include A).
+    def direct_inverse_dependencies_for(item)
+      inverted_graph[item] || []
+    end
+
+    # Returns all inverse dependencies (direct and indirect) for +item+, i.e.
+    # the items that will be marked as outdated when +item+ is outdated.
+    def all_inverse_dependencies_for(item)
+      # Init list of all found dependencies
+      all_dependencies = []
+
+      # Init lists with already checked and not yet checked dependencies
+      checked_direct_dependencies = []
+      pending_direct_dependencies = direct_inverse_dependencies_for(item)
+
+      while !pending_direct_dependencies.empty?
+        # Get next unchecked dependency
+        dependency = pending_direct_dependencies.shift
+        next if checked_direct_dependencies.include?(dependency)
+
+        # Add dependencies of this unchecked dependency
+        pending_direct_dependencies += direct_inverse_dependencies_for(dependency)
+
+        # Mark this dependency as handled
+        all_dependencies << dependency
+        checked_direct_dependencies << dependency
+      end
+
+      all_dependencies
+    end
+
+    # Records a dependency from +src+ to +dst+ in the dependency graph. When
+    # +dst+ is oudated, +src+ will also become outdated.
+    def record_dependency(src, dst)
+      # Initialize graph if necessary
+      @graph[src] ||= []
+
+      # Don't include self or doubles in dependencies
+      return if src == dst
+      return if @graph[src].include?(dst)
+
+      # Record dependency
+      invalidate_inverted_graph
+      @graph[src] << dst
+    end
+
+    # Stores the dependency graph into the file specified by the +filename+
+    # attribute.
+    def store_graph
+      # Create dir
+      FileUtils.mkdir_p(File.dirname(self.filename))
+
+      # Complete the graph
+      complete_graph
+
+      # Convert graph of items into graph of item identifiers
+      new_graph = {}
+      @graph.each_pair do |second_item, first_items|
+        # Don't store nil because that would be pointless (if first_item is
+        # outdated, something that does not exist is also outdated… makes no
+        # sense).
+        # FIXME can second_item really be nil?
+        next if second_item.nil?
+
+        new_graph[second_item.identifier] = first_items.map { |f| f && f.identifier }.compact
+      end
+
+      # Store dependencies
+      store = PStore.new(self.filename)
+      store.transaction do
+        store[:dependencies] = new_graph
+      end
+    end
+
+    # Loads the dependency graph from the file specified by the +filename+
+    # attribute. This method will overwrite an existing dependency graph.
+    def load_graph
+      # Create new graph
+      @graph = {}
+
+      # Don't do anything if dependencies haven't been stored yet
+      return if !File.file?(self.filename)
+
+      # Load dependencies
+      store = PStore.new(self.filename)
+      store.transaction do
+        # Convert graph of identifiers into graph of items
+        store[:dependencies].each_pair do |second_item_identifier, first_item_identifiers|
+          # Convert second and first item identifiers into items
+          second_item = item_with_identifier(second_item_identifier)
+          first_items = first_item_identifiers.map { |p| item_with_identifier(p) }
+
+          @graph[second_item] = first_items
+        end
+      end
+    end
+
+    # Traverses the dependency graph and marks all items that (directly or
+    # indirectly) depend on an outdated item as outdated.
+    def mark_outdated_items
+      # Unmark everything
+      @items.each { |i| i.dependencies_outdated = false }
+
+      # Mark items that appear in @items but not in the dependency graph
+      added_items = @items - @graph.keys
+      added_items.each { |i| i.dependencies_outdated = true }
+
+      # Walk graph and mark items as outdated if necessary
+      # (#keys and #sort is used instead of #each_pair to add determinism)
+      first_items = inverted_graph.keys.sort_by { |i| i.nil? ? '/' : i.identifier }
+      something_changed = true
+      while something_changed
+        something_changed = false
+
+        first_items.each do |first_item|
+          second_items = inverted_graph[first_item]
+
+          if first_item.nil? ||                # item was removed
+             first_item.outdated? ||           # item itself is outdated
+             first_item.dependencies_outdated? # item is outdated because of its dependencies
+            second_items.each do |item|
+              # Ignore this item
+              next if item.nil?
+
+              something_changed = true if !item.dependencies_outdated?
+              item.dependencies_outdated = true
+            end
+          end
+        end
+      end
+    end
+
+    # Empties the list of dependencies for the given item. This is necessary
+    # before recompiling the given item, because otherwise old dependencies
+    # will stick around and new dependencies will appear twice.
+    def forget_dependencies_for(item)
+      @graph[item] = []
+    end
+
+  private
+
+    # Returns the item with the given identifier, or nil if no item is found.
+    def item_with_identifier(identifier)
+      @items.find { |i| i.identifier == identifier }
+    end
+
+    # Returns the inverted dependency graph, creating it first if it does not
+    # exist yet or is outdated. In this graph, the keys will be outdated when
+    # any of the values are outdated.
+    def inverted_graph
+      @inverted_graph ||= invert_graph(@graph)
+    end
+
+    # Marks the inverted graph as outdated so that it will be regenerated the
+    # next time it is used.
+    def invalidate_inverted_graph
+      @inverted_graph = nil
+    end
+
+    # Inverts the given graph (keys become values and values become keys).
+    #
+    # For example, this graph
+    #
+    #   {
+    #     :a => [ :b, :c ],
+    #     :b => [ :x, :c ]
+    #   }
+    #
+    # is turned into
+    #
+    #   {
+    #     :b => [ :a ],
+    #     :c => [ :a, :b ],
+    #     :x => [ :b ]
+    #   }
+    def invert_graph(graph)
+      inverted_graph = {}
+
+      graph.each_pair do |key, values|
+        values.each do |v|
+          inverted_graph[v] ||= []
+          inverted_graph[v] << key
+        end
+      end
+
+      inverted_graph
+    end
+
+    # Ensures that all items in the dependency graph have a list of
+    # dependecies, even if it is empty. Items without a list of dependencies
+    # will be treated as "added" and will depend on all other pages, which is
+    # not necessary for non-added items.
+    def complete_graph
+      @items.each do |item|
+        @graph[item] ||= []
+      end
+
+    end
+
+  end
+
+end