sitemap_generator_ftbpro 5.0.4

data/lib/sitemap_generator/sitemap_location.rb

@@ -0,0 +1,192 @@
+require 'sitemap_generator/helpers/number_helper'
+
+module SitemapGenerator
+  # A class for determining the exact location at which to write sitemap data.
+  # Handles reserving filenames from namers, constructing paths and sending
+  # data to the adapter to be written out.
+  class SitemapLocation < Hash
+    include SitemapGenerator::Helpers::NumberHelper
+
+    PATH_OUTPUT_WIDTH = 47 # Character width of the path in the summary lines
+
+    [:host, :adapter].each do |method|
+      define_method(method) do
+        raise SitemapGenerator::SitemapError, "No value set for #{method}" unless self[method]
+        self[method]
+      end
+    end
+
+    [:public_path, :sitemaps_path].each do |method|
+      define_method(method) do
+        Pathname.new(SitemapGenerator::Utilities.append_slash(self[method]))
+      end
+    end
+
+    # If no +filename+ or +namer+ is provided, the default namer is used, which
+    # generates names like <tt>sitemap.xml.gz</tt>, <tt>sitemap1.xml.gz</tt>, <tt>sitemap2.xml.gz</tt> and so on.
+    #
+    # === Options
+    # * <tt>:adapter</tt> - SitemapGenerator::Adapter subclass
+    # * <tt>:filename</tt> - full name of the file e.g. <tt>'sitemap1.xml.gz'<tt>
+    # * <tt>:host</tt> - host name for URLs.  The full URL to the file is then constructed from
+    #   the <tt>host</tt>, <tt>sitemaps_path</tt> and <tt>filename</tt>
+    # * <tt>:namer</tt> - a SitemapGenerator::SimpleNamer instance for generating file names.
+    #   Should be passed if no +filename+ is provided.
+    # * <tt>:public_path</tt> - path to the "public" directory, or the directory you want to
+    #   write sitemaps in.  Default is a directory <tt>public/</tt>
+    #   in the current working directory, or relative to the Rails root
+    #   directory if running under Rails.
+    # * <tt>:sitemaps_path</tt> - gives the path relative to the <tt>public_path</tt> in which to
+    #   write sitemaps e.g. <tt>sitemaps/</tt>.
+    # * <tt>:verbose</tt> - whether to output summary into to STDOUT.  Default +false+.
+    # * <tt>:create_index</tt> - whether to create a sitemap index.  Default `:auto`.  See <tt>LinkSet::create_index=</tt>
+    #   for possible values. Only applies to the SitemapIndexLocation object.
+    # * <tt>compress</tt> - The LinkSet compress setting.  Default: true.  If `false` any `.gz` extension is
+    #   stripped from the filename.  If `:all_but_first`, only the `.gz` extension of the first
+    #   filename is stripped off.  If `true` the extensions are left unchanged.
+    def initialize(opts={})
+      SitemapGenerator::Utilities.assert_valid_keys(opts, [:adapter, :public_path, :sitemaps_path, :host, :filename, :namer, :verbose, :create_index, :compress])
+      opts[:adapter] ||= SitemapGenerator::FileAdapter.new
+      opts[:public_path] ||= SitemapGenerator.app.root + 'public/'
+      # This is a bit of a hack to make the SimpleNamer act like the old SitemapNamer.
+      # It doesn't really make sense to create a default namer like this because the
+      # namer instance should be shared by the location objects of the sitemaps and
+      # sitemap index files.  However, this greatly eases testing, so I'm leaving it in
+      # for now.
+      if !opts[:filename] && !opts[:namer]
+        opts[:namer] = SitemapGenerator::SimpleNamer.new(:sitemap, :start => 2, :zero => 1)
+      end
+      opts[:verbose] = !!opts[:verbose]
+      self.merge!(opts)
+    end
+
+    # Return a new Location instance with the given options merged in
+    def with(opts={})
+      self.merge(opts)
+    end
+
+    # Full path to the directory of the file.
+    def directory
+      (public_path + sitemaps_path).expand_path.to_s
+    end
+
+    # Full path of the file including the filename.
+    def path
+      (public_path + sitemaps_path + filename).expand_path.to_s
+    end
+
+    # Relative path of the file (including the filename) relative to <tt>public_path</tt>
+    def path_in_public
+      (sitemaps_path + filename).to_s
+    end
+
+    # Full URL of the file.
+    def url
+      URI.join(host, sitemaps_path.to_s, filename.to_s).to_s
+    end
+
+    # Return the size of the file at <tt>path</tt>
+    def filesize
+      File.size?(path)
+    end
+
+    # Return the filename.  Raises an exception if no filename or namer is set.
+    # If using a namer once the filename has been retrieved from the namer its
+    # value is locked so that it is unaffected by further changes to the namer.
+    def filename
+      raise SitemapGenerator::SitemapError, "No filename or namer set" unless self[:filename] || self[:namer]
+      unless self[:filename]
+        self.send(:[]=, :filename, self[:namer].to_s, :super => true)
+
+        # Post-process the filename for our compression settings.
+        # Strip the `.gz` from the extension if we aren't compressing this file.
+        # If you're setting the filename manually, :all_but_first won't work as
+        # expected.  Ultimately I should force using a namer in all circumstances.
+        # Changing the filename here will affect how the FileAdapter writes out the file.
+        if self[:compress] == false ||
+           (self[:namer] && self[:namer].start? && self[:compress] == :all_but_first)
+          self[:filename].gsub!(/\.gz$/, '')
+        end
+      end
+      self[:filename]
+    end
+
+    # If a namer is set, reserve the filename and increment the namer.
+    # Returns the reserved name.
+    def reserve_name
+      if self[:namer]
+        filename
+        self[:namer].next
+      end
+      self[:filename]
+    end
+
+    # Return true if this location has a fixed filename.  If no name has been
+    # reserved from the namer, for instance, returns false.
+    def reserved_name?
+      !!self[:filename]
+    end
+
+    def namer
+      self[:namer]
+    end
+
+    def verbose?
+      self[:verbose]
+    end
+
+    # If you set the filename, clear the namer and vice versa.
+    def []=(key, value, opts={})
+      if !opts[:super]
+        case key
+        when :namer
+          super(:filename, nil)
+        when :filename
+          super(:namer, nil)
+        end
+      end
+      super(key, value)
+    end
+
+    # Write `data` out to a file.
+    # Output a summary line if verbose is true.
+    def write(data, link_count)
+      adapter.write(self, data)
+      puts summary(link_count) if verbose?
+    end
+
+    # Return a summary string
+    def summary(link_count)
+      filesize = number_to_human_size(self.filesize)
+      width = self.class::PATH_OUTPUT_WIDTH
+      path = SitemapGenerator::Utilities.ellipsis(self.path_in_public, width)
+      "+ #{('%-'+width.to_s+'s') % path} #{'%10s' % link_count} links / #{'%10s' % filesize}"
+    end
+  end
+
+  class SitemapIndexLocation < SitemapLocation
+    def initialize(opts={})
+      if !opts[:filename] && !opts[:namer]
+        opts[:namer] = SitemapGenerator::SimpleNamer.new(:sitemap)
+      end
+      super(opts)
+    end
+
+    # Whether to create a sitemap index.  Default `:auto`.  See <tt>LinkSet::create_index=</tt>
+    # for possible values.
+    #
+    # A placeholder for an option which should really go into some
+    # kind of options class.
+    def create_index
+      self[:create_index]
+    end
+
+    # Return a summary string
+    def summary(link_count)
+      filesize = number_to_human_size(self.filesize)
+      width = self.class::PATH_OUTPUT_WIDTH - 3
+      path = SitemapGenerator::Utilities.ellipsis(self.path_in_public, width)
+      "+ #{('%-'+width.to_s+'s') % path} #{'%10s' % link_count} sitemaps / #{'%10s' % filesize}"
+    end
+  end
+end

data/lib/sitemap_generator/sitemap_namer.rb

@@ -0,0 +1,75 @@
+module SitemapGenerator
+  # A class for generating sitemap filenames.
+  #
+  # The SimpleNamer uses the same namer instance for the sitemap index and the sitemaps.
+  # If no index is needed, the first sitemap gets the first name.  However, if
+  # an index is needed, the index gets the first name.
+  #
+  # A typical sequence would looks like this:
+  #   * sitemap.xml.gz
+  #   * sitemap1.xml.gz
+  #   * sitemap2.xml.gz
+  #   * sitemap3.xml.gz
+  #   * ...
+  #
+  # Arguments:
+  #   base - string or symbol that forms the base of the generated filename e.g.
+  #          if `:geo` files are generated like `geo.xml.gz`, `geo1.xml.gz`, `geo2.xml.gz` etc.
+  #
+  # Options:
+  #   :extension - Default: '.xml.gz'. File extension to append.
+  #   :start     - Default: 1. Numerical index at which to start counting.
+  #   :zero      - Default: nil.  A string or number that is appended to +base+
+  #                to create the first name in the sequence.  So setting this
+  #                to '_index' would produce 'sitemap_index.xml.gz' as
+  #                the first name.  Thereafter, the numerical index defined by +start+
+  #                is used, and subsequent names would be 'sitemap1.xml.gz', 'sitemap2.xml.gz', etc.
+  #                In these examples the `base` string is assumed to be 'sitemap'.
+  class SimpleNamer
+    def initialize(base, options={})
+      @options = SitemapGenerator::Utilities.reverse_merge(options,
+        :zero => nil,  # identifies the marker for the start of the series
+        :extension => '.xml.gz',
+        :start => 1
+      )
+      @base = base
+      reset
+    end
+
+    def to_s
+      extension = @options[:extension]
+      "#{@base}#{@count}#{extension}"
+    end
+
+    # Reset to the first name
+    def reset
+      @count = @options[:zero]
+    end
+
+    # True if on the first name
+    def start?
+      @count == @options[:zero]
+    end
+
+    # Return this instance set to the next name
+    def next
+      if start?
+        @count = @options[:start]
+      else
+        @count += 1
+      end
+      self
+    end
+
+    # Return this instance set to the previous name
+    def previous
+      raise NameError, "Already at the start of the series" if start?
+      if @count <= @options[:start]
+        @count = @options[:zero]
+      else
+        @count -= 1
+      end
+      self
+    end
+  end
+end

data/lib/sitemap_generator/tasks.rb

@@ -0,0 +1,53 @@
+# require this file to load the tasks
+require 'rake'
+
+# Require sitemap_generator at runtime.  If we don't do this the ActionView helpers are included
+# before the Rails environment can be loaded by other Rake tasks, which causes problems
+# for those tasks when rendering using ActionView.
+namespace :sitemap do
+  # Require sitemap_generator only.  When installed as a plugin the require will fail, so in
+  # that case, load the environment first.
+  task :require do
+    begin
+      require 'sitemap_generator'
+    rescue LoadError => e
+      if defined?(Rails)
+        Rake::Task['sitemap:require_environment'].invoke
+      else
+        raise e
+      end
+    end
+  end
+
+  # Require sitemap_generator after loading the Rails environment.  We still need the require
+  # in case we are installed as a gem and are setup to not automatically be required.
+  task :require_environment do
+    if defined?(Rails)
+      Rake::Task['environment'].invoke
+    end
+    require 'sitemap_generator'
+  end
+
+  desc "Install a default config/sitemap.rb file"
+  task :install => ['sitemap:require'] do
+    SitemapGenerator::Utilities.install_sitemap_rb(verbose)
+  end
+
+  desc "Delete all Sitemap files in public/ directory"
+  task :clean => ['sitemap:require'] do
+    SitemapGenerator::Utilities.clean_files
+  end
+
+  desc "Generate sitemaps and ping search engines."
+  task :refresh => ['sitemap:create'] do
+    SitemapGenerator::Sitemap.ping_search_engines
+  end
+
+  desc "Generate sitemaps but don't ping search engines."
+  task 'refresh:no_ping' => ['sitemap:create']
+
+  desc "Generate sitemaps but don't ping search engines.  Alias for refresh:no_ping."
+  task :create => ['sitemap:require_environment'] do
+    SitemapGenerator::Interpreter.run(:config_file => ENV["CONFIG_FILE"], :verbose => verbose)
+  end
+end

data/lib/sitemap_generator/templates.rb

@@ -0,0 +1,41 @@
+module SitemapGenerator
+  # Provide convenient access to template files.  E.g.
+  #
+  #   SitemapGenerator.templates.sitemap_index
+  #
+  # Lazy-load and cache for efficient access.
+  # Define an accessor method for each template file.
+  class Templates
+    FILES = {
+      :sitemap_sample =>  'sitemap.rb',
+    }
+
+    # Dynamically define accessors for each key defined in <tt>FILES</tt>
+    attr_accessor *FILES.keys
+    FILES.keys.each do |name|
+      eval <<-END
+        define_method(:#{name}) do
+          @#{name} ||= read_template(:#{name})
+        end
+      END
+    end
+
+    def initialize(root = SitemapGenerator.root)
+      @root = root
+    end
+
+    # Return the full path to a template.
+    #
+    # <tt>file</tt> template symbol e.g. <tt>:sitemap_sample</tt>
+    def template_path(template)
+      File.join(@root, 'templates', self.class::FILES[template])
+    end
+
+    protected
+
+    # Read the template file and return its contents.
+    def read_template(template)
+      File.read(template_path(template))
+    end
+  end
+end

data/lib/sitemap_generator/utilities.rb

@@ -0,0 +1,181 @@
+module SitemapGenerator
+  module Utilities
+    extend self
+
+    # Copy templates/sitemap.rb to config if not there yet.
+    def install_sitemap_rb(verbose=false)
+      if File.exist?(SitemapGenerator.app.root + 'config/sitemap.rb')
+        puts "already exists: config/sitemap.rb, file not copied" if verbose
+      else
+        FileUtils.cp(
+          SitemapGenerator.templates.template_path(:sitemap_sample),
+          SitemapGenerator.app.root + 'config/sitemap.rb')
+        puts "created: config/sitemap.rb" if verbose
+      end
+    end
+
+    # Remove config/sitemap.rb if exists.
+    def uninstall_sitemap_rb
+      if File.exist?(SitemapGenerator.app.root + 'config/sitemap.rb')
+        File.rm(SitemapGenerator.app.root + 'config/sitemap.rb')
+      end
+    end
+
+    # Clean sitemap files in output directory.
+    def clean_files
+      FileUtils.rm(Dir[SitemapGenerator.app.root + 'public/sitemap*.xml.gz'])
+    end
+
+    # Validate all keys in a hash match *valid keys, raising ArgumentError on a
+    # mismatch. Note that keys are NOT treated indifferently, meaning if you use
+    # strings for keys but assert symbols as keys, this will fail.
+    def assert_valid_keys(hash, *valid_keys)
+      unknown_keys = hash.keys - [valid_keys].flatten
+      raise(ArgumentError, "Unknown key(s): #{unknown_keys.join(", ")}") unless unknown_keys.empty?
+    end
+
+    # Return a new hash with all keys converted to symbols, as long as
+    # they respond to +to_sym+.
+    def symbolize_keys(hash)
+      symbolize_keys!(hash.dup)
+    end
+
+    # Destructively convert all keys to symbols, as long as they respond
+    # to +to_sym+.
+    def symbolize_keys!(hash)
+      hash.keys.each do |key|
+        hash[(key.to_sym rescue key) || key] = hash.delete(key)
+      end
+      hash
+    end
+
+    # Make a list of `value` if it is not a list already.  If `value` is
+    # nil, an empty list is returned.  If `value` is already a list, return it unchanged.
+    def as_array(value)
+      if value.nil?
+        []
+      elsif value.is_a?(Array)
+        value
+      else
+        [value]
+      end
+    end
+
+    # Rounds the float with the specified precision.
+    #
+    #   x = 1.337
+    #   x.round    # => 1
+    #   x.round(1) # => 1.3
+    #   x.round(2) # => 1.34
+    def round(float, precision = nil)
+      if precision
+        magnitude = 10.0 ** precision
+        (float * magnitude).round / magnitude
+      else
+        float.round
+      end
+    end
+
+    # Allows for reverse merging two hashes where the keys in the calling hash take precedence over those
+    # in the <tt>other_hash</tt>. This is particularly useful for initializing an option hash with default values:
+    #
+    #   def setup(options = {})
+    #     options.reverse_merge! :size => 25, :velocity => 10
+    #   end
+    #
+    # Using <tt>merge</tt>, the above example would look as follows:
+    #
+    #   def setup(options = {})
+    #     { :size => 25, :velocity => 10 }.merge(options)
+    #   end
+    #
+    # The default <tt>:size</tt> and <tt>:velocity</tt> are only set if the +options+ hash passed in doesn't already
+    # have the respective key.
+    def reverse_merge(hash, other_hash)
+      other_hash.merge(hash)
+    end
+
+    # Performs the opposite of <tt>merge</tt>, with the keys and values from the first hash taking precedence over the second.
+    # Modifies the receiver in place.
+    def reverse_merge!(hash, other_hash)
+      hash.merge!( other_hash ){|k,o,n| o }
+    end
+
+    # An object is blank if it's false, empty, or a whitespace string.
+    # For example, "", "   ", +nil+, [], and {} are blank.
+    #
+    # This simplifies:
+    #
+    #   if !address.nil? && !address.empty?
+    #
+    # ...to:
+    #
+    #   if !address.blank?
+    def blank?(object)
+      case object
+      when NilClass, FalseClass
+        true
+      when TrueClass, Numeric
+        false
+      when String
+        object !~ /\S/
+      when Hash, Array
+        object.empty?
+      when Object
+        object.respond_to?(:empty?) ? object.empty? : !object
+      end
+    end
+
+    # An object is present if it's not blank.
+    def present?(object)
+      !blank?(object)
+    end
+
+    # Sets $VERBOSE for the duration of the block and back to its original value afterwards.
+    def with_warnings(flag)
+      old_verbose, $VERBOSE = $VERBOSE, flag
+      yield
+    ensure
+      $VERBOSE = old_verbose
+    end
+
+    def titleize(string)
+      string.gsub!(/_/, ' ')
+      string.split(/(\W)/).map(&:capitalize).join
+    end
+
+    def truthy?(value)
+      ['1', 1, 't', 'true', true].include?(value)
+    end
+
+    def falsy?(value)
+      ['0', 0, 'f', 'false', false].include?(value)
+    end
+
+    # Append a slash to `path` if it does not already end in a slash.
+    # Returns a string.  Expects a string or Pathname object.
+    def append_slash(path)
+      strpath = path.to_s
+      if strpath[-1] != nil && strpath[-1].chr != '/'
+        strpath + '/'
+      else
+        strpath
+      end
+    end
+
+    # Replace the last 3 characters of string with ... if the string is as big
+    # or bigger than max.
+    def ellipsis(string, max)
+      if string.size > max
+        (string[0, max - 3] || '') + '...'
+      else
+        string
+      end
+    end
+
+    # Return the bytesize length of the string.  Ruby 1.8.6 compatible.
+    def bytesize(string)
+      string.respond_to?(:bytesize) ? string.bytesize : string.length
+    end
+  end
+end

data/lib/sitemap_generator.rb

@@ -0,0 +1,82 @@
+require 'sitemap_generator/sitemap_namer'
+require 'sitemap_generator/builder'
+require 'sitemap_generator/link_set'
+require 'sitemap_generator/templates'
+require 'sitemap_generator/utilities'
+require 'sitemap_generator/application'
+require 'sitemap_generator/adapters'
+require 'sitemap_generator/sitemap_location'
+
+module SitemapGenerator
+  autoload(:Interpreter, 'sitemap_generator/interpreter')
+  autoload(:FileAdapter, 'sitemap_generator/adapters/file_adapter')
+  autoload(:S3Adapter,   'sitemap_generator/adapters/s3_adapter')
+  autoload(:WaveAdapter, 'sitemap_generator/adapters/wave_adapter')
+  autoload(:FogAdapter,  'sitemap_generator/adapters/fog_adapter')
+  autoload(:BigDecimal,  'sitemap_generator/core_ext/big_decimal')
+  autoload(:Numeric,     'sitemap_generator/core_ext/numeric')
+
+  SitemapError          = Class.new(StandardError)
+  SitemapFullError      = Class.new(SitemapError)
+  SitemapFinalizedError = Class.new(SitemapError)
+
+  Utilities.with_warnings(nil) do
+    VERSION = File.read(File.dirname(__FILE__) + "/../VERSION").strip
+    MAX_SITEMAP_FILES    = 50_000        # max sitemap links per index file
+    MAX_SITEMAP_LINKS    = 50_000        # max links per sitemap
+    MAX_SITEMAP_IMAGES   = 1_000         # max images per url
+    MAX_SITEMAP_NEWS     = 1_000         # max news sitemap per index_file
+    MAX_SITEMAP_FILESIZE = 10_000_000    # bytes
+    SCHEMAS = {
+      'geo'     => 'http://www.google.com/geo/schemas/sitemap/1.0',
+      'image'   => 'http://www.google.com/schemas/sitemap-image/1.1',
+      'mobile'  => 'http://www.google.com/schemas/sitemap-mobile/1.0',
+      'news'    => 'http://www.google.com/schemas/sitemap-news/0.9',
+      'pagemap' => 'http://www.google.com/schemas/sitemap-pagemap/1.0',
+      'video'   => 'http://www.google.com/schemas/sitemap-video/1.1'
+    }
+
+    # Lazy-initialize the LinkSet instance
+    Sitemap = (Class.new do
+      def method_missing(*args, &block)
+        (@link_set ||= reset!).send(*args, &block)
+      end
+
+      # Use a new LinkSet instance
+      def reset!
+        @link_set = LinkSet.new
+      end
+    end).new
+  end
+
+  class << self
+    attr_accessor :root, :app, :templates
+    attr_writer :yield_sitemap, :verbose
+  end
+
+  # Global default for the verbose setting.
+  def self.verbose
+    if @verbose.nil?
+      @verbose = if SitemapGenerator::Utilities.truthy?(ENV['VERBOSE'])
+        true
+      elsif SitemapGenerator::Utilities.falsy?(ENV['VERBOSE'])
+        false
+      else
+        nil
+      end
+    else
+      @verbose
+    end
+  end
+
+  # Returns true if we should yield the sitemap instance to the block, false otherwise.
+  def self.yield_sitemap?
+    !!@yield_sitemap
+  end
+
+  self.root      = File.expand_path(File.join(File.dirname(__FILE__), '../'))  # Root of the install dir, not the Rails app
+  self.templates = SitemapGenerator::Templates.new(self.root)
+  self.app       = SitemapGenerator::Application.new
+end
+
+require 'sitemap_generator/railtie' if SitemapGenerator.app.rails3?

data/lib/tasks/sitemap_generator_tasks.rake

@@ -0,0 +1 @@
+load(File.expand_path(File.join(File.dirname(__FILE__), '../sitemap_generator/tasks.rb')))

data/rails/install.rb

@@ -0,0 +1,2 @@
+# Install hook code here
+SitemapGenerator::Utilities.install_sitemap_rb

data/rails/uninstall.rb

@@ -0,0 +1,2 @@
+# Uninstall hook code here
+SitemapGenerator::Utilities.uninstall_sitemap_rb

data/spec/blueprint.rb

@@ -0,0 +1,15 @@
+require 'machinist/active_record'
+require 'sham'
+
+Sham.title { Time.now.to_i }
+Content.blueprint do
+  title
+end
+
+module Blueprint
+  def self.seed
+    14.times do |i|
+      content = Content.make(:title => "Link #{i}")
+    end
+  end
+end

data/spec/files/sitemap.create.rb

@@ -0,0 +1,12 @@
+SitemapGenerator::Sitemap.default_host = "http://www.example.com"
+
+SitemapGenerator::Sitemap.create do
+  add '/contents', :priority => 0.7, :changefreq => 'daily'
+
+  # add all individual articles
+  (1..10).each do |i|
+    add "/content/#{i}"
+  end
+
+  add "/merchant_path", :host => "https://www.example.com"
+end