sitemap_generator_ftbpro 5.0.4

data/Rakefile

@@ -0,0 +1,43 @@
+require 'bundler/setup'
+Bundler.require
+
+desc 'Default: run spec tests.'
+task :default => :spec
+
+require "rspec/core/rake_task"
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = Dir.glob(['spec/sitemap_generator/**/*'])
+  spec.rspec_opts = ['--backtrace']
+end
+
+#
+# Helpers
+#
+
+def name;         @name ||= Dir['*.gemspec'].first.split('.').first end
+def version;      File.read('VERSION').chomp end
+def gemspec_file; "#{name}.gemspec" end
+def gem_file;     "#{name}-#{version}.gem" end
+
+#
+# Release Tasks
+# @see https://github.com/mojombo/rakegem
+#
+
+desc "Create tag v#{version}, build the gem and push to Git"
+task :release => :build do
+  unless `git branch` =~ /^\* master$/
+    puts "You must be on the master branch to release!"
+    exit!
+  end
+  sh "git tag v#{version}"
+  sh "git push origin master --tags"
+end
+
+desc "Build #{gem_file} into the pkg/ directory"
+task :build do
+  sh "mkdir -p pkg"
+  sh "gem build #{gemspec_file}"
+  sh "mv #{gem_file} pkg"
+  sh "bundle --local"
+end

data/VERSION

@@ -0,0 +1 @@
+5.0.4

data/lib/capistrano/sitemap_generator.rb

@@ -0,0 +1 @@
+load File.expand_path(File.join('..', 'tasks', 'sitemap_generator.cap'), __FILE__)

data/lib/capistrano/tasks/sitemap_generator.cap

@@ -0,0 +1,36 @@
+namespace :deploy do
+  namespace :sitemap do
+    desc 'Create sitemap and ping search engines'
+    task :refresh do
+      on roles :web do
+        within release_path do
+          with rails_env: fetch(:rails_env) do
+           execute :rake, "sitemap:refresh"
+          end
+        end
+      end
+    end
+
+    desc 'Create sitemap without pinging search engines'
+    task :create do
+      on roles :web do
+        within release_path do
+          with rails_env: fetch(:rails_env) do
+           execute :rake, "sitemap:create"
+          end
+        end
+      end
+    end
+
+    desc 'Clean up sitemaps in sitemap_generator path'
+    task :clean do
+      on roles :web do
+        within release_path do
+          with rails_env: fetch(:rails_env) do
+           execute :rake, "sitemap:clean"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sitemap_generator/adapters/file_adapter.rb

@@ -0,0 +1,43 @@
+module SitemapGenerator
+  # Class for writing out data to a file.
+  class FileAdapter
+
+    # Write data to a file.
+    # @param location - File object giving the full path and file name of the file.
+    #    If the location specifies a directory(ies) which does not exist, the directory(ies)
+    #    will be created for you.  If the location path ends with `.gz` the data will be
+    #    compressed prior to being written out.  Otherwise the data will be written out
+    #    unchanged.
+    # @param raw_data - data to write to the file.
+    def write(location, raw_data)
+      # Ensure that the directory exists
+      dir = location.directory
+      if !File.exists?(dir)
+        FileUtils.mkdir_p(dir)
+      elsif !File.directory?(dir)
+        raise SitemapError.new("#{dir} should be a directory!")
+      end
+
+      stream = open(location.path, 'wb')
+      if location.path.to_s =~ /.gz$/
+        gzip(stream, raw_data)
+      else
+        plain(stream, raw_data)
+      end
+    end
+
+    # Write `data` to a stream, passing the data through a GzipWriter
+    # to compress it.
+    def gzip(stream, data)
+      gz = Zlib::GzipWriter.new(stream)
+      gz.write data
+      gz.close
+    end
+
+    # Write `data` to a stream as is.
+    def plain(stream, data)
+      stream.write data
+      stream.close
+    end
+  end
+end

data/lib/sitemap_generator/adapters/fog_adapter.rb

@@ -0,0 +1,28 @@
+begin
+  require 'fog'
+rescue LoadError
+  raise LoadError.new("Missing required 'fog'.  Please 'gem install fog' and require it in your application.")
+end
+
+module SitemapGenerator
+  class FogAdapter
+
+    def initialize(opts = {})
+      @fog_credentials = opts[:fog_credentials]
+      @fog_directory = opts[:fog_directory]
+    end
+
+    # Call with a SitemapLocation and string data
+    def write(location, raw_data)
+      SitemapGenerator::FileAdapter.new.write(location, raw_data)
+
+      storage   = Fog::Storage.new(@fog_credentials)
+      directory = storage.directories.new(:key => @fog_directory)
+      directory.files.create(
+        :key    => location.path_in_public,
+        :body   => File.open(location.path),
+        :public => true
+      )
+    end
+  end
+end

data/lib/sitemap_generator/adapters/s3_adapter.rb

@@ -0,0 +1,41 @@
+begin
+  require 'fog'
+rescue LoadError
+  raise LoadError.new("Missing required 'fog'.  Please 'gem install fog' and require it in your application.")
+end
+
+module SitemapGenerator
+  class S3Adapter
+
+    def initialize(opts = {})
+      @aws_access_key_id = opts[:aws_access_key_id] || ENV['AWS_ACCESS_KEY_ID']
+      @aws_secret_access_key = opts[:aws_secret_access_key] || ENV['AWS_SECRET_ACCESS_KEY']
+      @fog_provider = opts[:fog_provider] || ENV['FOG_PROVIDER']
+      @fog_directory = opts[:fog_directory] || ENV['FOG_DIRECTORY']
+      @fog_region = opts[:fog_region] || ENV['FOG_REGION']
+      @fog_path_style = opts[:fog_path_style] || ENV['FOG_PATH_STYLE']
+    end
+
+    # Call with a SitemapLocation and string data
+    def write(location, raw_data)
+      SitemapGenerator::FileAdapter.new.write(location, raw_data)
+
+      credentials = {
+        :aws_access_key_id     => @aws_access_key_id,
+        :aws_secret_access_key => @aws_secret_access_key,
+        :provider              => @fog_provider,
+      }
+      credentials[:region] = @fog_region if @fog_region
+      credentials[:path_style] = @fog_path_style if @fog_path_style
+
+      storage   = Fog::Storage.new(credentials)
+      directory = storage.directories.new(:key => @fog_directory)
+      directory.files.create(
+        :key    => location.path_in_public,
+        :body   => File.open(location.path),
+        :public => true
+      )
+    end
+
+  end
+end

data/lib/sitemap_generator/adapters/wave_adapter.rb

@@ -0,0 +1,21 @@
+begin
+  require 'carrierwave'
+rescue LoadError
+  raise LoadError.new("Missing required 'carrierwave'.  Please 'gem install carrierwave' and require it in your application.")
+end
+
+module SitemapGenerator
+  class WaveAdapter < ::CarrierWave::Uploader::Base
+    attr_accessor :store_dir
+
+    # Call with a SitemapLocation and string data
+    def write(location, raw_data)
+      SitemapGenerator::FileAdapter.new.write(location, raw_data)
+      directory = File.dirname(location.path_in_public)
+      if directory != '.'
+        self.store_dir = directory
+      end
+      store!(open(location.path, 'rb'))
+    end
+  end
+end

data/lib/sitemap_generator/application.rb

@@ -0,0 +1,49 @@
+require 'pathname'
+
+module SitemapGenerator
+  class Application
+    def rails?
+      defined?(Rails)
+    end
+
+    # Returns a boolean indicating whether this environment is Rails 3
+    #
+    # @return [Boolean]
+    def rails3?
+      rails? && Rails.version.to_f >= 3
+    rescue
+      false # Rails.version defined in 2.1.0
+    end
+
+    def root
+      Pathname.new(rails_root || Dir.getwd)
+    end
+
+    protected
+
+    # Returns the root of the Rails application,
+    # if this is running in a Rails context.
+    # Returns `nil` if no such root is defined.
+    #
+    # @return [String, nil]
+    def rails_root
+      if defined?(::Rails.root)
+        return ::Rails.root.to_s if ::Rails.root
+        raise "ERROR: Rails.root is nil!"
+      end
+      return RAILS_ROOT.to_s if defined?(RAILS_ROOT)
+      return nil
+    end
+
+    # Returns the environment of the Rails application,
+    # if this is running in a Rails context.
+    # Returns `nil` if no such environment is defined.
+    #
+    # @return [String, nil]
+    def rails_env
+      return ::Rails.env.to_s if defined?(::Rails.env)
+      return RAILS_ENV.to_s if defined?(RAILS_ENV)
+      return nil
+    end
+  end
+end

data/lib/sitemap_generator/builder/sitemap_file.rb

@@ -0,0 +1,171 @@
+require 'zlib'
+require 'fileutils'
+require 'sitemap_generator/helpers/number_helper'
+
+module SitemapGenerator
+  module Builder
+    #
+    # General Usage:
+    #
+    #   sitemap = SitemapFile.new(:location => SitemapLocation.new(...))
+    #   sitemap.add('/', { ... })    <- add a link to the sitemap
+    #   sitemap.finalize!            <- write the sitemap file and freeze the object to protect it from further modification
+    #
+    class SitemapFile
+      include SitemapGenerator::Helpers::NumberHelper
+      attr_reader :link_count, :filesize, :location, :news_count
+
+      # === Options
+      #
+      # * <tt>location</tt> - a SitemapGenerator::SitemapLocation instance or a Hash of options
+      #   from which a SitemapLocation will be created for you.
+      def initialize(opts={}, schemas)
+        @location = opts.is_a?(Hash) ? SitemapGenerator::SitemapLocation.new(opts) : opts
+        @link_count = 0
+        @news_count = 0
+        @xml_content = '' # XML urlset content
+        @schemas = schemas
+        @xml_wrapper_start = <<-HTML
+          "#{all_schemas}"
+        HTML
+        @xml_wrapper_start.gsub!(/\s+/, ' ').gsub!(/ *> */, '>').strip!
+        @xml_wrapper_start = @xml_wrapper_start.slice(1..-2)
+        @xml_wrapper_end   = %q[</urlset>]
+        @filesize = SitemapGenerator::Utilities.bytesize(@xml_wrapper_start) + SitemapGenerator::Utilities.bytesize(@xml_wrapper_end)
+        @written = false
+        @reserved_name = nil # holds the name reserved from the namer
+        @frozen = false      # rather than actually freeze, use this boolean
+      end
+
+      def all_schemas
+        xml_start = '<?xml version="1.0" encoding="UTF-8"?>
+            <urlset
+              xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+              xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9
+                http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"
+              xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" '
+        xml_schemas = (@schemas.collect do |schema, content|  "xmlns:#{schema}=\"#{content}\"\n" end).join(" ")
+        xml_end = 'xmlns:xhtml="http://www.w3.org/1999/xhtml" >'
+        return xml_start + xml_schemas + xml_end
+      end
+
+      # If a name has been reserved, use the last modified time from the file.
+      # Otherwise return nil.  We don't want to prematurely assign a name
+      # for this sitemap if one has not yet been reserved, because we may
+      # mess up the name-assignment sequence.
+      def lastmod
+        File.mtime(location.path) if location.reserved_name?
+      rescue
+        nil
+      end
+
+      def empty?
+        @link_count == 0
+      end
+
+      # Return a boolean indicating whether the sitemap file can fit another link
+      # of <tt>bytes</tt> bytes in size.  You can also pass a string and the
+      # bytesize will be calculated for you.
+      def file_can_fit?(bytes)
+        bytes = bytes.is_a?(String) ? SitemapGenerator::Utilities.bytesize(bytes) : bytes
+        (@filesize + bytes) < SitemapGenerator::MAX_SITEMAP_FILESIZE && @link_count < SitemapGenerator::MAX_SITEMAP_LINKS && @news_count < SitemapGenerator::MAX_SITEMAP_NEWS
+      end
+
+      # Add a link to the sitemap file.
+      #
+      # If a link cannot be added, for example if the file is too large or the link
+      # limit has been reached, a SitemapGenerator::SitemapFullError exception is raised
+      # and the sitemap is finalized.
+      #
+      # If the Sitemap has already been finalized a SitemapGenerator::SitemapFinalizedError
+      # exception is raised.
+      #
+      # Return the new link count.
+      #
+      # Call with:
+      #   sitemap_url - a SitemapUrl instance
+      #   sitemap, options - a Sitemap instance and options hash
+      #   path, options - a path for the URL and options hash.  For supported options
+      #                   see the SitemapGenerator::Builder::SitemapUrl class.
+      #
+      # The link added to the sitemap will use the host from its location object
+      # if no host has been specified.
+      def add(link, options={})
+        raise SitemapGenerator::SitemapFinalizedError if finalized?
+
+        sitemap_url = if link.is_a?(SitemapUrl)
+          link
+        else
+          options[:host] ||= @location.host
+          SitemapUrl.new(link, options)
+        end
+
+        xml = sitemap_url.to_xml
+        raise SitemapGenerator::SitemapFullError if !file_can_fit?(xml)
+
+        if sitemap_url.news?
+          @news_count += 1
+        end
+
+        # Add the XML to the sitemap
+        @xml_content << xml
+        @filesize += SitemapGenerator::Utilities.bytesize(xml)
+        @link_count += 1
+      end
+
+      # "Freeze" this object.  Actually just flags it as frozen.
+      #
+      # A SitemapGenerator::SitemapFinalizedError exception is raised if the Sitemap
+      # has already been finalized.
+      def finalize!
+        raise SitemapGenerator::SitemapFinalizedError if finalized?
+        @frozen = true
+      end
+
+      def finalized?
+        @frozen
+      end
+
+      # Write out the sitemap and free up memory.
+      #
+      # All the xml content in the instance is cleared, but attributes like
+      # <tt>filesize</tt> are still available.
+      #
+      # A SitemapGenerator::SitemapError exception is raised if the file has
+      # already been written.
+      def write
+        raise SitemapGenerator::SitemapError.new("Sitemap already written!") if written?
+        finalize! unless finalized?
+        reserve_name
+        @location.write(@xml_wrapper_start + @xml_content + @xml_wrapper_end, link_count)
+        @xml_content = @xml_wrapper_start = @xml_wrapper_end = ''
+        @written = true
+      end
+
+      # Return true if this file has been written out to disk
+      def written?
+        @written
+      end
+
+      # Reserve a name from the namer unless one has already been reserved.
+      # Safe to call more than once.
+      def reserve_name
+        @reserved_name ||= @location.reserve_name
+      end
+
+      # Return a boolean indicating whether a name has been reserved
+      def reserved_name?
+        !!@reserved_name
+      end
+
+      # Return a new instance of the sitemap file with the same options,
+      # and the next name in the sequence.
+      def new
+        location = @location.dup
+        location.delete(:filename) if location.namer
+        self.class.new(location)
+      end
+    end
+  end
+end
+

data/lib/sitemap_generator/builder/sitemap_index_file.rb

@@ -0,0 +1,149 @@
+module SitemapGenerator
+  module Builder
+    class SitemapIndexFile < SitemapFile
+
+      # === Options
+      #
+      # * <tt>location</tt> - a SitemapGenerator::SitemapIndexLocation instance or a Hash of options
+      #   from which a SitemapLocation will be created for you.
+      def initialize(opts={})
+        @location = opts.is_a?(Hash) ? SitemapGenerator::SitemapIndexLocation.new(opts) : opts
+        @link_count = 0
+        @sitemaps_link_count = 0
+        @xml_content = '' # XML urlset content
+        @xml_wrapper_start = <<-HTML
+          <?xml version="1.0" encoding="UTF-8"?>
+            <sitemapindex
+              xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+              xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9
+                http://www.sitemaps.org/schemas/sitemap/0.9/siteindex.xsd"
+              xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+            >
+        HTML
+        @xml_wrapper_start.gsub!(/\s+/, ' ').gsub!(/ *> */, '>').strip!
+        @xml_wrapper_end   = %q[</sitemapindex>]
+        @filesize = SitemapGenerator::Utilities.bytesize(@xml_wrapper_start) + SitemapGenerator::Utilities.bytesize(@xml_wrapper_end)
+        @written = false
+        @reserved_name = nil # holds the name reserved from the namer
+        @frozen = false      # rather than actually freeze, use this boolean
+        @first_sitemap = nil # reference to the first thing added to this index
+        # Store the URL of the first sitemap added because if create_index is
+        # false this is the "index" URL
+        @first_sitemap_url = nil
+      end
+
+      # Finalize sitemaps as they are added to the index.
+      # If it's the first sitemap, finalize it but don't
+      # write it out, because we don't yet know if we need an index.  If it's
+      # the second sitemap, we know we need an index, so reserve a name for the
+      # index, and go and write out the first sitemap.  If it's the third or
+      # greater sitemap, just finalize and write it out as usual, nothing more
+      # needs to be done.
+      #
+      # If a link is being added to the index manually as a string, then we
+      # can assume that the index is required (unless create_index is false of course).
+      # This seems like the logical thing to do.
+      alias_method :super_add, :add
+      def add(link, options={})
+        if file = link.is_a?(SitemapFile) && link
+          @sitemaps_link_count += file.link_count
+          file.finalize! unless file.finalized?
+
+          # First link.  If it's a SitemapFile store a reference to it and the options
+          # so that we can create a URL from it later.  We can't create the URL yet
+          # because doing so fixes the sitemap file's name, and we have to wait to see
+          # if we have more than one link in the index before we can know who gets the
+          # first name (the index, or the sitemap).  If the item is not a SitemapFile,
+          # then it has been manually added and we can be sure that the user intends
+          # for there to be an index.
+          if @link_count == 0
+            @first_sitemap = SitemapGenerator::Builder::LinkHolder.new(file, options)
+            @link_count += 1     # pretend it's added, but don't add it yet
+          else
+            # need an index so make sure name is reserved and first sitemap is written out
+            reserve_name unless @location.create_index == false
+            write_first_sitemap
+            file.write
+            super(SitemapGenerator::Builder::SitemapIndexUrl.new(file, options))
+          end
+        else
+          # A link is being added manually.  Obviously the user wants an index.
+          # This overrides the create_index setting.
+          unless @location.create_index == false
+            @create_index = true
+            reserve_name
+          end
+
+          # Use the host from the location if none provided
+          options[:host] ||= @location.host
+          super(SitemapGenerator::Builder::SitemapIndexUrl.new(link, options))
+        end
+      end
+
+      # Return a boolean indicating whether the sitemap file can fit another link
+      # of <tt>bytes</tt> bytes in size.  You can also pass a string and the
+      # bytesize will be calculated for you.
+      def file_can_fit?(bytes)
+        bytes = bytes.is_a?(String) ? SitemapGenerator::Utilities.bytesize(bytes) : bytes
+        (@filesize + bytes) < SitemapGenerator::MAX_SITEMAP_FILESIZE && @link_count < SitemapGenerator::MAX_SITEMAP_FILES
+      end
+
+      # Return the total number of links in all sitemaps reference by this index file
+      def total_link_count
+        @sitemaps_link_count
+      end
+
+      def stats_summary(opts={})
+        str = "Sitemap stats: #{number_with_delimiter(@sitemaps_link_count)} links / #{@link_count} sitemaps"
+        str += " / %dm%02ds" % opts[:time_taken].divmod(60) if opts[:time_taken]
+      end
+
+      def finalize!
+        raise SitemapGenerator::SitemapFinalizedError if finalized?
+        reserve_name if create_index?
+        write_first_sitemap
+        @frozen = true
+      end
+
+      # Write out the index if an index is needed
+      def write
+        super if create_index?
+      end
+
+      # Whether or not we need to create an index file.  True if create_index is true
+      # or if create_index is :auto and we have more than one link in the index.
+      # If a link is added manually and create_index is not false, we force index
+      # creation because they obviously intend for there to be an index.  False otherwise.
+      def create_index?
+        @create_index || @location.create_index == true || @location.create_index == :auto && @link_count > 1
+      end
+
+      # Return the index file URL.  If create_index is true, this is the URL
+      # of the actual index file.  If create_index is false, this is the URL
+      # of the first sitemap that was written out.  Only call this method
+      # *after* the files have been finalized.
+      def index_url
+        if create_index? || !@first_sitemap_url
+          @location.url
+        else
+          @first_sitemap_url
+        end
+      end
+
+      protected
+
+      # Make sure the first sitemap has been written out and added to the index
+      def write_first_sitemap
+        if @first_sitemap
+          @first_sitemap.link.write unless @first_sitemap.link.written?
+          super_add(SitemapGenerator::Builder::SitemapIndexUrl.new(@first_sitemap.link, @first_sitemap.options))
+          @link_count -= 1   # we already counted it, don't count it twice
+          # Store the URL because if create_index is false, this is the
+          # "index" URL
+          @first_sitemap_url = @first_sitemap.link.location.url
+          @first_sitemap = nil
+        end
+      end
+    end
+  end
+end

data/lib/sitemap_generator/builder/sitemap_index_url.rb

@@ -0,0 +1,28 @@
+require 'builder'
+
+module SitemapGenerator
+  module Builder
+    class SitemapIndexUrl < SitemapUrl
+
+      def initialize(path, options={})
+        if index = path.is_a?(SitemapGenerator::Builder::SitemapIndexFile) && path
+          options = SitemapGenerator::Utilities.reverse_merge(options, :host => index.location.host, :lastmod => Time.now, :changefreq => 'always', :priority => 1.0)
+          path = index.location.path_in_public
+          super(path, options)
+        else
+          super
+        end
+      end
+
+      # Return the URL as XML
+      def to_xml(builder=nil)
+        builder = ::Builder::XmlMarkup.new if builder.nil?
+        builder.sitemap do
+          builder.loc        self[:loc]
+          builder.lastmod    w3c_date(self[:lastmod])   if self[:lastmod]
+        end
+        builder << '' # force to string
+      end
+    end
+  end
+end