ftbpro_sitemap_generator 5.0.8

data/Rakefile

@@ -0,0 +1,43 @@
+require 'bundler/setup'
+Bundler.require
+
+desc 'Default: run spec tests.'
+task :default => :spec
+
+require "rspec/core/rake_task"
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = Dir.glob(['spec/sitemap_generator/**/*'])
+  spec.rspec_opts = ['--backtrace']
+end
+
+#
+# Helpers
+#
+
+def name;         @name ||= Dir['*.gemspec'].first.split('.').first end
+def version;      File.read('VERSION').chomp end
+def gemspec_file; "#{name}.gemspec" end
+def gem_file;     "#{name}-#{version}.gem" end
+
+#
+# Release Tasks
+# @see https://github.com/mojombo/rakegem
+#
+
+desc "Create tag v#{version}, build the gem and push to Git"
+task :release => :build do
+  unless `git branch` =~ /^\* master$/
+    puts "You must be on the master branch to release!"
+    exit!
+  end
+  sh "git tag v#{version}"
+  sh "git push origin master --tags"
+end
+
+desc "Build #{gem_file} into the pkg/ directory"
+task :build do
+  sh "mkdir -p pkg"
+  sh "gem build #{gemspec_file}"
+  sh "mv #{gem_file} pkg"
+  sh "bundle --local"
+end

data/VERSION

@@ -0,0 +1 @@
+5.0.8

data/lib/capistrano/sitemap_generator.rb

@@ -0,0 +1 @@
+load File.expand_path(File.join('..', 'tasks', 'sitemap_generator.cap'), __FILE__)

data/lib/capistrano/tasks/sitemap_generator.cap

@@ -0,0 +1,36 @@
+namespace :deploy do
+  namespace :sitemap do
+    desc 'Create sitemap and ping search engines'
+    task :refresh do
+      on roles :web do
+        within release_path do
+          with rails_env: fetch(:rails_env) do
+           execute :rake, "sitemap:refresh"
+          end
+        end
+      end
+    end
+
+    desc 'Create sitemap without pinging search engines'
+    task :create do
+      on roles :web do
+        within release_path do
+          with rails_env: fetch(:rails_env) do
+           execute :rake, "sitemap:create"
+          end
+        end
+      end
+    end
+
+    desc 'Clean up sitemaps in sitemap_generator path'
+    task :clean do
+      on roles :web do
+        within release_path do
+          with rails_env: fetch(:rails_env) do
+           execute :rake, "sitemap:clean"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sitemap_generator.rb

@@ -0,0 +1,85 @@
+require 'sitemap_generator/sitemap_namer'
+require 'sitemap_generator/builder'
+require 'sitemap_generator/link_set'
+require 'sitemap_generator/templates'
+require 'sitemap_generator/utilities'
+require 'sitemap_generator/application'
+require 'sitemap_generator/adapters'
+require 'sitemap_generator/sitemap_location'
+
+module SitemapGenerator
+  autoload(:Interpreter, 'sitemap_generator/interpreter')
+  autoload(:FileAdapter, 'sitemap_generator/adapters/file_adapter')
+  autoload(:S3Adapter,   'sitemap_generator/adapters/s3_adapter')
+  autoload(:WaveAdapter, 'sitemap_generator/adapters/wave_adapter')
+  autoload(:FogAdapter,  'sitemap_generator/adapters/fog_adapter')
+  autoload(:BigDecimal,  'sitemap_generator/core_ext/big_decimal')
+  autoload(:Numeric,     'sitemap_generator/core_ext/numeric')
+
+  SitemapError          = Class.new(StandardError)
+  SitemapFullError      = Class.new(SitemapError)
+  SitemapFinalizedError = Class.new(SitemapError)
+
+  Utilities.with_warnings(nil) do
+    VERSION = File.read(File.dirname(__FILE__) + "/../VERSION").strip
+    MAX_SITEMAP_FILES    = 50_000        # max sitemap links per index file
+    MAX_SITEMAP_LINKS    = 50_000        # max links per sitemap
+    MAX_SITEMAP_IMAGES   = 1_000         # max images per url
+    MAX_SITEMAP_NEWS     = 1_000         # max news sitemap per index_file
+    MAX_SITEMAP_FILESIZE = 10_000_000    # bytes
+    SCHEMAS = {
+      'geo'     => 'http://www.google.com/geo/schemas/sitemap/1.0',
+      'image'   => 'http://www.google.com/schemas/sitemap-image/1.1',
+      'mobile'  => 'http://www.google.com/schemas/sitemap-mobile/1.0',
+      'news'    => 'http://www.google.com/schemas/sitemap-news/0.9',
+      'pagemap' => 'http://www.google.com/schemas/sitemap-pagemap/1.0',
+      'video'   => 'http://www.google.com/schemas/sitemap-video/1.1'
+    }
+    SCHEMA_LOCATION = "http://www.sitemaps.org/schemas/sitemap/0.9
+                       http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"
+
+    # Lazy-initialize the LinkSet instance
+    Sitemap = (Class.new do
+      def method_missing(*args, &block)
+        (@link_set ||= reset!).send(*args, &block)
+      end
+
+      # Use a new LinkSet instance
+      def reset!
+        @link_set = LinkSet.new
+      end
+    end).new
+  end
+
+  class << self
+    attr_accessor :root, :app, :templates
+    attr_writer :yield_sitemap, :verbose
+  end
+
+  # Global default for the verbose setting.
+  def self.verbose
+    if @verbose.nil?
+      @verbose = if SitemapGenerator::Utilities.truthy?(ENV['VERBOSE'])
+        true
+      elsif SitemapGenerator::Utilities.falsy?(ENV['VERBOSE'])
+        false
+      else
+        nil
+      end
+    else
+      @verbose
+    end
+  end
+
+  # Returns true if we should yield the sitemap instance to the block, false otherwise.
+  def self.yield_sitemap?
+    !!@yield_sitemap
+  end
+
+  self.root      = File.expand_path(File.join(File.dirname(__FILE__), '../'))  # Root of the install dir, not the Rails app
+  self.templates = SitemapGenerator::Templates.new(self.root)
+  self.app       = SitemapGenerator::Application.new
+end
+
+require 'sitemap_generator/railtie' if SitemapGenerator.app.rails3?
+

data/lib/sitemap_generator/adapters/file_adapter.rb

@@ -0,0 +1,43 @@
+module SitemapGenerator
+  # Class for writing out data to a file.
+  class FileAdapter
+
+    # Write data to a file.
+    # @param location - File object giving the full path and file name of the file.
+    #    If the location specifies a directory(ies) which does not exist, the directory(ies)
+    #    will be created for you.  If the location path ends with `.gz` the data will be
+    #    compressed prior to being written out.  Otherwise the data will be written out
+    #    unchanged.
+    # @param raw_data - data to write to the file.
+    def write(location, raw_data)
+      # Ensure that the directory exists
+      dir = location.directory
+      if !File.exists?(dir)
+        FileUtils.mkdir_p(dir)
+      elsif !File.directory?(dir)
+        raise SitemapError.new("#{dir} should be a directory!")
+      end
+
+      stream = open(location.path, 'wb')
+      if location.path.to_s =~ /.gz$/
+        gzip(stream, raw_data)
+      else
+        plain(stream, raw_data)
+      end
+    end
+
+    # Write `data` to a stream, passing the data through a GzipWriter
+    # to compress it.
+    def gzip(stream, data)
+      gz = Zlib::GzipWriter.new(stream)
+      gz.write data
+      gz.close
+    end
+
+    # Write `data` to a stream as is.
+    def plain(stream, data)
+      stream.write data
+      stream.close
+    end
+  end
+end

data/lib/sitemap_generator/adapters/fog_adapter.rb

@@ -0,0 +1,28 @@
+begin
+  require 'fog'
+rescue LoadError
+  raise LoadError.new("Missing required 'fog'.  Please 'gem install fog' and require it in your application.")
+end
+
+module SitemapGenerator
+  class FogAdapter
+
+    def initialize(opts = {})
+      @fog_credentials = opts[:fog_credentials]
+      @fog_directory = opts[:fog_directory]
+    end
+
+    # Call with a SitemapLocation and string data
+    def write(location, raw_data)
+      SitemapGenerator::FileAdapter.new.write(location, raw_data)
+
+      storage   = Fog::Storage.new(@fog_credentials)
+      directory = storage.directories.new(:key => @fog_directory)
+      directory.files.create(
+        :key    => location.path_in_public,
+        :body   => File.open(location.path),
+        :public => true
+      )
+    end
+  end
+end

data/lib/sitemap_generator/adapters/s3_adapter.rb

@@ -0,0 +1,41 @@
+begin
+  require 'fog'
+rescue LoadError
+  raise LoadError.new("Missing required 'fog'.  Please 'gem install fog' and require it in your application.")
+end
+
+module SitemapGenerator
+  class S3Adapter
+
+    def initialize(opts = {})
+      @aws_access_key_id = opts[:aws_access_key_id] || ENV['AWS_ACCESS_KEY_ID']
+      @aws_secret_access_key = opts[:aws_secret_access_key] || ENV['AWS_SECRET_ACCESS_KEY']
+      @fog_provider = opts[:fog_provider] || ENV['FOG_PROVIDER']
+      @fog_directory = opts[:fog_directory] || ENV['FOG_DIRECTORY']
+      @fog_region = opts[:fog_region] || ENV['FOG_REGION']
+      @fog_path_style = opts[:fog_path_style] || ENV['FOG_PATH_STYLE']
+    end
+
+    # Call with a SitemapLocation and string data
+    def write(location, raw_data)
+      SitemapGenerator::FileAdapter.new.write(location, raw_data)
+
+      credentials = {
+        :aws_access_key_id     => @aws_access_key_id,
+        :aws_secret_access_key => @aws_secret_access_key,
+        :provider              => @fog_provider,
+      }
+      credentials[:region] = @fog_region if @fog_region
+      credentials[:path_style] = @fog_path_style if @fog_path_style
+
+      storage   = Fog::Storage.new(credentials)
+      directory = storage.directories.new(:key => @fog_directory)
+      directory.files.create(
+        :key    => location.path_in_public,
+        :body   => File.open(location.path),
+        :public => true
+      )
+    end
+
+  end
+end

data/lib/sitemap_generator/adapters/wave_adapter.rb

@@ -0,0 +1,21 @@
+begin
+  require 'carrierwave'
+rescue LoadError
+  raise LoadError.new("Missing required 'carrierwave'.  Please 'gem install carrierwave' and require it in your application.")
+end
+
+module SitemapGenerator
+  class WaveAdapter < ::CarrierWave::Uploader::Base
+    attr_accessor :store_dir
+
+    # Call with a SitemapLocation and string data
+    def write(location, raw_data)
+      SitemapGenerator::FileAdapter.new.write(location, raw_data)
+      directory = File.dirname(location.path_in_public)
+      if directory != '.'
+        self.store_dir = directory
+      end
+      store!(open(location.path, 'rb'))
+    end
+  end
+end

data/lib/sitemap_generator/application.rb

@@ -0,0 +1,49 @@
+require 'pathname'
+
+module SitemapGenerator
+  class Application
+    def rails?
+      defined?(Rails)
+    end
+
+    # Returns a boolean indicating whether this environment is Rails 3
+    #
+    # @return [Boolean]
+    def rails3?
+      rails? && Rails.version.to_f >= 3
+    rescue
+      false # Rails.version defined in 2.1.0
+    end
+
+    def root
+      Pathname.new(rails_root || Dir.getwd)
+    end
+
+    protected
+
+    # Returns the root of the Rails application,
+    # if this is running in a Rails context.
+    # Returns `nil` if no such root is defined.
+    #
+    # @return [String, nil]
+    def rails_root
+      if defined?(::Rails.root)
+        return ::Rails.root.to_s if ::Rails.root
+        raise "ERROR: Rails.root is nil!"
+      end
+      return RAILS_ROOT.to_s if defined?(RAILS_ROOT)
+      return nil
+    end
+
+    # Returns the environment of the Rails application,
+    # if this is running in a Rails context.
+    # Returns `nil` if no such environment is defined.
+    #
+    # @return [String, nil]
+    def rails_env
+      return ::Rails.env.to_s if defined?(::Rails.env)
+      return RAILS_ENV.to_s if defined?(RAILS_ENV)
+      return nil
+    end
+  end
+end

data/lib/sitemap_generator/builder.rb

@@ -0,0 +1,8 @@
+require 'sitemap_generator/builder/sitemap_file'
+require 'sitemap_generator/builder/sitemap_index_file'
+require 'sitemap_generator/builder/sitemap_url'
+require 'sitemap_generator/builder/sitemap_index_url'
+
+module SitemapGenerator::Builder
+  LinkHolder = Struct.new(:link, :options)
+end

data/lib/sitemap_generator/builder/sitemap_file.rb

@@ -0,0 +1,172 @@
+require 'zlib'
+require 'fileutils'
+require 'sitemap_generator/helpers/number_helper'
+
+module SitemapGenerator
+  module Builder
+    #
+    # General Usage:
+    #
+    #   sitemap = SitemapFile.new(:location => SitemapLocation.new(...))
+    #   sitemap.add('/', { ... })    <- add a link to the sitemap
+    #   sitemap.finalize!            <- write the sitemap file and freeze the object to protect it from further modification
+    #
+    class SitemapFile
+      include SitemapGenerator::Helpers::NumberHelper
+      attr_reader :link_count, :filesize, :location, :news_count
+
+      # === Options
+      #
+      # * <tt>location</tt> - a SitemapGenerator::SitemapLocation instance or a Hash of options
+      #   from which a SitemapLocation will be created for you.
+      def initialize(opts={}, schemas, schema_location)
+        @location = opts.is_a?(Hash) ? SitemapGenerator::SitemapLocation.new(opts) : opts
+        @link_count = 0
+        @news_count = 0
+        @xml_content = '' # XML urlset content
+        @schemas = schemas
+        @schema_location = schema_location
+        @xml_wrapper_start = xml_wrapper_start.gsub(/\s+/, ' ').gsub(/ *> */, '>').strip.slice(1..-2)
+        @xml_wrapper_end   = %q[</urlset>]
+        @filesize = SitemapGenerator::Utilities.bytesize(@xml_wrapper_start) + SitemapGenerator::Utilities.bytesize(@xml_wrapper_end)
+        @written = false
+        @reserved_name = nil # holds the name reserved from the namer
+        @frozen = false      # rather than actually freeze, use this boolean
+      end
+
+      def xml_wrapper_start
+        wrapper_start = '<?xml version="1.0" encoding="UTF-8"?>
+            <urlset
+              xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
+        schema_location = "xsi:schemaLocation=\"#{@schema_location}\""
+        general_schema = 'xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"'
+        customized_schemas = @schemas.collect do |schema, content|
+          "xmlns:#{schema}=\"#{content}\""
+        end
+        wrapper_end = 'xmlns:xhtml="http://www.w3.org/1999/xhtml">'
+        xml_start = [wrapper_start, schema_location, general_schema, customized_schemas, wrapper_end].join(" ")
+        return <<-HTML
+          "#{xml_start}"
+        HTML
+      end
+
+      # If a name has been reserved, use the last modified time from the file.
+      # Otherwise return nil.  We don't want to prematurely assign a name
+      # for this sitemap if one has not yet been reserved, because we may
+      # mess up the name-assignment sequence.
+      def lastmod
+        File.mtime(location.path) if location.reserved_name?
+      rescue
+        nil
+      end
+
+      def empty?
+        @link_count == 0
+      end
+
+      # Return a boolean indicating whether the sitemap file can fit another link
+      # of <tt>bytes</tt> bytes in size.  You can also pass a string and the
+      # bytesize will be calculated for you.
+      def file_can_fit?(bytes)
+        bytes = bytes.is_a?(String) ? SitemapGenerator::Utilities.bytesize(bytes) : bytes
+        (@filesize + bytes) < SitemapGenerator::MAX_SITEMAP_FILESIZE && @link_count < SitemapGenerator::MAX_SITEMAP_LINKS && @news_count < SitemapGenerator::MAX_SITEMAP_NEWS
+      end
+
+      # Add a link to the sitemap file.
+      #
+      # If a link cannot be added, for example if the file is too large or the link
+      # limit has been reached, a SitemapGenerator::SitemapFullError exception is raised
+      # and the sitemap is finalized.
+      #
+      # If the Sitemap has already been finalized a SitemapGenerator::SitemapFinalizedError
+      # exception is raised.
+      #
+      # Return the new link count.
+      #
+      # Call with:
+      #   sitemap_url - a SitemapUrl instance
+      #   sitemap, options - a Sitemap instance and options hash
+      #   path, options - a path for the URL and options hash.  For supported options
+      #                   see the SitemapGenerator::Builder::SitemapUrl class.
+      #
+      # The link added to the sitemap will use the host from its location object
+      # if no host has been specified.
+      def add(link, options={})
+        raise SitemapGenerator::SitemapFinalizedError if finalized?
+
+        sitemap_url = if link.is_a?(SitemapUrl)
+          link
+        else
+          options[:host] ||= @location.host
+          SitemapUrl.new(link, options)
+        end
+
+        xml = sitemap_url.to_xml
+        raise SitemapGenerator::SitemapFullError if !file_can_fit?(xml)
+
+        if sitemap_url.news?
+          @news_count += 1
+        end
+
+        # Add the XML to the sitemap
+        @xml_content << xml
+        @filesize += SitemapGenerator::Utilities.bytesize(xml)
+        @link_count += 1
+      end
+
+      # "Freeze" this object.  Actually just flags it as frozen.
+      #
+      # A SitemapGenerator::SitemapFinalizedError exception is raised if the Sitemap
+      # has already been finalized.
+      def finalize!
+        raise SitemapGenerator::SitemapFinalizedError if finalized?
+        @frozen = true
+      end
+
+      def finalized?
+        @frozen
+      end
+
+      # Write out the sitemap and free up memory.
+      #
+      # All the xml content in the instance is cleared, but attributes like
+      # <tt>filesize</tt> are still available.
+      #
+      # A SitemapGenerator::SitemapError exception is raised if the file has
+      # already been written.
+      def write
+        raise SitemapGenerator::SitemapError.new("Sitemap already written!") if written?
+        finalize! unless finalized?
+        reserve_name
+        @location.write(@xml_wrapper_start + @xml_content + @xml_wrapper_end, link_count)
+        @xml_content = @xml_wrapper_start = @xml_wrapper_end = ''
+        @written = true
+      end
+
+      # Return true if this file has been written out to disk
+      def written?
+        @written
+      end
+
+      # Reserve a name from the namer unless one has already been reserved.
+      # Safe to call more than once.
+      def reserve_name
+        @reserved_name ||= @location.reserve_name
+      end
+
+      # Return a boolean indicating whether a name has been reserved
+      def reserved_name?
+        !!@reserved_name
+      end
+
+      # Return a new instance of the sitemap file with the same options,
+      # and the next name in the sequence.
+      def new
+        location = @location.dup
+        location.delete(:filename) if location.namer
+        self.class.new(location, @schemas, @schema_location)
+      end
+    end
+  end
+end
+