sitemap_generator 5.3.1 → 6.1.1

data/VERSION

@@ -1 +1 @@
-5.3.1
+6.1.1

data/lib/capistrano/tasks/sitemap_generator.cap

@@ -1,7 +1,7 @@
 namespace :sitemap do
   desc 'Create sitemap and ping search engines'
   task :refresh do
-    on roles :web do
+    on roles fetch(:sitemap_roles, :web) do
       within release_path do
         with rails_env: (fetch(:rails_env) || fetch(:stage)) do
          execute :rake, "sitemap:refresh"
@@ -12,7 +12,7 @@ namespace :sitemap do
 
   desc 'Create sitemap without pinging search engines'
   task :create do
-    on roles :web do
+    on roles fetch(:sitemap_roles, :web) do
       within release_path do
         with rails_env: (fetch(:rails_env) || fetch(:stage)) do
          execute :rake, "sitemap:create"
@@ -23,7 +23,7 @@ namespace :sitemap do
 
   desc 'Clean up sitemaps in sitemap_generator path'
   task :clean do
-    on roles :web do
+    on roles fetch(:sitemap_roles, :web) do
       within release_path do
         with rails_env: (fetch(:rails_env) || fetch(:stage)) do
          execute :rake, "sitemap:clean"

data/lib/sitemap_generator.rb

@@ -7,14 +7,15 @@ require 'sitemap_generator/application'
 require 'sitemap_generator/sitemap_location'
 
 module SitemapGenerator
-  autoload(:Interpreter,   'sitemap_generator/interpreter')
-  autoload(:FileAdapter,   'sitemap_generator/adapters/file_adapter')
-  autoload(:S3Adapter,     'sitemap_generator/adapters/s3_adapter')
-  autoload(:AwsSdkAdapter, 'sitemap_generator/adapters/aws_sdk_adapter')
-  autoload(:WaveAdapter,   'sitemap_generator/adapters/wave_adapter')
-  autoload(:FogAdapter,    'sitemap_generator/adapters/fog_adapter')
-  autoload(:BigDecimal,    'sitemap_generator/core_ext/big_decimal')
-  autoload(:Numeric,       'sitemap_generator/core_ext/numeric')
+  autoload(:Interpreter,              'sitemap_generator/interpreter')
+  autoload(:FileAdapter,              'sitemap_generator/adapters/file_adapter')
+  autoload(:S3Adapter,                'sitemap_generator/adapters/s3_adapter')
+  autoload(:AwsSdkAdapter,            'sitemap_generator/adapters/aws_sdk_adapter')
+  autoload(:WaveAdapter,              'sitemap_generator/adapters/wave_adapter')
+  autoload(:FogAdapter,               'sitemap_generator/adapters/fog_adapter')
+  autoload(:GoogleStorageAdapter,     'sitemap_generator/adapters/google_storage_adapter')
+  autoload(:BigDecimal,               'sitemap_generator/core_ext/big_decimal')
+  autoload(:Numeric,                  'sitemap_generator/core_ext/numeric')
 
   SitemapError          = Class.new(StandardError)
   SitemapFullError      = Class.new(SitemapError)
@@ -26,7 +27,7 @@ module SitemapGenerator
     MAX_SITEMAP_LINKS    = 50_000        # max links per sitemap
     MAX_SITEMAP_IMAGES   = 1_000         # max images per url
     MAX_SITEMAP_NEWS     = 1_000         # max news sitemap per index_file
-    MAX_SITEMAP_FILESIZE = 10_000_000    # bytes
+    MAX_SITEMAP_FILESIZE = 50_000_000    # bytes
     SCHEMAS = {
       'image'   => 'http://www.google.com/schemas/sitemap-image/1.1',
       'mobile'  => 'http://www.google.com/schemas/sitemap-mobile/1.0',
@@ -56,6 +57,7 @@ module SitemapGenerator
     attr_accessor :root, :app, :templates
     attr_writer :yield_sitemap, :verbose
   end
+  @yield_sitemap = nil
 
   # Global default for the verbose setting.
   def self.verbose
@@ -82,4 +84,4 @@ module SitemapGenerator
   self.app       = SitemapGenerator::Application.new
 end
 
-require 'sitemap_generator/railtie' if SitemapGenerator.app.rails3?
+require 'sitemap_generator/railtie' if SitemapGenerator.app.is_at_least_rails3?

data/lib/sitemap_generator/adapters/aws_sdk_adapter.rb

@@ -1,45 +1,60 @@
-begin
-  require 'aws-sdk'
-rescue LoadError
-  raise LoadError.new("Missing required 'aws-sdk'.  Please 'gem install "\
-                      "aws-sdk' and require it in your application, or "\
-                      "add: gem 'aws-sdk' to your Gemfile.")
+if !defined?(Aws::S3::Resource) or !defined?(Aws::Credentials)
+  raise "Error: `Aws::S3::Resource` and/or `Aws::Credentials` are not defined.\n\n"\
+        "Please `require 'aws-sdk'` - or another library that defines these classes."
 end
 
 module SitemapGenerator
-  # Class for uploading the sitemaps to an S3 bucket using the plain AWS SDK gem
+  # Class for uploading sitemaps to an S3 bucket using the AWS SDK gem.
   class AwsSdkAdapter
-    # @param [String] bucket name of the S3 bucket
-    # @param [Hash]   opts   alternate means of configuration other than ENV
-    # @option opts  [String] :aws_access_key_id instead of ENV['AWS_ACCESS_KEY_ID']
-    # @option opts  [String] :aws_region instead of ENV['AWS_REGION']
-    # @option opts  [String] :aws_secret_access_key instead of ENV['AWS_SECRET_ACCESS_KEY']
-    # @option opts  [String] :path use this prefix on the object key instead of 'sitemaps/'
-    def initialize(bucket, opts = {})
+    # Specify your AWS bucket name, credentials, and/or region.  By default
+    # the AWS SDK will auto-detect your credentials and region, but you can use
+    # the following options to configure - or override - them manually:
+    #
+    # Options:
+    #   :aws_access_key_id [String] Your AWS access key id
+    #   :aws_secret_access_key [String] Your AWS secret access key
+    #   :aws_region [String] Your AWS region
+    #
+    # Requires Aws::S3::Resource and Aws::Credentials to be defined.
+    #
+    # @param [String] bucket Name of the S3 bucket
+    # @param [Hash] options AWS credential overrides, see above
+    def initialize(bucket, options = {})
       @bucket = bucket
-
-      @aws_access_key_id = opts[:aws_access_key_id] || ENV['AWS_ACCESS_KEY_ID']
-      @aws_region = opts[:aws_region] || ENV['AWS_REGION']
-      @aws_secret_access_key = opts[:aws_secret_access_key] || ENV['AWS_SECRET_ACCESS_KEY']
-
-      @path = opts[:path] || 'sitemaps/'
+      @aws_access_key_id = options[:aws_access_key_id]
+      @aws_secret_access_key = options[:aws_secret_access_key]
+      @aws_region = options[:aws_region]
+      @aws_endpoint = options[:aws_endpoint]
     end
 
     # Call with a SitemapLocation and string data
     def write(location, raw_data)
       SitemapGenerator::FileAdapter.new.write(location, raw_data)
+      s3_object = s3_resource.bucket(@bucket).object(location.path_in_public)
+      s3_object.upload_file(location.path,
+        acl: 'public-read',
+        cache_control: 'private, max-age=0, no-cache',
+        content_type: location[:compress] ? 'application/x-gzip' : 'application/xml'
+      )
+    end
 
-      credentials = Aws::Credentials.new(@aws_access_key_id, @aws_secret_access_key)
-      s3 = Aws::S3::Resource.new(credentials: credentials, region: @aws_region)
+    private
 
-      s3_object_key = "#{@path}#{location.path_in_public}"
-      s3_object = s3.bucket(@bucket).object(s3_object_key)
+    def s3_resource
+      @s3_resource ||= Aws::S3::Resource.new(s3_resource_options)
+    end
 
-      content_type = location[:compress] ? 'application/x-gzip' : 'application/xml'
-      s3_object.upload_file(location.path,
-                            acl: 'public-read',
-                            cache_control: 'private, max-age=0, no-cache',
-                            content_type: content_type)
+    def s3_resource_options
+      options = {}
+      options[:region] = @aws_region if !@aws_region.nil?
+      options[:endpoint] = @aws_endpoint if !@aws_endpoint.nil?
+      if !@aws_access_key_id.nil? && !@aws_secret_access_key.nil?
+        options[:credentials] = Aws::Credentials.new(
+          @aws_access_key_id,
+          @aws_secret_access_key
+        )
+      end
+      options
     end
   end
 end

data/lib/sitemap_generator/adapters/file_adapter.rb

@@ -1,7 +1,6 @@
 module SitemapGenerator
   # Class for writing out data to a file.
   class FileAdapter
-
     # Write data to a file.
     # @param location - File object giving the full path and file name of the file.
     #    If the location specifies a directory(ies) which does not exist, the directory(ies)
@@ -12,7 +11,7 @@ module SitemapGenerator
     def write(location, raw_data)
       # Ensure that the directory exists
       dir = location.directory
-      if !File.exists?(dir)
+      if !File.exist?(dir)
         FileUtils.mkdir_p(dir)
       elsif !File.directory?(dir)
         raise SitemapError.new("#{dir} should be a directory!")

data/lib/sitemap_generator/adapters/fog_adapter.rb

@@ -1,12 +1,16 @@
-begin
-  require 'fog'
-rescue LoadError
-  raise LoadError.new("Missing required 'fog'.  Please 'gem install fog' and require it in your application.")
+if !defined?(Fog::Storage)
+  raise "Error: `Fog::Storage` is not defined.\n\n"\
+        "Please `require 'fog'` - or another library that defines this class."
 end
 
 module SitemapGenerator
+  # Class for uploading sitemaps to a Fog supported endpoint.
   class FogAdapter
-
+    # Requires Fog::Storage to be defined.
+    #
+    # @param [Hash] opts Fog configuration options
+    # @option :fog_credentials [Hash] Credentials for connecting to the remote server
+    # @option :fog_directory [String] Your AWS S3 bucket or similar directory name
     def initialize(opts = {})
       @fog_credentials = opts[:fog_credentials]
       @fog_directory = opts[:fog_directory]

data/lib/sitemap_generator/adapters/google_storage_adapter.rb

@@ -0,0 +1,37 @@
+if !defined?(Google::Cloud::Storage)
+  raise "Error: `Google::Cloud::Storage` is not defined.\n\n"\
+        "Please `require 'google/cloud/storage'` - or another library that defines this class."
+end
+
+module SitemapGenerator
+  # Class for uploading sitemaps to a Google Storage using `google-cloud-storage` gem.
+  class GoogleStorageAdapter
+    # Requires Google::Cloud::Storage to be defined.
+    #
+    # @param [Hash] opts Google::Cloud::Storage configuration options.
+    # @option :bucket [String] Required. Name of Google Storage Bucket where the file is to be uploaded.
+    #
+    # All options other than the `:bucket` option are passed to the `Google::Cloud::Storage.new`
+    # initializer.  See https://googleapis.dev/ruby/google-cloud-storage/latest/file.AUTHENTICATION.html
+    # for all the supported environment variables and https://github.com/googleapis/google-cloud-ruby/blob/master/google-cloud-storage/lib/google/cloud/storage.rb
+    # for supported options.
+    #
+    # Suggested Options:
+    # @option :credentials [String] Path to Google service account JSON file, or JSON contents.
+    # @option :project_id [String] Google Accounts project id where the storage bucket resides.
+    def initialize(opts = {})
+      opts = opts.clone
+      @bucket = opts.delete(:bucket)
+      @storage_options = opts
+    end
+
+    # Call with a SitemapLocation and string data
+    def write(location, raw_data)
+      SitemapGenerator::FileAdapter.new.write(location, raw_data)
+
+      storage = Google::Cloud::Storage.new(@storage_options)
+      bucket = storage.bucket(@bucket)
+      bucket.create_file(location.path, location.path_in_public, acl: 'public')
+    end
+  end
+end

data/lib/sitemap_generator/adapters/s3_adapter.rb

@@ -1,17 +1,21 @@
-# Without this require, fog-core 1.2.0 raises
-# NameError: uninitialized constant Fog::ServicesMixin.
-# I don't know which versions this affects.
-require 'fog/core/services_mixin' rescue nil
-
-begin
-  require 'fog/storage'
-rescue LoadError
-  raise LoadError.new("Missing required 'fog-aws'.  Please 'gem install fog-aws' and require it in your application.")
+if !defined?(Fog::Storage)
+  raise "Error: `Fog::Storage` is not defined.\n\n"\
+        "Please `require 'fog-aws'` - or another library that defines this class."
 end
 
 module SitemapGenerator
+  # Class for uploading sitemaps to an S3 bucket using the Fog gem.
   class S3Adapter
-
+    # Requires Fog::Storage to be defined.
+    #
+    # @param [Hash] opts Fog configuration options
+    # @option :aws_access_key_id [String] Your AWS access key id
+    # @option :aws_secret_access_key [String] Your AWS secret access key
+    # @option :fog_provider [String]
+    # @option :fog_directory [String]
+    # @option :fog_region [String]
+    # @option :fog_path_style [String]
+    # @option :fog_storage_options [Hash] Other options to pass to `Fog::Storage`
     def initialize(opts = {})
       @aws_access_key_id = opts[:aws_access_key_id] || ENV['AWS_ACCESS_KEY_ID']
       @aws_secret_access_key = opts[:aws_secret_access_key] || ENV['AWS_SECRET_ACCESS_KEY']
@@ -46,6 +50,5 @@ module SitemapGenerator
         :public => true
       )
     end
-
   end
 end

data/lib/sitemap_generator/adapters/wave_adapter.rb

@@ -1,10 +1,10 @@
-begin
-  require 'carrierwave'
-rescue LoadError
-  raise LoadError.new("Missing required 'carrierwave'.  Please 'gem install carrierwave' and require it in your application.")
+if !defined?(::CarrierWave::Uploader::Base)
+  raise "Error: `CarrierWave::Uploader::Base` is not defined.\n\n"\
+        "Please `require 'carrierwave'` - or another library that defines this class."
 end
 
 module SitemapGenerator
+  # Class for uploading sitemaps to a remote server using the CarrierWave gem.
   class WaveAdapter < ::CarrierWave::Uploader::Base
     attr_accessor :store_dir
 

data/lib/sitemap_generator/application.rb

@@ -2,15 +2,15 @@ require 'pathname'
 
 module SitemapGenerator
   class Application
-    def rails?
+    def is_rails?
       !!defined?(Rails::VERSION)
     end
 
     # Returns a boolean indicating whether this environment is Rails 3
     #
     # @return [Boolean]
-    def rails3?
-      rails? && Rails.version.to_f >= 3
+    def is_at_least_rails3?
+      is_rails? && Rails.version.to_f >= 3
     rescue
       false # Rails.version defined in 2.1.0
     end
@@ -27,12 +27,9 @@ module SitemapGenerator
     #
     # @return [String, nil]
     def rails_root
-      if defined?(::Rails.root)
-        return ::Rails.root.to_s if ::Rails.root
-        raise "ERROR: Rails.root is nil!"
-      end
+      return ::Rails.root.to_s if defined?(::Rails.root) && ::Rails.root
       return RAILS_ROOT.to_s if defined?(RAILS_ROOT)
-      return nil
+      nil
     end
 
     # Returns the environment of the Rails application,
@@ -43,7 +40,7 @@ module SitemapGenerator
     def rails_env
       return ::Rails.env.to_s if defined?(::Rails.env)
       return RAILS_ENV.to_s if defined?(RAILS_ENV)
-      return nil
+      nil
     end
   end
 end

data/lib/sitemap_generator/builder/sitemap_index_file.rb

@@ -30,6 +30,7 @@ module SitemapGenerator
         # Store the URL of the first sitemap added because if create_index is
         # false this is the "index" URL
         @first_sitemap_url = nil
+        @create_index = nil
       end
 
       # Finalize sitemaps as they are added to the index.

data/lib/sitemap_generator/core_ext/big_decimal.rb

@@ -8,18 +8,30 @@ end
 require 'yaml'
 
 # Define our own class rather than modify the global class
-class SitemapGenerator::BigDecimal < BigDecimal
+class SitemapGenerator::BigDecimal
   YAML_TAG = 'tag:yaml.org,2002:float'
   YAML_MAPPING = { 'Infinity' => '.Inf', '-Infinity' => '-.Inf', 'NaN' => '.NaN' }
 
-  yaml_as YAML_TAG
+  yaml_tag YAML_TAG
+
+  def initialize(num)
+    @value = BigDecimal(num)
+  end
+
+  def *(other)
+    other * @value
+  end
+
+  def /(other)
+    SitemapGenerator::BigDecimal === other ? @value / other.instance_variable_get(:@value) : @value / other
+  end
 
   # This emits the number without any scientific notation.
   # This is better than self.to_f.to_s since it doesn't lose precision.
   #
   # Note that reconstituting YAML floats to native floats may lose precision.
   def to_yaml(opts = {})
-    return super if defined?(YAML::ENGINE) && !YAML::ENGINE.syck?
+    return super unless defined?(YAML::ENGINE) && YAML::ENGINE.syck?
 
     YAML.quick_emit(nil, opts) do |out|
       string = to_s
@@ -37,9 +49,7 @@ class SitemapGenerator::BigDecimal < BigDecimal
   end
 
   DEFAULT_STRING_FORMAT = 'F'
-  def to_formatted_s(format = DEFAULT_STRING_FORMAT)
-    _original_to_s(format)
+  def to_s(format = DEFAULT_STRING_FORMAT)
+    @value.to_s(format)
   end
-  alias_method :_original_to_s, :to_s
-  alias_method :to_s, :to_formatted_s
 end

data/lib/sitemap_generator/interpreter.rb

@@ -6,9 +6,11 @@ module SitemapGenerator
   # and API methods available to it.
   class Interpreter
 
-    if SitemapGenerator.app.rails3?
-      include ::Rails.application.routes.url_helpers
-    elsif SitemapGenerator.app.rails?
+    if SitemapGenerator.app.is_at_least_rails3?
+      if !::Rails.application.nil?
+        include ::Rails.application.routes.url_helpers
+      end
+    elsif SitemapGenerator.app.is_rails?
       require 'action_controller'
       include ActionController::UrlWriter
     end
@@ -34,7 +36,7 @@ module SitemapGenerator
     def add_to_index(*args)
       @linkset.add_to_index(*args)
     end
-    
+
     # Start a new group of sitemaps.  Any of the options to SitemapGenerator.new may
     # be passed.  Pass a block with calls to +add+ to add links to the sitemaps.
     #

data/lib/sitemap_generator/link_set.rb

@@ -118,6 +118,8 @@ module SitemapGenerator
     # Note: When adding a new option be sure to include it in `options_for_group()` if
     # the option should be inherited by groups.
     def initialize(options={})
+      @default_host, @sitemaps_host, @yield_sitemap, @sitemaps_path, @adapter, @verbose, @protect_index, @sitemap_index, @added_default_links, @created_group, @sitemap = nil
+
       options = SitemapGenerator::Utilities.reverse_merge(options,
         :include_root => true,
         :include_index => false,
@@ -217,6 +219,7 @@ module SitemapGenerator
         @sitemap.location.merge!(@group.sitemap_location)
         if block_given?
           @group.interpreter.eval(:yield_sitemap => @yield_sitemap || SitemapGenerator.yield_sitemap?, &block)
+          @group.finalize_sitemap!
           @sitemap.location.merge!(@original_location)
         end
       else
@@ -292,7 +295,7 @@ module SitemapGenerator
         name = Utilities.titleize(engine.to_s)
         begin
           Timeout::timeout(10) {
-            open(link)
+            URI.open(link)
           }
           output("  Successful ping of #{name}")
         rescue Timeout::Error, StandardError => e
@@ -430,13 +433,14 @@ module SitemapGenerator
     # Add default links if those options are turned on.  Record the fact that we have done so
     # in an instance variable.
     def add_default_links
+      @added_default_links = true
+      link_options = { :lastmod => Time.now, :changefreq => 'always', :priority => 1.0 }
       if include_root?
-        sitemap.add('/', :lastmod => Time.now, :changefreq => 'always', :priority => 1.0, :host => @default_host)
+        add('/', link_options)
       end
       if include_index?
-        sitemap.add(sitemap_index, :lastmod => Time.now, :changefreq => 'always', :priority => 1.0)
+        add(sitemap_index, link_options)
       end
-      @added_default_links = true
     end
 
     # Finalize a sitemap by including it in the index and outputting a summary line.