big_sitemap 0.8.2 → 0.8.3

data/History.txt

@@ -1,9 +1,15 @@
-=== 0.8.2 / 2010-01-25
+=== 0.8.3 / 2011-03-08
+
+* Separate URL and file paths are now supported via the :document_path
+  and :url_path options
+* Fixes an issue when initializing in Rails 3
+
+=== 0.8.2 / 2011-01-25
 
 * Fixes an issue where sitemap files were not being generated if the same model
   was added more than once (fixes issue #5: https://github.com/alexrabarts/big_sitemap/issues/#issue/5)
 
-=== 0.8.1 / 2010-01-25
+=== 0.8.1 / 2011-01-25
 
 * API change: Rails/Merb are no longer automatically detected - use BigSitemapRails and BigSitemapMerb instead
 * API change: Rails' polymorphic_url helper is no longer used to generate URLs (use a lambda with the new :location option instead)

data/README.rdoc

@@ -61,8 +61,11 @@ Via gem:
 
 * <code>:url_options</code> -- hash with <code>:host</code>, optionally <code>:port</code> and <code>:protocol</code>
 * <code>:base_url</code> -- string alternative to <code>:url_options</code>, e.g. <code>'https://example.com:8080/'</code>
+* <code>:url_path</code> -- string path_name to sitemaps folder, defaults to <code>:document_path</code>
 * <code>:document_root</code> -- string
-* <code>:path</code> -- string defaults to <code>'sitemaps'</code>, which places sitemap files under the <code>/sitemaps</code> directory
+* <code>:document_path</code> -- string document path to generation folder, relative to :document_root, defaults to <code>'sitemaps/'</code>
+* <code>:path</code> -- string, alias for ":document_path" for legacy reasons
+* <code>:document_full</code> -- string absolute document path to generation folder - defaults to <code>:document_root/:document_path</code>
 * <code>:max_per_sitemap</code> -- <code>50000</code>, which is the limit dictated by Google but can be less
 * <code>:batch_size</code> -- <code>1001</code> (not <code>1000</code> due to a bug in DataMapper)
 * <code>:gzip</code> -- <code>true</code>

data/Rakefile

@@ -31,22 +31,4 @@ Rake::TestTask.new(:test) do |t|
   t.verbose = false
 end
 
-begin
-  require 'rcov/rcovtask'
-  Rcov::RcovTask.new do |t|
-    t.libs << 'test'
-    t.test_files = FileList['test/**/*_test.rb']
-    t.verbose = true
-  end
-rescue LoadError
-  puts "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
-end
-
-begin
-  require 'cucumber/rake/task'
-  Cucumber::Rake::Task.new(:features)
-rescue LoadError
-  puts "Cucumber is not available. In order to run features, you must: sudo gem install cucumber"
-end
-
 task :default => :test

data/VERSION.yml

@@ -1,4 +1,5 @@
 --- 
-:patch: 2
 :major: 0
 :minor: 8
+:patch: 3
+:build: 

data/big_sitemap.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{big_sitemap}
-  s.version = "0.8.2"
+  s.version = "0.8.3"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Alex Rabarts", "Tobias Bielohlawek"]
-  s.date = %q{2011-01-25}
+  s.date = %q{2011-03-14}
   s.description = %q{A Sitemap generator specifically designed for large sites (although it works equally well with small sites)}
   s.email = ["alexrabarts@gmail.com", "tobi@soundcloud.com"]
   s.extra_rdoc_files = [

data/lib/big_sitemap.rb

@@ -7,7 +7,7 @@ class BigSitemap
   DEFAULTS = {
     :max_per_sitemap => Builder::MAX_URLS,
     :batch_size      => 1001,
-    :path            => 'sitemaps',
+    :document_path   => 'sitemaps/',
     :gzip            => true,
 
     # opinionated
@@ -24,36 +24,31 @@ class BigSitemap
 
   def initialize(options={})
     @options = DEFAULTS.merge options
-
-    @default_url_options = options.delete(:default_url_options) || {}
+    @options[:document_path] ||= @options[:path] #for legacy reasons
 
     if @options[:max_per_sitemap] <= 1
       raise ArgumentError, '":max_per_sitemap" must be greater than 1'
     end
 
     if @options[:url_options]
-      @default_url_options.update @options[:url_options]
-    elsif @options[:base_url]
-      uri = URI.parse(@options[:base_url])
-      @default_url_options[:host]     = uri.host
-      @default_url_options[:port]     = uri.port
-      @default_url_options[:protocol] = uri.scheme
-    else
+      @options[:base_url] = URI::Generic.build( {:scheme => "http"}.merge(@options.delete(:url_options)) ).to_s
+    end
+
+    unless @options[:base_url]
       raise ArgumentError, 'you must specify either ":url_options" hash or ":base_url" string'
     end
+    @options[:url_path] ||= @options[:document_path]
 
     if @options[:batch_size] > @options[:max_per_sitemap]
       raise ArgumentError, '":batch_size" must be less than ":max_per_sitemap"'
     end
 
-    @options[:document_root] ||= document_root
-
-    unless @options[:document_root]
-      raise ArgumentError, 'Document root must be specified with the ":document_root" option'
+    @options[:document_full] ||= File.join(@options[:document_root], @options[:document_path])
+    unless @options[:document_full]
+      raise ArgumentError, 'Document root must be specified with the ":document_root" option, the full path with ":document_full"'
     end
 
-    @file_path = "#{@options[:document_root]}/#{strip_leading_slash(@options[:path])}"
-    Dir.mkdir(@file_path) unless File.exists? @file_path
+    Dir.mkdir(@options[:document_full]) unless File.exists?(@options[:document_full])
 
     @sources       = []
     @models        = []
@@ -100,14 +95,15 @@ class BigSitemap
 
   def file_name(name)
     name = table_name(name) unless name.is_a? String
-    "#{@file_path}/sitemap_#{name}"
+    File.join(@options[:document_full], "sitemap_#{name}")
   end
 
-  def document_root
+  def dir_files
+    File.join(@options[:document_full], "sitemap_*.{xml,xml.gz}")
   end
 
   def clean
-    Dir["#{@file_path}/sitemap_*.{xml,xml.gz}"].each do |file|
+    Dir[dir_files].each do |file|
       FileUtils.rm file
     end
     self
@@ -161,8 +157,8 @@ class BigSitemap
 
             if last_id && primary_column
               find_options.update(:limit => limit, :offset => nil)
-              primary_column_value = last_id.to_s.gsub("'", %q(\\\')) #escape '
-              find_options.update(:conditions => [find_options[:conditions], "(#{primary_column} > '#{primary_column_value}')"].compact.join(' AND '))
+              primary_column_value = escape_if_string last_id #escape '
+              find_options.update(:conditions => [find_options[:conditions], "(#{primary_column} > #{primary_column_value})"].compact.join(' AND '))
             end
 
             model.send(find_method, find_options).each do |record|
@@ -176,12 +172,12 @@ class BigSitemap
 
               param_method = pick_method(record, PARAM_METHODS)
 
-              location = options[:location]
-              if location.is_a?(Proc)
-                location = location.call(record)
-              else
-                location = "#{root_url}/#{strip_leading_slash(options[:path])}/#{record.send(param_method)}"
-              end
+              location =
+                if options[:location].is_a?(Proc)
+                  options[:location].call(record)
+                else
+                  File.join @options[:base_url], options[:path], record.send(param_method).to_s
+                end
 
               change_frequency = options[:change_frequency] || 'weekly'
               freq = change_frequency.is_a?(Proc) ? change_frequency.call(record) : change_frequency
@@ -211,7 +207,7 @@ class BigSitemap
 
   # Create a sitemap index document
   def generate_sitemap_index(files = nil)
-    files ||= Dir["#{@file_path}/sitemap_*.{xml,xml.gz}"]
+    files ||= Dir[dir_files]
     with_sitemap 'index', :type => 'index' do |sitemap|
       for path in files
         next if path =~ /index/
@@ -238,7 +234,7 @@ class BigSitemap
             "appid=#{@options[:yahoo_app_id]}&url=#{sitemap_uri}"
         )
       else
-        $stderr.puts 'unable to ping Yahoo: no ":yahoo_app_id" provided'
+        STDERR.puts 'unable to ping Yahoo: no ":yahoo_app_id" provided'
       end
     end
 
@@ -251,36 +247,27 @@ class BigSitemap
     end
   end
 
-  def root_url
-    @root_url ||= begin
-      url = ''
-      url << (@default_url_options[:protocol] || 'http')
-      url << '://' unless url.match('://')
-      url << @default_url_options[:host]
-      url << ":#{port}" if port = @default_url_options[:port] and port != 80
-      url
-    end
-  end
-
   private
 
   def prepare_update
     @files_to_move = []
     @sources.each do |model, options|
-      if options[:partial_update] && primary_column = options[:primary_column] && last_id = get_last_id(options[:filename])
-        primary_column_value       = last_id.to_s.gsub("'", %q(\\\')) #escape '
-        options[:conditions]       = [options[:conditions], "(#{primary_column} >= '#{primary_column_value}')"].compact.join(' AND ')
+      if options[:partial_update] && (primary_column = options[:primary_column]) && (last_id = get_last_id(options[:filename]))
+        primary_column_value       = escape_if_string last_id #escape '
+        options[:conditions]       = [options[:conditions], "(#{primary_column} >= #{primary_column_value})"].compact.join(' AND ')
         options[:start_part_id]    = last_id
       end
     end
   end
 
   def lock!(lock_file = 'generator.lock')
-    File.open("#{@file_path}/#{lock_file}", 'w', File::EXCL)
+    lock_file = File.join(@options[:document_full], lock_file)
+    File.open(lock_file, 'w', File::EXCL)
   end
 
   def unlock!(lock_file = 'generator.lock')
-    FileUtils.rm "#{@file_path}/#{lock_file}"
+    lock_file = File.join(@options[:document_full], lock_file)
+    FileUtils.rm lock_file
   end
 
   def with_sitemap(name, options={})
@@ -307,10 +294,6 @@ class BigSitemap
     end
   end
 
-  def strip_leading_slash(str)
-    str.sub(/^\//, '')
-  end
-
   def get_last_id(filename)
     Dir["#{filename}*.{xml,xml.gz}"].map do |file|
       file.to_s.scan(/#{filename}_(.+).xml/).flatten.last.to_i
@@ -328,44 +311,50 @@ class BigSitemap
     method
   end
 
+  def escape_if_string(value)
+    (value.to_i.to_s == value.to_s) ?  value.to_i : "'#{value.gsub("'", %q(\\\'))}'"
+  end
+
   def url_for_sitemap(path)
-    [root_url, @options[:path], File.basename(path)].compact.join('/')
+    File.join @options[:base_url], @options[:url_path], File.basename(path)
   end
 
 end
 
 
-
 class BigSitemapRails < BigSitemap
 
-  include ActionController::UrlWriter if defined? Rails
+  if defined?(Rails) && Rails.version < "3"
+    include ActionController::UrlWriter
+  end
 
   def initialize(options={})
+    raise "No Rails Environment loaded" unless defined? Rails
     require 'action_controller'
 
-    super options.merge(:default_url_options => default_url_options)
-  end
+    if Rails.version >= "3"
+      self.class.send(:include, Rails.application.routes.url_helpers)
+    end
 
-  def document_root
-    "#{Rails.root}/public"
+    DEFAULTS.merge!(:document_root => "#{Rails.root}/public", :url_options => default_url_options)
+    super(options)
   end
-end
 
+end
 
 
 class BigSitemapMerb < BigSitemap
 
   def initialize(options={})
+    raise "No Merb Environment loaded" unless defined? Merb
     require 'extlib'
-    super
-  end
 
-  def document_root
-    "#{Merb.root}/public"
+    DEFAULTS.merge!(:document_root => "#{Merb.root}/public")
+    super(options)
   end
 
   def table_name(model)
     Extlib::Inflection.tableize(model.to_s)
   end
 
-end
+end

data/test/big_sitemap_test.rb

@@ -15,12 +15,20 @@ class BigSitemapTest < Test::Unit::TestCase
     assert_raise(ArgumentError) { BigSitemap.new(:document_root => tmp_dir) }
   end
 
-  should 'generate the same base URL' do
+  should 'generate the same base URL with :base_url option' do
     options = {:document_root => tmp_dir}
-    assert_equal(
-      BigSitemap.new(options.merge(:base_url => 'http://example.com')).root_url,
-      BigSitemap.new(options.merge(:url_options => {:host => 'example.com'})).root_url
-    )
+    url = 'http://example.com'
+    sitemap = BigSitemap.new(options.merge(:base_url => url))
+
+    assert_equal url, sitemap.instance_variable_get(:@options)[:base_url]
+  end
+
+  should 'generate the same base URL with :url_options option' do
+    options = {:document_root => tmp_dir}
+    url = 'http://example.com'
+    sitemap = BigSitemap.new(options.merge(:url_options => {:host => 'example.com'}))
+
+    assert_equal url, sitemap.instance_variable_get(:@options)[:base_url]
   end
 
   should 'generate a sitemap index file' do
@@ -287,6 +295,32 @@ class BigSitemapTest < Test::Unit::TestCase
   end
 
   context 'partial update' do
+
+    context 'prepare_update' do
+      should 'generate correct condition for partial update' do
+        filename = "#{sitemaps_dir}/sitemap_test_models"
+
+        create_sitemap(:partial_update => true).clean
+        add_model(:num_items => 50) #TestModel
+
+        File.open("#{filename}_23.xml", 'w')
+        assert_equal "(id >= 23)", @sitemap.send(:prepare_update).first.last[:conditions]
+
+        File.open("#{filename}_42.xml", 'w')
+        assert_equal "(id >= 23) AND (id >= 42)", @sitemap.send(:prepare_update).first.last[:conditions]
+      end
+
+      should 'generate correct condition for partial update with custom column' do
+        filename = "#{sitemaps_dir}/sitemap_test_models"
+
+        create_sitemap(:partial_update => true).clean
+        add_model(:num_items => 50, :primary_column => 'name') #TestModel
+
+        File.open("#{filename}_666.xml", 'w')
+        assert_equal "(name >= 666)", @sitemap.send(:prepare_update).first.last[:conditions]
+      end
+    end
+
     should 'generate for all xml files in directory and delete last file' do
       TestModel.current_id = last_id = 27
       filename = "#{sitemaps_dir}/sitemap_test_models"
@@ -354,6 +388,24 @@ class BigSitemapTest < Test::Unit::TestCase
       assert_equal 3, elements("#{filename}_46.xml", 'loc').size
     end
 
+    context 'escape' do
+      should 'add if not number' do
+        create_sitemap
+        data = {
+           42 => 42,
+          '23' => 23,
+          "test" => "'test'",
+          "test10" => "'test10'",
+          "10test" => "'10test'",
+          "10t' est" => "'10t\\' est'",
+        }
+        data.each do |key, value|
+          assert_equal value, @sitemap.send(:escape_if_string, key)
+        end
+
+      end
+    end
+
     context 'lockfile' do
       should 'create and delete lock file' do
         sitemap = BigSitemap.new(:base_url => 'http://example.com', :document_root => tmp_dir)

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: big_sitemap
 version: !ruby/object:Gem::Version 
-  hash: 59
+  hash: 57
   prerelease: false
   segments: 
   - 0
   - 8
-  - 2
-  version: 0.8.2
+  - 3
+  version: 0.8.3
 platform: ruby
 authors: 
 - Alex Rabarts
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-01-25 00:00:00 +00:00
+date: 2011-03-14 00:00:00 +00:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency