big_sitemap 1.0.1 → 1.0.2

data/.gitignore

@@ -0,0 +1,3 @@
+.rvmrc
+._*
+pkg

data/Gemfile

@@ -1,14 +1,2 @@
-source :rubygems
-
-gem 'rake'
-gem 'rdoc'
-
-group :development do
-  gem 'jeweler'
-end
-
-group :test do
-  gem 'shoulda'
-  gem 'mocha'
-  gem 'nokogiri'
-end
+# Specify your gem's dependencies in big_sitemap.gemspec
+gemspec

data/Gemfile.lock

@@ -1,27 +1,35 @@
+PATH
+  remote: .
+  specs:
+    big_sitemap (1.0.1)
+
 GEM
-  remote: http://rubygems.org/
   specs:
-    git (1.2.5)
-    jeweler (1.6.4)
-      bundler (~> 1.0)
-      git (>= 1.2.5)
-      rake
-    json (1.6.1)
-    mocha (0.9.10)
-      rake
-    nokogiri (1.4.4)
-    rake (0.8.7)
-    rdoc (3.11)
-      json (~> 1.4)
-    shoulda (2.11.3)
+    activesupport (3.2.3)
+      i18n (~> 0.6)
+      multi_json (~> 1.0)
+    bourne (1.1.2)
+      mocha (= 0.10.5)
+    i18n (0.6.0)
+    metaclass (0.0.1)
+    mocha (0.10.5)
+      metaclass (~> 0.0.1)
+    multi_json (1.3.4)
+    nokogiri (1.5.6)
+    shoulda (3.3.2)
+      shoulda-context (~> 1.0.1)
+      shoulda-matchers (~> 1.4.1)
+    shoulda-context (1.0.2)
+    shoulda-matchers (1.4.2)
+      activesupport (>= 3.0.0)
+      bourne (~> 1.1.2)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  jeweler
+  big_sitemap!
+  bundler
   mocha
   nokogiri
-  rake
-  rdoc
   shoulda

data/History.txt

@@ -1,3 +1,7 @@
+=== 1.0.2 / 2013-02-06
+
+* Fixes an issue with duplicate URLs
+
 === 1.0.1 / 2012-03-10
 
 * Fixes an issue generating correct URL paths

data/Rakefile

@@ -1,19 +1,4 @@
-require 'rake'
-
-begin
-  require 'jeweler'
-  Jeweler::Tasks.new do |s|
-    s.name = "big_sitemap"
-    s.summary = %Q{A Sitemap generator specifically designed for large sites (although it works equally well with small sites)}
-    s.email = %w(alexrabarts@gmail.com tobi@soundcloud.com)
-    s.homepage = "http://github.com/alexrabarts/big_sitemap"
-    s.description = "A Sitemap generator specifically designed for large sites (although it works equally well with small sites)"
-    s.authors = ["Alex Rabarts", "Tobias Bielohlawek"]
-    s.add_dependency 'bundler'
-  end
-rescue LoadError
-  puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
-end
+require 'bundler/gem_tasks'
 
 require 'rdoc/task'
 RDoc::Task.new do |rdoc|

data/VERSION

@@ -0,0 +1 @@
+1.0.2

data/big_sitemap.gemspec

@@ -1,60 +1,20 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
-  s.name = "big_sitemap"
-  s.version = "1.0.1"
-
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Alex Rabarts", "Tobias Bielohlawek"]
-  s.date = "2012-03-10"
-  s.description = "A Sitemap generator specifically designed for large sites (although it works equally well with small sites)"
-  s.email = ["alexrabarts@gmail.com", "tobi@soundcloud.com"]
-  s.extra_rdoc_files = [
-    "LICENSE",
-    "README.rdoc"
-  ]
-  s.files = [
-    "Gemfile",
-    "Gemfile.lock",
-    "History.txt",
-    "LICENSE",
-    "README.rdoc",
-    "Rakefile",
-    "VERSION.yml",
-    "big_sitemap.gemspec",
-    "lib/big_sitemap.rb",
-    "lib/big_sitemap/builder.rb",
-    "test/big_sitemap_test.rb",
-    "test/fixtures/test_model.rb",
-    "test/test_helper.rb"
-  ]
-  s.homepage = "http://github.com/alexrabarts/big_sitemap"
+  s.name          = "big_sitemap"
+  s.version       = File.read('VERSION').strip
+  s.authors       = ["Alex Rabarts", "Tobias Bielohlawek"]
+  s.email         = ["alexrabarts@gmail.com", "tobi@soundcloud.com"]
+  s.homepage      = %q{http://github.com/alexrabarts/big_sitemap}
+  s.summary       = %q{A Sitemap generator specifically designed for large sites (although it works equally well with small sites)}
+  s.description   = %q{BigSitemap is a Sitemapgenerator suitable for applications with greater than 50,000 URLs. It splits large Sitemaps into multiple files, gzips the files to minimize bandwidth usage, batches database queries to minimize memory usage, supports increment updates, can be set up with just a few lines of code and is compatible with just about any framework.}
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.10"
-  s.summary = "A Sitemap generator specifically designed for large sites (although it works equally well with small sites)"
-
-  if s.respond_to? :specification_version then
-    s.specification_version = 3
 
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<rake>, [">= 0"])
-      s.add_runtime_dependency(%q<rdoc>, [">= 0"])
-      s.add_development_dependency(%q<jeweler>, [">= 0"])
-      s.add_runtime_dependency(%q<bundler>, [">= 0"])
-    else
-      s.add_dependency(%q<rake>, [">= 0"])
-      s.add_dependency(%q<rdoc>, [">= 0"])
-      s.add_dependency(%q<jeweler>, [">= 0"])
-      s.add_dependency(%q<bundler>, [">= 0"])
-    end
-  else
-    s.add_dependency(%q<rake>, [">= 0"])
-    s.add_dependency(%q<rdoc>, [">= 0"])
-    s.add_dependency(%q<jeweler>, [">= 0"])
-    s.add_dependency(%q<bundler>, [">= 0"])
+  ["bundler", "shoulda", "mocha", "nokogiri"].each do |gem|
+    s.add_development_dependency *gem.split(' ')
   end
 end
-

data/lib/big_sitemap.rb

@@ -178,6 +178,7 @@ class BigSitemap
     return self if Array(@paths).empty?
 
     with_sitemap do |builder|
+      @paths.uniq!
       @paths.each do |path, options|
         url = URI.join(@options[:base_url], path)
         builder.add_url! url, options
@@ -260,6 +261,9 @@ class BigSitemap
           find_options[key] = options.delete(key)
         end
 
+        # Keep the intial conditions for later user
+        conditions = find_options[:conditions]
+
         primary_method   = options.delete(:primary_column)
         primary_column   = "#{table_name(model)}.#{primary_method}"
 
@@ -287,7 +291,7 @@ class BigSitemap
             if last_id && primary_column
               find_options.update(:limit => limit, :offset => nil)
               primary_column_value = escape_if_string last_id #escape '
-              find_options.update(:conditions => [find_options[:conditions], "(#{primary_column} > #{primary_column_value})"].compact.join(' AND '))
+              find_options[:conditions] = [conditions, "(#{primary_column} > #{primary_column_value})"].compact.join(' AND ')
             end
 
             model.send(find_method, find_options).each do |record|
@@ -378,9 +382,6 @@ class BigSitemap
 
     sitemap = if options[:type] == 'index'
       IndexBuilder.new(options)
-    elsif options[:geo]
-      options[:filename] << '_kml'
-      GeoBuilder.new(options)
     else
       Builder.new(options)
     end

data/lib/big_sitemap/builder.rb

@@ -4,10 +4,11 @@ require 'zlib'
 class BigSitemap
   class Builder
     MAX_URLS = 50000
+    HEADER_NAME = 'urlset'
     HEADER_ATTRIBUTES = {
       'xmlns' => 'http://www.sitemaps.org/schemas/sitemap/0.9',
       'xmlns:xsi' => "http://www.w3.org/2001/XMLSchema-instance",
-      'xsi:schemaLocation' => "http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd" 
+      'xsi:schemaLocation' => "http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"
     }
 
     def initialize(options)
@@ -75,11 +76,11 @@ class BigSitemap
       @gzip ? ::Zlib::GzipWriter.new(file) : file
     end
 
-    def _init_document(name='urlset', attrs=HEADER_ATTRIBUTES)
+    def _init_document
       @urls = 0
       target!.print '<?xml version="1.0" encoding="UTF-8"?>'
       _newline
-      _open_tag name, attrs
+      _open_tag self.class::HEADER_NAME, self.class::HEADER_ATTRIBUTES
     end
 
     def _rotate(part_nr=nil)
@@ -141,10 +142,10 @@ class BigSitemap
   end
 
   class IndexBuilder < Builder
-    def _init_document(name = 'sitemapindex', attrs = {'xmlns' => 'http://www.sitemaps.org/schemas/sitemap/0.9'})
-      attrs.merge('xmlns:geo' => "http://www.google.com/geo/schemas/sitemap/1.0")
-      super(name, attrs)
-    end
+    HEADER_NAME = 'sitemapindex'
+    HEADER_ATTRIBUTES = {
+      'xmlns' => 'http://www.sitemaps.org/schemas/sitemap/0.9'
+    }
 
     def add_url!(location, options={})
       _open_tag 'sitemap'
@@ -156,14 +157,5 @@ class BigSitemap
     end
   end
 
-  class GeoBuilder < Builder
-    #_build_geo if @geo
-
-    # def _build_geo
-    #   geo :geo do
-    #     geo :format, 'kml'
-    #   end
-    # end
-  end
 
 end

data/test/big_sitemap_test.rb

@@ -75,7 +75,7 @@ class BigSitemapTest < Test::Unit::TestCase
 
     should 'contain two loc elements' do
       generate_sitemap(:max_per_sitemap => 2) do
-        4.times { add '/' }
+        4.times { |i| add "/#{i}" }
       end
 
       assert_equal 2, num_elements(sitemaps_index_file, 'loc')
@@ -83,7 +83,7 @@ class BigSitemapTest < Test::Unit::TestCase
 
     should 'contain two lastmod elements' do
       generate_sitemap(:max_per_sitemap => 2) do
-        4.times { add '/' }
+        4.times { |i| add "/#{i}" }
       end
 
       assert_equal 2, num_elements(sitemaps_index_file, 'lastmod')
@@ -103,7 +103,7 @@ class BigSitemapTest < Test::Unit::TestCase
 
     should 'contain several loc elements' do
       generate_sitemap do
-        3.times { add '/' }
+        3.times { |i| add "/#{i}" }
       end
 
       assert_equal 3, num_elements(first_sitemap_file, 'loc')
@@ -111,7 +111,7 @@ class BigSitemapTest < Test::Unit::TestCase
 
     should 'contain several lastmod elements' do
       generate_sitemap do
-        3.times { add '/', :last_modified => Time.now }
+        3.times { |i| add "/#{i}", :last_modified => Time.now }
       end
 
       assert_equal 3, num_elements(first_sitemap_file, 'lastmod')
@@ -119,7 +119,7 @@ class BigSitemapTest < Test::Unit::TestCase
 
     should 'contain several changefreq elements' do
       generate_sitemap do
-        3.times { add '/' }
+        3.times { |i| add "/#{i}" }
       end
 
       assert_equal 3, num_elements(first_sitemap_file, 'changefreq')
@@ -127,7 +127,7 @@ class BigSitemapTest < Test::Unit::TestCase
 
     should 'contain several priority elements' do
       generate_sitemap do
-        3.times { add '/', :priority => 0.2 }
+        3.times { |i| add "/#{i}", :priority => 0.2 }
       end
 
       assert_equal 3, num_elements(first_sitemap_file, 'priority')
@@ -153,7 +153,7 @@ class BigSitemapTest < Test::Unit::TestCase
 
     should 'contain two loc element' do
       generate_sitemap(:max_per_sitemap => 2) do
-        4.times { add '/' }
+        4.times { |i| add "/#{i}" }
       end
 
       assert_equal 2, num_elements(first_sitemap_file, 'loc')
@@ -162,7 +162,7 @@ class BigSitemapTest < Test::Unit::TestCase
 
     should 'contain two changefreq elements' do
       generate_sitemap(:max_per_sitemap => 2) do
-        4.times { add '/' }
+        4.times { |i| add "/#{i}" }
       end
 
       assert_equal 2, num_elements(first_sitemap_file, 'changefreq')
@@ -171,7 +171,7 @@ class BigSitemapTest < Test::Unit::TestCase
 
     should 'contain two priority element' do
       generate_sitemap(:max_per_sitemap => 2) do
-        4.times { add '/', :priority => 0.2 }
+        4.times { |i| add "/#{i}", :priority => 0.2 }
       end
 
       assert_equal 2, num_elements(first_sitemap_file, 'priority')
@@ -182,6 +182,14 @@ class BigSitemapTest < Test::Unit::TestCase
       generate_sitemap(:gzip => false) { add '/' }
       assert File.exists?(unzipped_first_sitemap_file)
     end
+
+    should 'contain unique elements' do
+      generate_sitemap do
+        2.times { add '/' }
+      end
+
+      assert_equal 1, num_elements(first_sitemap_file, 'url')
+    end
   end
 
   context 'sanatize XML chars' do
@@ -336,60 +344,4 @@ class BigSitemapTest < Test::Unit::TestCase
     end
   end
 
-  private
-    def generate_sitemap(options={}, &block)
-      BigSitemap.generate(options.merge(:base_url => 'http://example.com', :document_root => tmp_dir), &block)
-    end
-
-    def delete_tmp_files
-      Dir["#{sitemaps_dir}/sitemap*"].each do |f|
-        FileUtils.rm_rf f
-      end
-    end
-
-    def sitemaps_index_file
-      "#{unzipped_sitemaps_index_file}.gz"
-    end
-
-    def unzipped_sitemaps_index_file
-      "#{sitemaps_dir}/sitemap_index.xml"
-    end
-
-    def unzipped_first_sitemap_file
-      "#{sitemaps_dir}/sitemap.xml"
-    end
-
-    def first_sitemap_file
-      "#{sitemaps_dir}/sitemap.xml.gz"
-    end
-
-    def second_sitemap_file
-      "#{sitemaps_dir}/sitemap_1.xml.gz"
-    end
-
-    def third_sitemap_file
-      "#{sitemaps_dir}/sitemap_2.xml.gz"
-    end
-
-    def sitemaps_dir
-      tmp_dir
-    end
-
-    def tmp_dir
-      '/tmp'
-    end
-
-    def ns
-      {'s' => 'http://www.sitemaps.org/schemas/sitemap/0.9'}
-    end
-
-    def elements(filename, el)
-      file_class = filename.include?('.gz') ? Zlib::GzipReader : File
-      data = Nokogiri::XML.parse(file_class.open(filename).read)
-      data.search("//s:#{el}", ns)
-    end
-
-    def num_elements(filename, el)
-      elements(filename, el).size
-    end
 end

data/test/test_helper.rb

@@ -9,4 +9,62 @@ require 'test/fixtures/test_model'
 require 'big_sitemap'
 
 class Test::Unit::TestCase
+
+  #TestHelper
+  private
+  def generate_sitemap(options={}, &block)
+    BigSitemap.generate(options.merge(:base_url => 'http://example.com', :document_root => tmp_dir), &block)
+  end
+
+  def delete_tmp_files
+    Dir["#{sitemaps_dir}/sitemap*"].each do |f|
+      FileUtils.rm_rf f
+    end
+  end
+
+  def sitemaps_index_file
+    "#{unzipped_sitemaps_index_file}.gz"
+  end
+
+  def unzipped_sitemaps_index_file
+    "#{sitemaps_dir}/sitemap_index.xml"
+  end
+
+  def unzipped_first_sitemap_file
+    "#{sitemaps_dir}/sitemap.xml"
+  end
+
+  def first_sitemap_file
+    "#{sitemaps_dir}/sitemap.xml.gz"
+  end
+
+  def second_sitemap_file
+    "#{sitemaps_dir}/sitemap_1.xml.gz"
+  end
+
+  def third_sitemap_file
+    "#{sitemaps_dir}/sitemap_2.xml.gz"
+  end
+
+  def sitemaps_dir
+    tmp_dir
+  end
+
+  def tmp_dir
+    '/tmp'
+  end
+
+  def ns
+    {'s' => 'http://www.sitemaps.org/schemas/sitemap/0.9'}
+  end
+
+  def elements(filename, el)
+    file_class = filename.include?('.gz') ? Zlib::GzipReader : File
+    data = Nokogiri::XML.parse(file_class.open(filename).read)
+    data.search("//s:#{el}", ns)
+  end
+
+  def num_elements(filename, el)
+    elements(filename, el).size
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: big_sitemap
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -10,33 +10,33 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-10 00:00:00.000000000Z
+date: 2013-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: rake
-  requirement: &2156893720 !ruby/object:Gem::Requirement
+  name: bundler
+  requirement: &70198116635740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
-  version_requirements: *2156893720
+  version_requirements: *70198116635740
 - !ruby/object:Gem::Dependency
-  name: rdoc
-  requirement: &2156893240 !ruby/object:Gem::Requirement
+  name: shoulda
+  requirement: &70198116635220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
-  version_requirements: *2156893240
+  version_requirements: *70198116635220
 - !ruby/object:Gem::Dependency
-  name: jeweler
-  requirement: &2156892760 !ruby/object:Gem::Requirement
+  name: mocha
+  requirement: &70198116634760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -44,36 +44,38 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2156892760
+  version_requirements: *70198116634760
 - !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: &2156892280 !ruby/object:Gem::Requirement
+  name: nokogiri
+  requirement: &70198116634240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
-  version_requirements: *2156892280
-description: A Sitemap generator specifically designed for large sites (although it
-  works equally well with small sites)
+  version_requirements: *70198116634240
+description: BigSitemap is a Sitemapgenerator suitable for applications with greater
+  than 50,000 URLs. It splits large Sitemaps into multiple files, gzips the files
+  to minimize bandwidth usage, batches database queries to minimize memory usage,
+  supports increment updates, can be set up with just a few lines of code and is compatible
+  with just about any framework.
 email:
 - alexrabarts@gmail.com
 - tobi@soundcloud.com
 executables: []
 extensions: []
-extra_rdoc_files:
-- LICENSE
-- README.rdoc
+extra_rdoc_files: []
 files:
+- .gitignore
 - Gemfile
 - Gemfile.lock
 - History.txt
 - LICENSE
 - README.rdoc
 - Rakefile
-- VERSION.yml
+- VERSION
 - big_sitemap.gemspec
 - lib/big_sitemap.rb
 - lib/big_sitemap/builder.rb
@@ -105,4 +107,7 @@ signing_key:
 specification_version: 3
 summary: A Sitemap generator specifically designed for large sites (although it works
   equally well with small sites)
-test_files: []
+test_files:
+- test/big_sitemap_test.rb
+- test/fixtures/test_model.rb
+- test/test_helper.rb

data/VERSION.yml

@@ -1,5 +0,0 @@
----
-:major: 1
-:minor: 0
-:patch: 1
-:build: !!null