dynamic_sitemaps 1.0.8 → 2.0.0.beta

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5df17dde12a20c3a3cca9273c46aa26834d93c47
+  data.tar.gz: e026ef0473a20a0e0bafe8551d846983a23c2f49
+SHA512:
+  metadata.gz: 12da553942f24fdcbefe729b747e32389d2eeed1f14b9bb3363fbbf0ac58c0a502e9630d949bbd85e8db74bac2e837cd156ce9b2aa9cfb3df0135fe656d41be0
+  data.tar.gz: b52815f7ae238b587867cf3a288f2ea44df35bfe98d38c2f4759fc4f61fb0305a85b6c2cd4c387d7d752ed4a16920b9c3d98e42b29b45c8eaac8a354f11a5af0

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.DS_Store

data/Gemfile

@@ -0,0 +1,17 @@
+source "http://rubygems.org"
+
+# Declare your gem's dependencies in dynamic_sitemaps.gemspec.
+# Bundler will treat runtime dependencies like base dependencies, and
+# development dependencies will be added by default to the :development group.
+gemspec
+
+# jquery-rails is used by the dummy application
+gem "jquery-rails"
+
+# Declare any dependencies that are still in development here instead of in
+# your gemspec. These might include edge Rails or gems from your path or
+# Git. Remember to move these dependencies to your gemspec before releasing
+# your gem to rubygems.org.
+
+# To use debugger
+# gem 'debugger'

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2010-2013 Lasse Bunk
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,106 @@
+# DynamicSitemaps
+
+Dynamic Sitemaps is a plugin for Ruby on Rails that enables you to easily create flexible, dynamic sitemaps. It creates sitemaps in the [sitemaps.org](http://sitemaps.org) standard which is supported by several crawlers including Google, Bing, and Yahoo.
+
+Dynamic Sitemaps is designed to be very (very) simple so there's a lot you cannot do, but possibly don't need (I didn't). If you need an advanced sitemap generator, please see Karl Varga's [SitemapGenerator](https://github.com/kjvarga/sitemap_generator).
+
+## Planned for version 2.0 (this branch)
+
+Version 2.0 will make it possible to make very large sitemaps (up to 2.5 billion URLs) in a fast and memory efficient way; it will be built for large amounts of data, i.e. millions of URLs without pushing your server to the limit, memory and CPU wise.
+
+Version 2.0 will not be compatible with version 1.0 as version 2.0 will generate static sitemap XML files whereas 1.0 generates them dynamically on each request.
+
+Idea for the version 2.0 DSL, in ```config/sitemap.rb```:
+
+```ruby
+host "www.mysite.com"
+
+sitemap :site do
+  url root_url, last_mod: Time.now, change_freq: "daily", priority: 1.0
+  url contact_url
+  Page.all.each do |page|
+    url page, last_mod: page.updated_at
+  end
+end
+
+# All products and editions URLs
+sitemap :products, Product do |product|
+  url product, last_mod: product.updated_at
+  url product_editions_url(product)
+end
+
+# Autogenerate a tags sitemap with URLs for all tags containing products.
+# Automatically sets last_mod to tag.updated_at.
+sitemap Tag.where("products_count > 0")
+```
+
+You then run:
+
+```bash
+$ rake sitemap:generate
+```
+
+This will generate the sitemaps, remove all files from ```public/sitemaps``` replacing them with:
+
+```bash
+- public/sitemaps/index.xml
+- public/sitemaps/site.xml
+- public/sitemaps/products.xml
+- public/sitemaps/products2.xml
+- public/sitemaps/products3.xml # etc.
+- public/sitemaps/tags.xml
+- public/sitemaps/tags2.xml # etc.
+```
+
+And symlink ```/sitemap.xml``` to ```/sitemaps/sitemap.xml``` because the sitemaps.org spec only allows URLs in a sitemap to be *below* the ```sitemap.xml``` index file:
+
+```bash
+$ ln -s /var/www/mysite/public/sitemaps/index.xml /var/www/mysite/public/sitemap.xml
+```
+
+If you use Capistrano and have a ```shared``` folder:
+
+```bash
+$ mkdir -p /var/www/mysite/shared/sitemaps
+$ rm -r /var/www/mysite/current/public/sitemaps # if uploaded accidentally
+$ ln -s /var/www/mysite/shared/sitemaps /var/www/mysite/current/public/sitemaps
+```
+
+If for example you have multiple subdomains, then in ```config/sitemap.rb```:
+
+```ruby
+Site.all.each do |site|
+  host "#{site.subdomain}.mysite.com"
+  path Rails.root.join("public", "sitemaps", site.subdomain)
+
+  url root_url
+  sitemap site.products
+  # etc.
+end
+```
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem "dynamic_sitemaps", git: "git://github.com/lassebunk/dynamic_sitemaps.git", branch: "2.0"
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install dynamic_sitemaps
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,10 @@
+require 'rake/testtask'
+require 'bundler/gem_tasks'
+
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList['test/*_test.rb']
+  t.verbose = true
+end
+
+task :default => :test

data/dynamic_sitemaps.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'dynamic_sitemaps/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "dynamic_sitemaps"
+  gem.version       = DynamicSitemaps::VERSION
+  gem.authors       = ["Lasse Bunk"]
+  gem.email         = ["lassebunk@gmail.com"]
+  gem.description   = %q{Dynamic Sitemaps is a plugin for Ruby on Rails that enables you to easily create flexible, dynamic sitemaps for Google, Bing, and Yahoo.}
+  gem.summary       = %q{Dynamic sitemap generation plugin for Ruby on Rails.}
+  gem.homepage      = "http://github.com/lassebunk/dynamic_sitemaps"
+  gem.license       = "MIT"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.test_files    = gem.files.grep(%r{^test/})
+  gem.require_paths = ["lib"]
+
+  gem.add_development_dependency "rails", "~> 3.2.13"
+  gem.add_development_dependency "sqlite3"
+end

data/lib/dynamic_sitemaps.rb

@@ -1,4 +1,89 @@
-require 'dynamic_sitemaps/page'
-require 'dynamic_sitemaps/sitemap'
-require 'dynamic_sitemaps/url'
-require 'dynamic_sitemaps/engine'
+require "dynamic_sitemaps/rails/engine"
+require "dynamic_sitemaps/sitemap"
+require "dynamic_sitemaps/generator"
+require "dynamic_sitemaps/sitemap_generator"
+require "dynamic_sitemaps/index_generator"
+require "dynamic_sitemaps/sitemap_result"
+require "dynamic_sitemaps/pinger"
+
+module DynamicSitemaps
+  DEFAULT_PER_PAGE = 50000
+  DEFAULT_FOLDER = "sitemaps"
+  DEFAULT_INDEX_FILE_NAME = "sitemap.xml"
+  DEFAULT_ALWAYS_GENERATE_INDEX = false
+  SEARCH_ENGINE_PING_URLS = [
+    "http://www.google.com/webmasters/sitemaps/ping?sitemap=%s",
+    "http://www.bing.com/webmaster/ping.aspx?siteMap=%s"
+  ]
+
+  class << self
+    attr_writer :path, :folder, :index_file_name, :always_generate_index, :config_path, :search_engine_ping_urls
+
+    def generate_sitemap
+      DynamicSitemaps::Generator.generate
+    end
+
+    # Configure DynamicSitemaps.
+    # Defaults:
+    # 
+    #   DynamicSitemaps.configure do |config|
+    #     config.path = Rails.root.join("public")
+    #     config.folder = "sitemaps"
+    #     config.index_file_name = "sitemap.xml"
+    #     config.always_generate_index = false
+    #     config.config_path = Rails.root.join("config", "sitemap.rb")
+    #   end
+    # 
+    # To ping search engines after generating the sitemap:
+    # 
+    #   DynamicSitemaps.configure do |config|
+    #     config.search_engine_ping_urls << "http://customsearchengine.com/ping?url=%s" # Default is Google and Bing
+    #     config.sitemap_ping_urls = ["http://www.domain.com/sitemap.xml"]
+    #     # or dynamically:
+    #     config.sitemap_ping_urls = -> { Site.all.map { |site| "http://#{site.domain}/sitemap.xml" } }
+    #   end
+    def configure
+      yield self
+    end
+
+    def folder
+      @folder ||= DEFAULT_FOLDER
+    end
+
+    def path
+      @path ||= Rails.root.join("public")
+    end
+
+    def index_file_name
+      @index_file_name ||= DEFAULT_INDEX_FILE_NAME
+    end
+
+    def always_generate_index
+      return @always_generate_index if instance_variable_defined?(:@always_generate_index)
+      @always_generate_index = DEFAULT_ALWAYS_GENERATE_INDEX
+    end
+
+    def config_path
+      @config_path ||= Rails.root.join("config", "sitemap.rb")
+    end
+
+    def search_engine_ping_urls
+      @search_engine_ping_urls ||= SEARCH_ENGINE_PING_URLS
+    end
+
+    def sitemap_ping_urls
+      case @sitemap_ping_urls
+      when Array then @sitemap_ping_urls
+      when Proc then @sitemap_ping_urls.call
+      else []
+      end
+    end
+
+    def sitemap_ping_urls=(array_or_proc)
+      unless array_or_proc.is_a?(Array) || array_or_proc.is_a?(Proc)
+        raise "Unknown type #{array_or_proc.class.name} for sitemap_ping_urls."
+      end
+      @sitemap_ping_urls = array_or_proc
+    end
+  end
+end

data/lib/dynamic_sitemaps/generator.rb

@@ -0,0 +1,59 @@
+module DynamicSitemaps
+  class Generator
+    class << self
+      def generate
+        instance_eval open(DynamicSitemaps.config_path).read
+        generate_index
+      end
+
+      def generate_index
+        IndexGenerator.new(sitemaps).generate
+      end
+
+      def sitemap(*args, &block)
+        args << {} unless args.last.is_a?(Hash)
+        args.last[:host] ||= host
+        args.last[:folder] ||= folder
+        sitemap = Sitemap.new(*args, &block)
+        sitemaps << SitemapGenerator.new(sitemap).generate
+      end
+
+      def sitemap_for(collection, options = {}, &block)
+        raise "The collection given to `sitemap_for` must respond to #find_each. This is for performance. Use `Model.scoped` to get an ActiveRecord relation that responds to #find_each." unless collection.respond_to?(:find_each)
+
+        name = options.delete(:name) || collection.model_name.underscore.pluralize.to_sym
+        options[:collection] = collection
+
+        sitemap(name, options, &block)
+      end
+
+      # Array of SitemapResult
+      def sitemaps
+        @sitemaps ||= []
+      end
+
+      def host(*args)
+        if args.any?
+          @host = args.first
+          Rails.application.routes.default_url_options[:host] = @host
+        else
+          @host
+        end
+      end
+
+      def folder(*args)
+        if args.any?
+          @folder = args.first
+          raise ArgumentError, "Folder can't be blank." if @folder.blank?
+
+          FileUtils.rm_rf Dir.glob("#{DynamicSitemaps.path}/#{folder}/*")
+        else
+          # Ensure that the default folder is set and cleaned.
+          folder DynamicSitemaps.folder if @folder.blank?
+
+          @folder
+        end
+      end
+    end
+  end
+end

data/lib/dynamic_sitemaps/index_generator.rb

@@ -0,0 +1,47 @@
+module DynamicSitemaps
+  class IndexGenerator
+    attr_reader :sitemaps # Array of sitemap results
+
+    # Initialize the class with an array of SitemapResult
+    def initialize(sitemaps)
+      @sitemaps = sitemaps
+    end
+
+    def generate
+      sitemaps.group_by(&:folder).each do |folder, sitemaps|
+        index_path = "#{DynamicSitemaps.path}/#{folder}/#{DynamicSitemaps.index_file_name}"
+
+        if !DynamicSitemaps.always_generate_index && sitemaps.count == 1 && sitemaps.first.files.count == 1
+          file_path = "#{DynamicSitemaps.path}/#{folder}/#{sitemaps.first.files.first}"
+          FileUtils.copy file_path, index_path
+          File.delete file_path
+        else
+          File.open(index_path, "w") do |file|
+            write_beginning(file)
+            write_sitemaps(file, sitemaps)
+            write_end(file)
+          end
+        end
+      end
+    end
+
+    def write_beginning(file)
+      file.puts '<?xml version="1.0" encoding="UTF-8"?>',
+                '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
+    end
+
+    def write_sitemaps(file, sitemaps)
+      sitemaps.each do |sitemap|
+        sitemap.files.each do |file_name|
+          file.puts '<sitemap>',
+                    "<loc>http://#{sitemap.host}/#{sitemap.folder}/#{file_name}</loc>",
+                    '</sitemap>'
+        end
+      end
+    end
+
+    def write_end(file)
+      file.puts '</sitemapindex>'
+    end
+  end
+end

data/lib/dynamic_sitemaps/pinger.rb

@@ -0,0 +1,35 @@
+module DynamicSitemaps
+  class Pinger
+    class << self
+      def ping_search_engines
+        sitemap_urls = DynamicSitemaps.sitemap_ping_urls
+        if sitemap_urls.any?
+          puts "Pinging search engines..."
+
+          sitemap_urls.each do |url|
+            ping_search_engines_with_sitemap_url url
+          end
+
+          puts "Done pinging search engines."
+        end
+      end
+
+      def ping_search_engines_with_sitemap_url(sitemap_url)
+        sitemap_url = CGI::escape(sitemap_url)
+        DynamicSitemaps.search_engine_ping_urls.each do |ping_url|
+          url = ping_url % sitemap_url
+          ping url
+        end
+      end
+
+      def ping(url)
+        puts "Pinging #{url} ..."
+        begin
+          Net::HTTP.get(URI.parse(url))
+        rescue Exception => e
+          puts "Failed to ping #{url} : #{e}"
+        end
+      end
+    end
+  end
+end