static_sitemap_tasks 0.1

data/.gitignore

@@ -0,0 +1,2 @@
+pkg/
+Gemfile.lock

data/Gemfile

@@ -0,0 +1,6 @@
+source :rubygems
+
+gem "bundler", ">= 1.0"
+gem "hpricot"
+gem "builder"
+gem "rdoc"

data/MIT-LICENSE

@@ -0,0 +1,22 @@
+Originally Authored by Chris Martin on 2007-04-30 (http://chriscodes.com/articles/view/54)
+Updated by Tom Cocca on 2008-10-10.
+Updated by Michael Leinartas on 2011-10-12
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,59 @@
+== Generate Sitemap Plugin
+
+Originally Authored by Chris Marting (http://chriscodes.com/articles/view/54)
+Updates by Tom Cocca
+Updates include:
+	- Adding a YAML config file to set URL and Change Frequency
+
+	
+== Generate Sitemap Rake Task
+
+This is a Rails plugin that consists of a rake task to generate a sitemap.xml file.
+The task crawls a domain (specified in a YAML config per environment) for all it's url's, 
+then builds the sitemap.xml file in public.
+
+Since this simply crawls a domain for url's, this could be used to generate 
+sitemaps for any site, not just a Rails application.
+
+
+== Requirements
+
+Both of these are available via RubyGems
+Hpricot - http://code.whytheluckystiff.net/hpricot/
+Builder - http://rubyforge.org/projects/builder/
+
+
+== Configuration/Usage
+
+On initialization generate_sitemap will copy the generate_sitemap_key.yml file you your
+RAILS_ROOT/config directory.
+
+Open this file and specify your domain for the different environments and specify the changefreq for
+your site.
+
+For changefreq options see this page: http://www.sitemaps.org/protocol.php
+I haven't devised a way to set this dynamically, as most pages will vary.
+
+The plugin will pull the correct domain name for crawling the site by using the RAILS_ENV variable
+in the rake task.
+
+Run the task with the command
+	rake plugin:generate_sitemap
+	
+Go to http://localhost:3000/sitemap.xml (or open public/sitemap.xml)
+
+
+== TODO
+
+ - set changefreq, lastmod, priority dynamically during generation
+ - allow generation of sitemap index files
+ - write tests
+ - allow for exclusions to be specified in an array
+
+
+== More Info
+
+http://www.sitemaps.org/protocol.php
+
+Questions, comments, patches, etc. can be sent to 
+	tom dot cocca at gmail dot com

data/Rakefile

@@ -0,0 +1,7 @@
+require 'rubygems'
+require 'bundler'
+require 'rake'
+
+Bundler::GemHelper.install_tasks
+
+task :default => :build

data/lib/static_sitemap_tasks.rb

@@ -0,0 +1,150 @@
+require 'rubygems'
+require 'builder'
+require 'hpricot'
+require 'uri'
+
+
+module SitemapGenerator
+  class Tasks
+    include Rake::DSL
+
+    def self.install(options = {})
+      dir = options.delete(:dir) || Dir.pwd
+      self.new(options).install
+    end
+
+    def initialize(options = {})
+      # Root of files to crawl
+      @public_root = options[:public_root] || Dir.pwd
+      # Change frequency - see: http://www.sitemaps.org/protocol.php#changefreqdef
+      @change_frequency = options[:change_frequency]
+      # Canonical domain of published site
+      @base_url = options[:base_url]
+      # Index pages
+      @index_files = options[:index_files] || [ 'index.html', 'index.htm' ]
+      # Compress output to sitemap.xml.gz
+      @gzip_output = options[:gzip_output] || true
+    end
+
+    def install
+      desc "Generate a sitemap based on the contents of #{@public_root}"
+      task 'generate_sitemap' do
+        generate_sitemap
+      end
+    end
+
+    def generate_sitemap
+      # holds pages to go into map, and pages crawled
+      @pages = []
+      @pages_crawled = []
+
+      # start with index pages
+      crawl_for_links('/')
+
+      # crawl each page in pages array unless it's already been crawled
+      @pages.each {|page|
+        crawl_for_links(page) unless @pages_crawled.include?(page)
+      }
+
+      # create xml for sitemap
+      xml = Builder::XmlMarkup.new( :indent => 2 )
+      xml.instruct!
+      xml.comment! "Generated on: " + Time.now.to_s
+      xml.urlset("xmlns" => "http://www.sitemaps.org/schemas/sitemap/0.9") {
+        # loop through array of pages, and build sitemap.xml
+        @pages.sort.each {|link|
+          xml.url {
+            xml.loc URI.join(@base_url, link)
+            # TODO - set changefreq dynamically per page
+            xml.changefreq @change_frequency unless @change_frequency.nil?
+          }
+        }
+      }
+
+      # convert builder xml to xml string, and save
+      xml_string = xml.to_s.gsub("<to_s/>","")
+      filename = File.join(@public_root,'sitemap.xml')
+
+      if @gzip_output
+        require 'zlib'
+        filename << '.gz'
+        xml_file = Zlib::GzipWriter.open(filename)
+      else
+        xml_file = File.open(filename, 'w')
+      end
+
+      xml_file << xml_string
+      xml_file.close
+    end
+
+    # uses Hpricot to grab links from a URI
+    # adds uri to @pages_crawled
+    # loops each link found
+    # adds link to pages array if it should be included, unless it already exists
+    def crawl_for_links(link_path)
+      if link_path.include?('http')
+        return unless link_path.include?(@base_url)
+        link_path.sub!(@base_url,'')
+      end
+      file_path = resolve_file_path(File.join(@public_root, link_path))
+
+      if file_path.nil?
+        puts "Warning: Unable to resolve #{link_path} to a local file"
+        return
+      end
+
+      puts "Inspecting #{file_path}...\n"
+      doc = Hpricot(open(file_path)) rescue nil
+      return unless doc
+      @pages_crawled << link_path
+      (doc/"a").each do |a|
+        if a['href'] && should_be_included?(a['href'])
+          @pages << a['href'] unless(link_exists?(a['href'],@pages))
+        end
+      end
+    end
+
+    def resolve_file_path(path)
+      file_path = nil
+
+      if File.directory?(path)
+        @index_files.each do |f|
+          index_file = File.join(path,f)
+          if File.exists?(index_file)
+            file_path = index_file
+            break
+          end
+        end
+      else
+        if File.exists?(path)
+          file_path = path
+        end
+      end
+
+      file_path
+    end
+
+    # returns true if any of the following are true:
+    # - link isn't external (eg, contains 'http://') and doesn't contain 'mailto:'
+    # - is equal to '/'
+    # - link contains @base_url
+    def should_be_included?(str)
+      if ((!str.include?('http://') && !str.include?('mailto:')) || str == '/' || str.include?(@base_url))
+        unless str.slice(0,1) == "#"
+          return true
+        end
+      end
+    end
+
+    # checks each value in a given array for the given string
+    # removes '/' character before comparison
+    def link_exists?(str, array)
+      array.detect{|l| strip_slashes(l) == strip_slashes(str)}
+    end
+
+    # removes '/' character from string
+    def strip_slashes(str)
+      str.gsub('/','')
+    end
+  end
+end

data/static_sitemap_tasks.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "static_sitemap_tasks"
+  s.version     = "0.1"
+  s.platform    = Gem::Platform::RUBY
+  s.summary     = 'Rake tasks to manage sitemap.xml generation for static sites'
+  s.description = 'Rake tasks to manage sitemap.xml generation for static sites'
+
+  s.required_ruby_version     = ">= 1.8.7"
+  s.required_rubygems_version = ">= 1.3.6"
+
+  s.authors     = ["Michael Leinartas", "Tim Cocca", "Chris Martin"]
+  s.email       = ["mleinartas@gmail.com"]
+  s.homepage    = "https://github.com/mleinart/static_sitemap_tasks"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_path  = 'lib'
+
+  s.add_development_dependency "rake", ">= 0.8.7"
+  s.add_development_dependency "bundler", ">= 1.0"
+end

metadata

@@ -0,0 +1,76 @@
+--- !ruby/object:Gem::Specification
+name: static_sitemap_tasks
+version: !ruby/object:Gem::Version
+  version: '0.1'
+  prerelease: 
+platform: ruby
+authors:
+- Michael Leinartas
+- Tim Cocca
+- Chris Martin
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-10-12 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &70212813523740 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.8.7
+  type: :development
+  prerelease: false
+  version_requirements: *70212813523740
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: &70212813523280 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: *70212813523280
+description: Rake tasks to manage sitemap.xml generation for static sites
+email:
+- mleinartas@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- MIT-LICENSE
+- README.md
+- Rakefile
+- lib/static_sitemap_tasks.rb
+- static_sitemap_tasks.gemspec
+homepage: https://github.com/mleinart/static_sitemap_tasks
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.8.7
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.3.6
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: Rake tasks to manage sitemap.xml generation for static sites
+test_files: []