static_sitemap_tasks 0.1 → 0.2

data/ChangeLog

@@ -0,0 +1,9 @@
+v0.2
+ * Namespace tasks with :sitemap
+ * Add ping task from http://github.com/adamsalter/sitemap_generator
+ * Add lastmod tags ability
+
+v0.1
+ * Initial fork from http://github.com/tcocca/generate_sitemap
+ * Refactor to scan local directory
+ * Refactor to set config items on rake task install

data/README.md

@@ -1,59 +1,35 @@
-== Generate Sitemap Plugin
+## Generate Sitemap Rake Task ##
 
-Originally Authored by Chris Marting (http://chriscodes.com/articles/view/54)
-Updates by Tom Cocca
-Updates include:
-	- Adding a YAML config file to set URL and Change Frequency
-
-	
-== Generate Sitemap Rake Task
-
-This is a Rails plugin that consists of a rake task to generate a sitemap.xml file.
-The task crawls a domain (specified in a YAML config per environment) for all it's url's, 
-then builds the sitemap.xml file in public.
-
-Since this simply crawls a domain for url's, this could be used to generate 
-sitemaps for any site, not just a Rails application.
-
-
-== Requirements
-
-Both of these are available via RubyGems
-Hpricot - http://code.whytheluckystiff.net/hpricot/
-Builder - http://rubyforge.org/projects/builder/
+This is a small rake task that will crawl a static site locally in the specific directory and generate a sitemap.xml file with a list of links, optionally compressing it.
 
+## Installation ##
 
-== Configuration/Usage
+  gem install static_sitemap_tasks
 
-On initialization generate_sitemap will copy the generate_sitemap_key.yml file you your
-RAILS_ROOT/config directory.
+## Configuration/Usage ##
+To use, require the gem in your Rakefile and install the task with configuration
 
-Open this file and specify your domain for the different environments and specify the changefreq for
-your site.
+    require 'static_sitemap_tasks'
 
-For changefreq options see this page: http://www.sitemaps.org/protocol.php
-I haven't devised a way to set this dynamically, as most pages will vary.
+    SitemapGenerator::Tasks.install(
+      :base_url => 'http://www.mysite.com', # Required
+      :change_frequency => 'daily', # Optional, see http://www.sitemaps.org/protocol.php#changefreqdef
+      :date_mode => 'git' # Optional, one of 'git' or 'mtime'. default: nil (disabled)
+      :gzip_output => true, # Optional, default: true
+      :index_files => [ 'index.html' ], # Optional, default: [ 'index.html', 'index.htm' ]
+      :public_root => 'public' # Optional, default: Dir.pwd
+    )
 
-The plugin will pull the correct domain name for crawling the site by using the RAILS_ENV variable
-in the rake task.
+To execute,
+  rake generate_sitemap
 
-Run the task with the command
-	rake plugin:generate_sitemap
-	
-Go to http://localhost:3000/sitemap.xml (or open public/sitemap.xml)
-
-
-== TODO
-
- - set changefreq, lastmod, priority dynamically during generation
- - allow generation of sitemap index files
- - write tests
- - allow for exclusions to be specified in an array
-
-
-== More Info
+## More Info ##
 
 http://www.sitemaps.org/protocol.php
 
-Questions, comments, patches, etc. can be sent to 
-	tom dot cocca at gmail dot com
+## Credits ##
+
+Originally Authored by Chris Marting (http://chriscodes.com/articles/view/54)
+Updates by Tom Cocca
+Rewrite for static sites by Michael Leinartas
+ping_search_engines() lifted from http://github.com/adamsalter/sitemap_generator by Adam Salter

data/lib/static_sitemap_tasks.rb

@@ -1,6 +1,8 @@
 require 'rubygems'
 require 'builder'
 require 'hpricot'
+require 'time'
+require 'cgi'
 require 'uri'
 
 
@@ -14,22 +16,74 @@ module SitemapGenerator
     end
 
     def initialize(options = {})
-      # Root of files to crawl
-      @public_root = options[:public_root] || Dir.pwd
-      # Change frequency - see: http://www.sitemaps.org/protocol.php#changefreqdef
-      @change_frequency = options[:change_frequency]
       # Canonical domain of published site
       @base_url = options[:base_url]
-      # Index pages
-      @index_files = options[:index_files] || [ 'index.html', 'index.htm' ]
+      # Change frequency - see: http://www.sitemaps.org/protocol.php#changefreqdef
+      @change_frequency = options[:change_frequency]
+      # Date mode - one of [ 'git', 'mtime' ]
+      @date_mode = options[:date_mode]
       # Compress output to sitemap.xml.gz
       @gzip_output = options[:gzip_output] || true
+      # Index pages
+      @index_files = options[:index_files] || [ 'index.html', 'index.htm' ]
+      # Root of files to crawl
+      @public_root = options[:public_root] || Dir.pwd
     end
 
     def install
-      desc "Generate a sitemap based on the contents of #{@public_root}"
-      task 'generate_sitemap' do
-        generate_sitemap
+      namespace :sitemap do
+        desc "Generate a sitemap based on the contents of #{@public_root}"
+        task :generate do
+          generate_sitemap
+        end
+
+        desc "Ping providers to notify them that a new sitemap.xml is available"
+        task :ping do
+          ping_search_engines
+        end
+      end
+    end
+
+    # uses Hpricot to grab links from a URI
+    # adds uri to @pages_crawled
+    # loops each link found
+    # adds link to pages array if it should be included, unless it already exists
+    def crawl_for_links(link)
+      if link.include?('http')
+        return unless link_path.include?(@base_url)
+        link_path = link.sub!(@base_url,'')
+      else
+        link_path = link
+      end
+      file_path = resolve_file_path(File.join(@public_root, link_path))
+
+      if file_path.nil?
+        puts "Warning: Unable to resolve #{link_path} to a local file"
+        return
+      end
+
+      puts "Inspecting #{file_path}...\n"
+      doc = Hpricot(open(file_path)) rescue nil
+      return unless doc
+      @pages_crawled << link
+      last_updated = find_date(file_path)
+      @page_times[link] = last_updated if last_updated
+
+      (doc/"a").each do |a|
+        if a['href'] && should_be_included?(a['href'])
+          @pages << a['href'] unless(link_exists?(a['href'],@pages))
+        end
+      end
+    end
+
+    def find_date(file)
+      case @date_mode
+      when 'git'
+        date = %x[git log -n 1 --date=iso --format="%ad" #{file}]
+        date.strip()
+      when 'mtime'
+        mtime = File.mtime(file) rescue nil
+        mtime.iso8601 if mtime
       end
     end
 
@@ -37,6 +91,7 @@ module SitemapGenerator
       # holds pages to go into map, and pages crawled
       @pages = []
       @pages_crawled = []
+      @page_times = {}
 
       # start with index pages
       crawl_for_links('/')
@@ -54,9 +109,10 @@ module SitemapGenerator
         # loop through array of pages, and build sitemap.xml
         @pages.sort.each {|link|
           xml.url {
-            xml.loc URI.join(@base_url, link)
+            xml.loc URI.join(@base_url, link).to_s
             # TODO - set changefreq dynamically per page
             xml.changefreq @change_frequency unless @change_frequency.nil?
+            xml.lastmod @page_times[link] unless @page_times[link].nil?
           }
         }
       }
@@ -77,29 +133,25 @@ module SitemapGenerator
       xml_file.close
     end
 
-    # uses Hpricot to grab links from a URI
-    # adds uri to @pages_crawled
-    # loops each link found
-    # adds link to pages array if it should be included, unless it already exists
-    def crawl_for_links(link_path)
-      if link_path.include?('http')
-        return unless link_path.include?(@base_url)
-        link_path.sub!(@base_url,'')
-      end
-      file_path = resolve_file_path(File.join(@public_root, link_path))
-
-      if file_path.nil?
-        puts "Warning: Unable to resolve #{link_path} to a local file"
-        return
+    def ping_search_engines
+      require 'open-uri'
+      if @gzip_output
+        url = URI.join(@base_url,'sitemap.xml.gz').to_s
+      else
+        url = URI.join(@base_url,'sitemap.xml').to_s
       end
-
-      puts "Inspecting #{file_path}...\n"
-      doc = Hpricot(open(file_path)) rescue nil
-      return unless doc
-      @pages_crawled << link_path
-      (doc/"a").each do |a|
-        if a['href'] && should_be_included?(a['href'])
-          @pages << a['href'] unless(link_exists?(a['href'],@pages))
+      index_location = CGI.escape(url)
+
+      # engines list from http://en.wikipedia.org/wiki/Sitemap_index
+      {:google => "http://www.google.com/webmasters/sitemaps/ping?sitemap=#{index_location}",
+        :ask => "http://submissions.ask.com/ping?sitemap=#{index_location}",
+        :bing => "http://www.bing.com/webmaster/ping.aspx?siteMap=#{index_location}",
+        :sitemap_writer => "http://www.sitemapwriter.com/notify.php?crawler=all&url=#{index_location}"}.each do |engine, link|
+        begin
+          open(link)
+          puts "Successful ping of #{engine.to_s}" if verbose
+        rescue Timeout::Error, StandardError => e
+          puts "Ping failed for #{engine.to_s}: #{e.inspect}" if verbose
         end
       end
     end

data/static_sitemap_tasks.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "static_sitemap_tasks"
-  s.version     = "0.1"
+  s.version     = "0.2"
   s.platform    = Gem::Platform::RUBY
   s.summary     = 'Rake tasks to manage sitemap.xml generation for static sites'
   s.description = 'Rake tasks to manage sitemap.xml generation for static sites'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: static_sitemap_tasks
 version: !ruby/object:Gem::Version
-  version: '0.1'
+  version: '0.2'
   prerelease: 
 platform: ruby
 authors:
@@ -11,11 +11,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-10-12 00:00:00.000000000Z
+date: 2011-10-14 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70212813523740 !ruby/object:Gem::Requirement
+  requirement: &70099743651700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -23,10 +23,10 @@ dependencies:
         version: 0.8.7
   type: :development
   prerelease: false
-  version_requirements: *70212813523740
+  version_requirements: *70099743651700
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &70212813523280 !ruby/object:Gem::Requirement
+  requirement: &70099743644920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -34,7 +34,7 @@ dependencies:
         version: '1.0'
   type: :development
   prerelease: false
-  version_requirements: *70212813523280
+  version_requirements: *70099743644920
 description: Rake tasks to manage sitemap.xml generation for static sites
 email:
 - mleinartas@gmail.com
@@ -43,6 +43,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- ChangeLog
 - Gemfile
 - MIT-LICENSE
 - README.md