crawlable 0.0.1.5 → 0.0.1.6

data/README.markdown

@@ -8,21 +8,73 @@ Super DRY Sitemaps for Rails and Sinatra Apps (works on Heroku!)
 
     sudo gem install crawlable
     
-### Setup (config/sitemap.rb)
+### Sitemap (`config/sitemap.rb`)
 
-    Sitemap do |sitemap|
+    Sitemap "http://www.example.com" do
+      link articles_path, :priority => 0.7, :changes => 'daily'
+      
+      Post.all.each do |a|
+        link articles_path(a), :updated_at => a.updated_at do
+          image images_path(a.featured_image)
+        end
+      end
+    end
+    
+#### Result
 
-      sitemap.add posts_path, :priority => 1, :changes => 'daily'
+    <?xml version="1.0"?>
+    <urlset>
+      <url>
+        <loc>/articles</loc>
+        <lastmod>2010-06-20T09:38:26+00:00</lastmod>
+        <changefreq>daily</changefreq>
+        <priority>0.7</priority>
+      </url>
+      <url>
+        <loc>/articles/title-0</loc>
+        <lastmod>2010-06-20T09:38:26+00:00</lastmod>
+        <changefreq>weekly</changefreq>
+        <priority>0.5</priority>
+      </url>
+      <url>
+        <loc>/articles/title-1</loc>
+        <lastmod>2010-06-20T09:38:26+00:00</lastmod>
+        <changefreq>weekly</changefreq>
+        <priority>0.5</priority>
+      </url>
+      ...
+    </urlset>
+    
+### Feed
 
-      Post.all.each do |post|
-        sitemap.add posts_path(post), :last_modified => post.updated_at
+    Feed do
+      title "My RSS Feed"
+      author "Lance Pollard"
+      description "Something nice and tidy"
+      
+      Post.all.each do |a|
+        entry "/posts/#{a.to_param}", :updated_at => a.updated_at, :title => a.title
       end
-
     end
 
 ## Features
 
 - Works on Heroku
+- Pings Google, Bing, Yahoo!, and Ask whenever anything changes
 
 ## Alternatives
 
+- [SitemapGenerator](http://github.com/kjvarga/sitemap_generator)
+- [Sitemap](http://github.com/queso/sitemap)
+- [BigSitemap](http://github.com/alexrabarts/big_sitemap)
+- [Sitemap (diff than above)](http://github.com/flyerhzm/sitemap)
+- [SitemapGenerator (diff than above)](http://github.com/christianhellsten/sitemap-generator)
+- [Sitemapper](http://github.com/milk-it/sitemapper)
+
+## Resources
+
+- [Official Sitemap Protocol](http://sitemaps.org/protocol.php)
+- [Official RSS Feed Spec](http://cyber.law.harvard.edu/rss/rss.html)
+- [Sitemap Engine List](http://en.wikipedia.org/wiki/Sitemap_index)
+- [Yahoo Site Explorer](http://developer.yahoo.com/search/siteexplorer/V1/updateNotification.html)
+- [Comparison of Feed Aggregators](http://en.wikipedia.org/wiki/Comparison_of_feed_aggregators)

data/Rakefile

@@ -5,7 +5,7 @@ require 'rake/gempackagetask'
 spec = Gem::Specification.new do |s|
   s.name              = "crawlable"
   s.authors           = ["Lance Pollard"]
-  s.version           = "0.0.1.5"
+  s.version           = "0.0.1.6"
   s.summary           = "Crawlable: Super DRY Sitemaps for Rails and Sinatra Apps"
   s.homepage          = "http://github.com/viatropos/crawlable"
   s.email             = "lancejpollard@gmail.com"
@@ -14,6 +14,7 @@ spec = Gem::Specification.new do |s|
   s.rubyforge_project = "crawlable"
   s.platform          = Gem::Platform::RUBY
   s.files             = %w(README.markdown Rakefile init.rb MIT-LICENSE) + Dir["{lib,rails,test}/**/*"] - Dir["test/tmp"]
+  s.add_dependency("nokogiri", ">= 1.4.1")
   s.require_path      = "lib"
 end
 

data/lib/crawlable.rb

@@ -1,22 +1,20 @@
 require 'rubygems'
 require 'active_support'
 require 'active_record'
+require 'nokogiri'
+require 'open-uri'
 
 this = File.dirname(__FILE__)
-Dir["#{this}/crawlable/*"].each { |c| require c }
-
-class Settings
-  include Cockpit::Configuration
-end
+Dir["#{this}/crawlable/*"].each { |c| require c if File.extname(c) == ".rb" }
 
 def Sitemap(*args, &block)
-  Sitemap.configure(*args, &block)
+  Crawlable::Sitemap.define!(*args, &block)
 end
 
-def Crawlable(*args, &block)
-  Sitemap(*args, &block)
+def Feed(*args, &block)
+  Crawlable::Feed.define!(*args, &block)
 end
 
-def Crawl(*args, &block)
-  Sitemap(*args, &block)
-end
+def Crawlable(type, *args, &block)
+  type.to_s.constantize(*args, &block)
+end

data/lib/crawlable/feed.rb

@@ -0,0 +1,168 @@
+module Crawlable
+  class Feed
+    
+    class << self
+      attr_accessor :options, :call_options
+      
+      def define!(*args, &block)
+        self.options = args.extract_options!#.merge(:run => args.shift)
+        
+        instance_eval(&block) if block_given?
+      end
+      
+      def parse!(path, options = {})
+        path ||= File.join(::Rails.root, 'config/initializers/feeds.rb')
+        self.call_options = options.symbolize_keys
+        eval(IO.read(path))
+        self.call_options = nil
+      end
+      
+      def method_missing(meth, *args, &block)
+        if block_given?
+          if !self.call_options.blank?
+            if call_options.has_key?(meth.to_sym)
+              self.new(meth, *args, &block)
+            else
+              super(meth, *args, &block)
+            end
+          else
+            self.new(meth, *args, &block)
+          end
+        else
+          super(meth, *args, &block)
+        end
+      end
+      
+    end
+    
+    if defined?(::Rails)
+      if ActionPack::VERSION::MAJOR == 3
+        include ::Rails.application.routes.url_helpers
+      else
+        require 'action_controller'
+        include ActionController::UrlWriter
+      end
+    end
+    
+    attr_accessor :title, :url, :description, :master, :copyright, :updated_at
+    
+    def initialize(*args, &block)
+      options = args.extract_options!
+      
+      name    = args.shift
+      
+      options.each do |k, v|
+        self.send(k, v) if self.respond_to?(k)
+      end
+      
+      instance_eval(&block)
+    end
+    
+    def copyright(string = nil)
+      @copyright = string unless string.nil?
+      @copyright
+    end
+    
+    def title(string = nil)
+      @title = string unless string.nil?
+      @title
+    end
+    
+    def description(string = nil)
+      @description = string unless string.nil?
+      @description
+    end
+    
+    def url(string = nil)
+      @url = string unless string.nil?
+      @url
+    end
+    
+    def author(string = nil)
+      @author = string unless string.nil?
+      @author
+    end
+    
+    def master(string = nil)
+      @master = string unless string.nil?
+      @master
+    end
+    
+    def entries
+      @entries ||= []
+    end
+    
+    def entry(path, *args, &block)
+      options = args.extract_options!
+
+      result = options.dup
+      result.merge!(:url => path)
+      
+      self.entries.push(result)
+    end
+    
+    def to_rss
+      builder = Nokogiri::XML::Builder.new do |xml|
+        xml.rss "xmlns:dc" => "http://purl.org/dc/elements/1.1/" do
+          xml.channel do
+            xml.title           self.title
+            xml.link            self.url
+            xml.description     self.description
+            
+            self.entries.each do |entry|
+              puts entry.inspect
+              xml.item do
+                xml.title       entry[:title]
+                xml.link        entry[:url]
+                xml.description entry[:description]
+                xml.guid        entry[:guid] || entry[:url]
+                xml.author      entry[:author] unless entry[:author].blank?
+                entry[:categories].each do |category|
+                  xml.category category
+                end unless entry[:categories].blank?
+                xml.comments    entry[:comments] unless entry[:comments].blank?
+                xml.enclosure(
+                  :url    => entry[:asset],
+                  :length => IO.size(entry[:asset]).to_s,
+                  :type   => entry[:asset_type]
+                ) unless entry[:asset].blank?
+                xml.webMaster   self.master unless self.master.blank?
+              end
+            end
+          end
+        end
+      end
+      builder.to_xml
+    end
+    
+    def to_atom
+      builder = Nokogiri::XML::Builder.new do |xml|
+        xml.feed "xmlns" => "http://www.w3.org/2005/Atom" do
+          xml.title   self.title
+          xml.link    "rel" => "self", "href" => url_for(:only_path => false, :controller => 'feeds', :action => 'atom')
+          xml.link    "rel" => "alternate", "href" => url_for(:only_path => false, :controller => 'posts')
+          xml.id      url_for(:only_path => false, :controller => 'posts')
+          xml.updated self.updated_at.strftime "%Y-%m-%dT%H:%M:%SZ" if self.updated_at
+          xml.author  { xml.name self.author }
+          
+          self.entries.each do |entries|
+            xml.entry do
+              xml.title   entries.title
+              xml.link    "rel" => "alternate", "href" => url_for(:only_path => false, :controller => 'posts', :action => 'show', :id => entries.id)
+              xml.id      url_for(:only_path => false, :controller => 'posts', :action => 'show', :id => entries.id)
+              xml.updated entries.updated_at.strftime "%Y-%m-%dT%H:%M:%SZ"
+              xml.author  { xml.name entries.author.name }
+              xml.summary "Post summary"
+              xml.content "type" => "html" do
+                #xml.text! render(:partial => "posts/post", :post => post)
+              end
+            end
+          end
+
+        end
+      end
+      builder.to_xml
+    end
+    
+  end
+end

data/lib/crawlable/rack.rb

@@ -0,0 +1,25 @@
+module Crawlable
+  class Rack
+    
+    def initialize(app)
+      @app = app
+    end
+    
+    def call(env)
+      if ENV["HEROKU"]
+        return sitmap if env['REQUEST_PATH'] =~ /\/sitemap\.xml/i
+      end
+      @app.call(env)
+    end
+    
+    def sitemap
+      file = File.join(heroku_writable_directory, "sitemap.xml.gz")
+      [200, { 'Cache-Control'  => 'public, max-age=86400', 'Content-Length' => File.size(file).to_s, 'Content-Type'   => 'text/xml' }, IO.read(file)]
+    end
+    
+    def heroku_writable_directory
+      "#{Rails.root}/tmp"
+    end
+    
+  end
+end

data/lib/crawlable/sitemap.rb

@@ -0,0 +1,195 @@
+module Crawlable
+  class Sitemap
+    
+    class << self
+      attr_accessor :instance
+      
+      def define!(*args, &block)
+        self.instance = self.new(*args, &block)
+      end
+      
+      def parse!(path)
+        path ||= File.join(::Rails.root, 'config/sitemap.rb')
+        eval(IO.read(path))
+      end
+      
+      def write(to, compress = false)
+        self.instance.write(to, compress)
+      end
+      
+      def process(from, to, compress = false, &block)
+        parse!(from)
+        write(to, compress)
+      end
+      
+      def inspect
+        self.instance.inspect
+      end
+      
+      def to_xml
+        self.instance.to_xml
+      end
+      
+      def clear
+        self.instance.clear
+        self.instance = nil
+      end
+      
+    end
+    
+    attr_accessor :links, :host, :ping, :yahoo_app_id
+    
+    def initialize(*args, &block)
+      self.host = args.shift
+      raise "Please define a host: 'Sitemap 'http://my-site.com' do ..." if self.host.blank?
+      options = args.extract_options!
+      
+      options.each do |k, v|
+        self.send(k, v) if self.respond_to?(k)
+      end
+      
+      instance_eval(&block)
+    end
+    
+    def yahoo_app_id(string = nil)
+      @yahoo_app_id = string unless string.nil?
+      @yahoo_app_id
+    end
+    
+    def links
+      @links ||= []
+    end
+    
+    def host(*args)
+      @host = args unless args.empty?
+      @host
+    end
+    
+    def ping(*args)
+      @ping = args unless args.empty?
+      @ping
+    end
+    
+    def sitemap_path(string = nil)
+      @sitemap_path = string || "public/sitemap.xml"
+    end
+    
+    def link(path, *args, &block)
+      options = args.extract_options!
+      options.assert_valid_keys(:priority, :changes, :updated_at, :host)
+      options.reverse_merge!(
+        :priority => 0.5,
+        :changes => 'monthly',
+        :updated_at => Time.now,
+        :host => self.host
+      )
+
+      result = {
+        :host        => options[:host],
+        :path        => path,
+        :url         => URI.join(options[:host], path).to_s,
+        :priority    => options[:priority],
+        :changes     => options[:changes],
+        :updated_at  => options[:updated_at],
+        :images      => []
+      }
+      
+      self.links.push(result)
+      
+      instance_eval(&block) if block_given?
+      
+      result
+    end
+    
+    def image(path, *args, &block)
+      options = args.extract_options!
+      options.assert_valid_keys(:priority, :changes, :updated_at, :host)
+      
+      result = {
+        :path          => path,
+        :caption       => options[:caption],
+        :geo_location  => options[:geo_location],
+        :title         => options[:title],
+        :license       => options[:license]
+      }
+      
+      self.links.last[:images].push(result)
+    end
+    
+    def w3c_date(date)
+       date.utc.strftime("%Y-%m-%dT%H:%M:%S+00:00")
+    end
+    
+    def to_xml
+      namespaces = {
+        "xmlns" => "http://www.sitemaps.org/schemas/sitemap/0.9",
+        "xmlns:image" => "http://www.google.com/schemas/sitemap-image/1.1"
+      }
+      builder = Nokogiri::XML::Builder.new do |xml|
+        xml.urlset(namespaces) do
+          self.links.each do |link|
+            xml.url do
+              xml.loc        link[:path]
+              xml.lastmod    w3c_date(link[:updated_at])  if link[:updated_at]
+              xml.changefreq link[:changes]               if link[:changes]
+              xml.priority   link[:priority]              if link[:priority]
+              link[:images].each do |image|
+                xml["image"].image do
+                  xml["image"].loc          image[:path]
+                  xml["image"].caption      image[:caption]       if image[:caption]
+                  xml["image"].geo_location image[:geo_location]  if image[:geo_location]
+                  xml["image"].title        image[:title]         if image[:title]
+                  xml["image"].license      image[:license]       if image[:license]
+                end
+              end
+            end
+          end
+        end
+      end
+      builder.to_xml
+    end
+    
+    def write(path, compress)
+      to = path
+      if compress
+        to << ".gz" unless File.extname(path) == ".gz"
+        File.open(to, 'wb') do |file|
+          gz = Zlib::GzipWriter.new(file)
+          gz.write to_xml
+          gz.close
+        end
+      else
+        File.open(to, 'wb') do |file|
+          file.puts to_xml
+        end
+      end
+    end
+    
+    def notify
+      engines = {
+        :google => "http://www.google.com/webmasters/sitemaps/ping?sitemap=#{path}",
+        :yahoo => "http://search.yahooapis.com/SiteExplorerService/V1/ping?sitemap=#{path}&appid=#{yahoo_app_id}",
+        :ask => "http://submissions.ask.com/ping?sitemap=#{path}",
+        :bing => "http://www.bing.com/webmaster/ping.aspx?siteMap=#{path}",
+        :sitemap_writer => "http://www.sitemapwriter.com/notify.php?crawler=all&url=#{path}"
+      }
+      engines.each do |engine, link|
+        begin
+          open(link)
+          puts "Successful ping of #{engine.to_s.titleize}"
+        rescue Timeout::Error, StandardError => e
+          puts "Ping failed for #{engine.to_s.titleize}: #{e.inspect}"
+        end
+      end
+    end
+    
+    def clear
+      @links = nil
+    end
+    
+    def inspect
+      "<Sitemap @host='#{host.to_s}' @sitemap_path='#{sitemap_path.to_s}' @ping='#{ping.inspect}' @links='#{links.inspect}'/>"
+    end
+    
+  end
+end

data/lib/engine.rb

@@ -0,0 +1,9 @@
+module Crawlable
+  class Engine < Rails::Engine
+    
+    initializer "authlogic_connect.authentication_hook" do |app|
+      app.middleware.use Crawlable::Rack
+    end
+    
+  end
+end

data/test/lib/_database.rb

@@ -0,0 +1,16 @@
+begin
+  ActiveRecord::Base.establish_connection(:adapter => "sqlite3", :database => ":memory:")
+rescue ArgumentError
+  ActiveRecord::Base.establish_connection(:adapter => "sqlite3", :dbfile => ":memory:")
+end
+
+ActiveRecord::Base.configurations = true
+
+ActiveRecord::Schema.define(:version => 1) do
+
+  create_table :posts, :force => true do |t|
+    t.string :title
+    t.timestamps
+  end
+  
+end

data/test/lib/post.rb

@@ -0,0 +1,5 @@
+class Post < ActiveRecord::Base
+  def to_param
+    self.title.downcase.gsub(/\s+/, "-")
+  end
+end

data/test/test_feed.rb

@@ -0,0 +1,18 @@
+require File.join(File.dirname(__FILE__), 'test_helper')
+
+class FeedTest < ActiveSupport::TestCase
+  
+  context "Feed" do
+    
+    setup do
+      create_posts
+      load_feed
+    end
+    
+    should "create feed" do
+      puts Crawlable::Feed.to_rss
+    end
+    
+  end
+  
+end

data/test/test_helper.rb

@@ -9,6 +9,10 @@ require 'active_record/fixtures'
 require 'shoulda'
 require 'shoulda/active_record'
 
+this = File.expand_path(File.dirname(__FILE__))
+require File.expand_path(File.join(this, '/../lib/crawlable'))
+
+Dir["#{this}/lib/*"].each { |c| require c if File.extname(c) == ".rb" }
 require File.expand_path(File.join(File.dirname(__FILE__), '/../lib/crawlable'))
 
 ActiveRecord::Base.class_eval do
@@ -16,3 +20,42 @@ ActiveRecord::Base.class_eval do
     all.map(&:destroy)
   end
 end
+
+ActiveSupport::TestCase.class_eval do
+  
+  def create_posts(many = 10)
+    many.times do |i|
+      Post.create!(:title => "title-#{i.to_s}")
+    end
+  end
+  
+  def load_sitemap
+    Sitemap "http://www.example.com" do
+      ping :yahoo, :google
+      
+      link "/posts", :priority => 0.7, :changes => 'daily' do
+        3.times do |i|
+          image "/images/#{i}"
+        end
+      end
+      
+      Post.all.each do |a|
+        link "/posts/#{a.to_param}", :updated_at => a.updated_at
+      end
+    end
+  end
+  
+  def load_feed
+    Feed do
+      posts do
+        title "My RSS Feed"
+        author "Lance Pollard"
+        description "Something nice and tidy"
+        
+        Post.all.each do |a|
+          entry "/posts/#{a.to_param}", :updated_at => a.updated_at, :title => a.title
+        end
+      end
+    end
+  end
+end

data/test/test_sitemap.rb

@@ -4,8 +4,20 @@ class SitemapTest < ActiveSupport::TestCase
   
   context "Sitemap" do
     
+    setup do
+      create_posts
+      load_sitemap
+    end
     
+    should "create sitemap" do
+      Crawlable::Sitemap.write("test/sitemap.xml")
+      assert File.exists?("test/sitemap.xml.gz")
+    end
+    
+    teardown do
+      File.delete("test/sitemap.xml.gz") if File.exists?("test/sitemap.xml.gz")
+    end
     
   end
   
-end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification 
 name: crawlable
 version: !ruby/object:Gem::Version 
-  hash: 65
+  hash: 71
   prerelease: false
   segments: 
   - 0
   - 0
   - 1
-  - 5
-  version: 0.0.1.5
+  - 6
+  version: 0.0.1.6
 platform: ruby
 authors: 
 - Lance Pollard
@@ -16,10 +16,25 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-06-19 00:00:00 -07:00
+date: 2010-07-02 00:00:00 -07:00
 default_executable: 
-dependencies: []
-
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: nokogiri
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 5
+        segments: 
+        - 1
+        - 4
+        - 1
+        version: 1.4.1
+  type: :runtime
+  version_requirements: *id001
 description: Super DRY Sitemaps for Rails and Sinatra Apps
 email: lancejpollard@gmail.com
 executables: []
@@ -33,9 +48,16 @@ files:
 - Rakefile
 - init.rb
 - MIT-LICENSE
-- lib/crawlable/crawlable.rb
+- lib/crawlable/feed.rb
+- lib/crawlable/rack.rb
+- lib/crawlable/sitemap.rb
 - lib/crawlable.rb
+- lib/engine.rb
 - rails/init.rb
+- test/lib/_database.rb
+- test/lib/post.rb
+- test/lib/routes.rb
+- test/test_feed.rb
 - test/test_helper.rb
 - test/test_sitemap.rb
 has_rdoc: true

data/lib/crawlable/crawlable.rb

@@ -1,12 +0,0 @@
-module Crawlable
-  
-  def self.included(base)
-    base.extend ClassMethods
-  end
-  
-  module ClassMethods
-    def acts_as_crawlable(*args, &block)
-    end
-  end
-  
-end