crawlable 0.0.1.7 → 0.0.1.8

data/Rakefile

@@ -5,7 +5,7 @@ require 'rake/gempackagetask'
 spec = Gem::Specification.new do |s|
   s.name              = "crawlable"
   s.authors           = ["Lance Pollard"]
-  s.version           = "0.0.1.7"
+  s.version           = "0.0.1.8"
   s.summary           = "Crawlable: Super DRY Sitemaps for Rails and Sinatra Apps"
   s.homepage          = "http://github.com/viatropos/crawlable"
   s.email             = "lancejpollard@gmail.com"

data/lib/crawlable.rb

@@ -6,6 +6,7 @@ require 'open-uri'
 
 this = File.dirname(__FILE__)
 Dir["#{this}/crawlable/*"].each { |c| require c if File.extname(c) == ".rb" }
+require "#{this}/engine"
 
 def Sitemap(*args, &block)
   Crawlable::Sitemap.define!(*args, &block)

data/lib/crawlable/feed.rb

@@ -2,7 +2,16 @@ module Crawlable
   class Feed
     
     class << self
-      attr_accessor :options, :call_options
+      attr_accessor :options, :call_options, :feeds
+      
+      def feeds
+        @feeds ||= []
+        @feeds
+      end
+      
+      def paths
+        feeds.map(&:path)
+      end
       
       def define!(*args, &block)
         self.options = args.extract_options!#.merge(:run => args.shift)
@@ -10,6 +19,21 @@ module Crawlable
         instance_eval(&block) if block_given?
       end
       
+      def find(path, directory)
+        if path =~ /(\/feed\/?)(atom|rss2|rss|rdf)?$/i
+          feed   = $1
+          format = $2
+
+          format = format.blank? ? "rss" : format.to_s
+          file = path.gsub(format, "").split("/").delete_if {|i| i.blank? }[0..-2].join("/")
+
+          return unless self.paths.include?(file)
+          file = file.split("/").join("_")
+          file = File.join(directory, "feeds", "#{file}.#{format}")
+          return file
+        end
+      end
+      
       def parse!(path, options = {})
         path ||= File.join(::Rails.root, 'config/initializers/feeds.rb')
         self.call_options = options.symbolize_keys
@@ -17,16 +41,25 @@ module Crawlable
         self.call_options = nil
       end
       
+      def write(to, *formats)
+        self.feeds.each { |feed| feed.write(to, *formats) }
+      end
+      
+      def process!(from, to, *formats, &block)
+        parse!(from)
+        write(to, *formats)
+      end
+      
       def method_missing(meth, *args, &block)
         if block_given?
           if !self.call_options.blank?
             if call_options.has_key?(meth.to_sym)
-              self.new(meth, *args, &block)
+              self.feeds << self.new(meth, *args, &block)
             else
               super(meth, *args, &block)
             end
           else
-            self.new(meth, *args, &block)
+            self.feeds << self.new(meth, *args, &block)
           end
         else
           super(meth, *args, &block)
@@ -35,18 +68,26 @@ module Crawlable
       
     end
     
-    attr_accessor :title, :url, :description, :master, :copyright, :updated_at
+    attr_accessor :name, :title, :url, :description, :master, :copyright, :updated_at, :path
     
     def initialize(*args, &block)
       options = args.extract_options!
       
-      name    = args.shift
+      self.name    = args.shift
       
       options.each do |k, v|
         self.send(k, v) if self.respond_to?(k)
       end
       
       instance_eval(&block)
+      
+      self
+    end
+    
+    def path(value = nil)
+      @path = value if value
+      @path ||= name.to_s
+      @path
     end
     
     def copyright(string = nil)
@@ -101,7 +142,6 @@ module Crawlable
             xml.description     self.description
             
             self.entries.each do |entry|
-              puts entry.inspect
               xml.item do
                 xml.title       entry[:title]
                 xml.link        entry[:url]
@@ -130,19 +170,19 @@ module Crawlable
       builder = Nokogiri::XML::Builder.new do |xml|
         xml.feed "xmlns" => "http://www.w3.org/2005/Atom" do
           xml.title   self.title
-          xml.link    "rel" => "self", "href" => url_for(:only_path => false, :controller => 'feeds', :action => 'atom')
-          xml.link    "rel" => "alternate", "href" => url_for(:only_path => false, :controller => 'posts')
-          xml.id      url_for(:only_path => false, :controller => 'posts')
+          #xml.link    "rel" => "self", "href" => url_for(:only_path => false, :controller => 'feeds', :action => 'atom')
+          #xml.link    "rel" => "alternate", "href" => url_for(:only_path => false, :controller => 'posts')
+          #xml.id      url_for(:only_path => false, :controller => 'posts')
           xml.updated self.updated_at.strftime "%Y-%m-%dT%H:%M:%SZ" if self.updated_at
           xml.author  { xml.name self.author }
           
-          self.entries.each do |entries|
+          self.entries.each do |entry|
             xml.entry do
-              xml.title   entries.title
-              xml.link    "rel" => "alternate", "href" => url_for(:only_path => false, :controller => 'posts', :action => 'show', :id => entries.id)
-              xml.id      url_for(:only_path => false, :controller => 'posts', :action => 'show', :id => entries.id)
-              xml.updated entries.updated_at.strftime "%Y-%m-%dT%H:%M:%SZ"
-              xml.author  { xml.name entries.author.name }
+              xml.title   entry[:title]
+              #xml.link    "rel" => "alternate", "href" => url_for(:only_path => false, :controller => 'posts', :action => 'show', :id => entry[:id])
+              #xml.id      url_for(:only_path => false, :controller => 'posts', :action => 'show', :id => entry[:id])
+              xml.updated entry[:updated_at].strftime "%Y-%m-%dT%H:%M:%SZ"
+              xml.author  { xml.name entry[:author] } if entry[:author]
               xml.summary "Post summary"
               xml.content "type" => "html" do
                 #xml.text! render(:partial => "posts/post", :post => post)
@@ -155,5 +195,14 @@ module Crawlable
       builder.to_xml
     end
     
+    def write(to, *formats)
+      formats.each do |format|
+        FileUtils.mkdir_p(File.dirname(to))
+        File.open("#{to}.#{format.to_s}", 'wb') do |file|
+          file.puts send("to_#{format.to_s}")
+        end
+      end
+    end
+    
   end
 end

data/lib/crawlable/rack.rb

@@ -6,14 +6,21 @@ module Crawlable
     end
     
     def call(env)
-      if ENV["HEROKU"]
-        return sitmap if env['REQUEST_PATH'] =~ /\/sitemap\.xml/i
+      if using_heroku?(env) || true
+        if file = Crawlable::Sitemap.find(env['REQUEST_PATH'], heroku_writable_directory)
+          return sitemap(file)
+        elsif file = Crawlable::Feed.find(env['REQUEST_PATH'], heroku_writable_directory)
+          return feed(file)
+        end
       end
       @app.call(env)
     end
     
-    def sitemap
-      file = File.join(heroku_writable_directory, "sitemap.xml.gz")
+    def sitemap(file)
+      [200, { 'Cache-Control'  => 'public, max-age=86400', 'Content-Length' => File.size(file).to_s, 'Content-Type'   => 'text/xml' }, IO.read(file)]
+    end
+    
+    def feed(file)
       [200, { 'Cache-Control'  => 'public, max-age=86400', 'Content-Length' => File.size(file).to_s, 'Content-Type'   => 'text/xml' }, IO.read(file)]
     end
     
@@ -21,5 +28,17 @@ module Crawlable
       "#{Rails.root}/tmp"
     end
     
+    def using_heroku?(env)
+      if env["HEROKU"].nil?
+        if env["HEROKU_PORT"].nil? && ENV["HEROKU_TYPE"].nil?
+          env["HEROKU"] = false
+        else
+          env["HEROKU"] = true
+        end
+      end
+      
+      env["HEROKU"] == true
+    end
+    
   end
 end

data/lib/crawlable/sitemap.rb

@@ -8,6 +8,16 @@ module Crawlable
         self.instance = self.new(*args, &block)
       end
       
+      def path
+        self.instance ? self.instance.path : ""
+      end
+      
+      def find(path, directory)
+        if path =~ /#{Regexp.escape(self.path)}/i
+          return File.join(directory, self.path)
+        end
+      end
+      
       def parse!(path)
         path ||= File.join(::Rails.root, 'config/sitemap.rb')
         eval(IO.read(path))
@@ -17,7 +27,7 @@ module Crawlable
         self.instance.write(to, compress)
       end
       
-      def process(from, to, compress = false, &block)
+      def process!(from, to, compress = false, &block)
         parse!(from)
         write(to, compress)
       end
@@ -37,11 +47,10 @@ module Crawlable
       
     end
     
-    attr_accessor :links, :host, :ping, :yahoo_app_id
+    attr_accessor :links, :sitemap_host, :ping, :yahoo_app_id, :path, :stylesheet
     
     def initialize(*args, &block)
-      self.host = args.shift
-      raise "Please define a host: 'Sitemap 'http://my-site.com' do ..." if self.host.blank?
+      self.sitemap_host = args.shift
       options = args.extract_options!
       
       options.each do |k, v|
@@ -49,6 +58,15 @@ module Crawlable
       end
       
       instance_eval(&block)
+      
+      raise "Please define a host: 'Sitemap 'http://my-site.com' do ..." if self.sitemap_host.blank?
+      
+    end
+    
+    def path(value = nil)
+      @path = value if value
+      @path ||= "/sitemap.xml"
+      @path
     end
     
     def yahoo_app_id(string = nil)
@@ -60,9 +78,14 @@ module Crawlable
       @links ||= []
     end
     
-    def host(*args)
-      @host = args unless args.empty?
-      @host
+    def sitemap_host(*args)
+      @sitemap_host = args unless args.empty?
+      @sitemap_host
+    end
+    
+    def stylesheet(value = nil)
+      @stylesheet = value if value
+      @stylesheet
     end
     
     def ping(*args)
@@ -76,12 +99,12 @@ module Crawlable
     
     def link(path, *args, &block)
       options = args.extract_options!
-      options.assert_valid_keys(:priority, :changes, :updated_at, :host)
+      options.assert_valid_keys(:priority, :changes, :updated_at, :sitemap_host)
       options.reverse_merge!(
         :priority => 0.5,
         :changes => 'monthly',
         :updated_at => Time.now,
-        :host => self.host
+        :host => self.sitemap_host
       )
 
       result = {
@@ -146,7 +169,16 @@ module Crawlable
           end
         end
       end
-      builder.to_xml
+      xml = builder.to_xml
+      
+      # can't add processing instructions with nokogiri
+      xml.gsub!("<?xml version=\"1.0\"?>") do |head|
+        result = head
+        result << "\n"
+        result << "<?xml-stylesheet type=\"text/xsl\" href=\"#{stylesheet}\"?>"
+      end if stylesheet
+      
+      xml
     end
     
     def write(path, compress)
@@ -188,7 +220,7 @@ module Crawlable
     end
     
     def inspect
-      "<Sitemap @host='#{host.to_s}' @sitemap_path='#{sitemap_path.to_s}' @ping='#{ping.inspect}' @links='#{links.inspect}'/>"
+      "<Sitemap @sitemap_host='#{sitemap_host.to_s}' @sitemap_path='#{sitemap_path.to_s}' @ping='#{ping.inspect}' @links='#{links.inspect}'/>"
     end
     
   end

data/lib/engine.rb

@@ -3,6 +3,25 @@ module Crawlable
     
     initializer "authlogic_connect.authentication_hook" do |app|
       app.middleware.use Crawlable::Rack
+      add_helpers Crawlable::Sitemap, Crawlable::Feed
+    end
+    
+    def add_helpers(*clazzes)
+      clazzes.each do |clazz|
+        clazz.class_eval do
+          include ApplicationHelper
+
+          if defined?(::Rails)
+            if ActionPack::VERSION::MAJOR == 3
+              include ::Rails.application.routes.url_helpers
+            else
+              require 'action_controller'
+              include ActionController::UrlWriter
+            end
+            default_url_options[:host] = 'localhost:3000'
+          end
+        end
+      end
     end
     
   end

data/test/feed_data.atom

@@ -0,0 +1,67 @@
+<?xml version="1.0"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+  <title>My RSS Feed</title>
+  <author>
+    <name>Lance Pollard</name>
+  </author>
+  <entry>
+    <title>title-0</title>
+    <updated>2010-07-02T06:16:43Z</updated>
+    <summary>Post summary</summary>
+    <content type="html"/>
+  </entry>
+  <entry>
+    <title>title-1</title>
+    <updated>2010-07-02T06:16:43Z</updated>
+    <summary>Post summary</summary>
+    <content type="html"/>
+  </entry>
+  <entry>
+    <title>title-2</title>
+    <updated>2010-07-02T06:16:43Z</updated>
+    <summary>Post summary</summary>
+    <content type="html"/>
+  </entry>
+  <entry>
+    <title>title-3</title>
+    <updated>2010-07-02T06:16:43Z</updated>
+    <summary>Post summary</summary>
+    <content type="html"/>
+  </entry>
+  <entry>
+    <title>title-4</title>
+    <updated>2010-07-02T06:16:43Z</updated>
+    <summary>Post summary</summary>
+    <content type="html"/>
+  </entry>
+  <entry>
+    <title>title-5</title>
+    <updated>2010-07-02T06:16:43Z</updated>
+    <summary>Post summary</summary>
+    <content type="html"/>
+  </entry>
+  <entry>
+    <title>title-6</title>
+    <updated>2010-07-02T06:16:43Z</updated>
+    <summary>Post summary</summary>
+    <content type="html"/>
+  </entry>
+  <entry>
+    <title>title-7</title>
+    <updated>2010-07-02T06:16:43Z</updated>
+    <summary>Post summary</summary>
+    <content type="html"/>
+  </entry>
+  <entry>
+    <title>title-8</title>
+    <updated>2010-07-02T06:16:43Z</updated>
+    <summary>Post summary</summary>
+    <content type="html"/>
+  </entry>
+  <entry>
+    <title>title-9</title>
+    <updated>2010-07-02T06:16:43Z</updated>
+    <summary>Post summary</summary>
+    <content type="html"/>
+  </entry>
+</feed>

data/test/feed_data.rss

@@ -0,0 +1,68 @@
+<?xml version="1.0"?>
+<rss xmlns:dc="http://purl.org/dc/elements/1.1/">
+  <channel>
+    <title>My RSS Feed</title>
+    <link></link>
+    <description>Something nice and tidy</description>
+    <item>
+      <title>title-0</title>
+      <link>/posts/title-0</link>
+      <description></description>
+      <guid>/posts/title-0</guid>
+    </item>
+    <item>
+      <title>title-1</title>
+      <link>/posts/title-1</link>
+      <description></description>
+      <guid>/posts/title-1</guid>
+    </item>
+    <item>
+      <title>title-2</title>
+      <link>/posts/title-2</link>
+      <description></description>
+      <guid>/posts/title-2</guid>
+    </item>
+    <item>
+      <title>title-3</title>
+      <link>/posts/title-3</link>
+      <description></description>
+      <guid>/posts/title-3</guid>
+    </item>
+    <item>
+      <title>title-4</title>
+      <link>/posts/title-4</link>
+      <description></description>
+      <guid>/posts/title-4</guid>
+    </item>
+    <item>
+      <title>title-5</title>
+      <link>/posts/title-5</link>
+      <description></description>
+      <guid>/posts/title-5</guid>
+    </item>
+    <item>
+      <title>title-6</title>
+      <link>/posts/title-6</link>
+      <description></description>
+      <guid>/posts/title-6</guid>
+    </item>
+    <item>
+      <title>title-7</title>
+      <link>/posts/title-7</link>
+      <description></description>
+      <guid>/posts/title-7</guid>
+    </item>
+    <item>
+      <title>title-8</title>
+      <link>/posts/title-8</link>
+      <description></description>
+      <guid>/posts/title-8</guid>
+    </item>
+    <item>
+      <title>title-9</title>
+      <link>/posts/title-9</link>
+      <description></description>
+      <guid>/posts/title-9</guid>
+    </item>
+  </channel>
+</rss>

data/test/feeds.rb

@@ -0,0 +1,11 @@
+Feed do
+  posts do
+    title "My RSS Feed"
+    author "Lance Pollard"
+    description "Something nice and tidy"
+    
+    Post.all.each do |a|
+      entry "/posts/#{a.to_param}", :updated_at => a.updated_at, :title => a.title
+    end
+  end
+end

data/test/test_feed.rb

@@ -10,7 +10,8 @@ class FeedTest < ActiveSupport::TestCase
     end
     
     should "create feed" do
-      puts Crawlable::Feed.to_rss
+      puts Crawlable::Feed.process!("test/feeds.rb", "test/feed_data", :rss, :atom)
+      assert true
     end
     
   end

data/test/test_helper.rb

@@ -13,6 +13,7 @@ this = File.expand_path(File.dirname(__FILE__))
 require File.expand_path(File.join(this, '/../lib/crawlable'))
 
 Dir["#{this}/lib/*"].each { |c| require c if File.extname(c) == ".rb" }
+require "#{this}/feeds"
 require File.expand_path(File.join(File.dirname(__FILE__), '/../lib/crawlable'))
 
 ActiveRecord::Base.class_eval do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification 
 name: crawlable
 version: !ruby/object:Gem::Version 
-  hash: 69
+  hash: 91
   prerelease: false
   segments: 
   - 0
   - 0
   - 1
-  - 7
-  version: 0.0.1.7
+  - 8
+  version: 0.0.1.8
 platform: ruby
 authors: 
 - Lance Pollard
@@ -54,6 +54,9 @@ files:
 - lib/crawlable.rb
 - lib/engine.rb
 - rails/init.rb
+- test/feed_data.atom
+- test/feed_data.rss
+- test/feeds.rb
 - test/lib/_database.rb
 - test/lib/post.rb
 - test/lib/routes.rb