crawlable 0.0.1.5 → 0.0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.markdown +58 -6
- data/Rakefile +2 -1
- data/lib/crawlable.rb +9 -11
- data/lib/crawlable/feed.rb +168 -0
- data/lib/crawlable/rack.rb +25 -0
- data/lib/crawlable/sitemap.rb +195 -0
- data/lib/engine.rb +9 -0
- data/test/lib/_database.rb +16 -0
- data/test/lib/post.rb +5 -0
- data/test/lib/routes.rb +0 -0
- data/test/test_feed.rb +18 -0
- data/test/test_helper.rb +43 -0
- data/test/test_sitemap.rb +13 -1
- metadata +29 -7
- data/lib/crawlable/crawlable.rb +0 -12
data/README.markdown
CHANGED
@@ -8,21 +8,73 @@ Super DRY Sitemaps for Rails and Sinatra Apps (works on Heroku!)
|
|
8
8
|
|
9
9
|
sudo gem install crawlable
|
10
10
|
|
11
|
-
###
|
11
|
+
### Sitemap (`config/sitemap.rb`)
|
12
12
|
|
13
|
-
Sitemap do
|
13
|
+
Sitemap "http://www.example.com" do
|
14
|
+
link articles_path, :priority => 0.7, :changes => 'daily'
|
15
|
+
|
16
|
+
Post.all.each do |a|
|
17
|
+
link articles_path(a), :updated_at => a.updated_at do
|
18
|
+
image images_path(a.featured_image)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
#### Result
|
14
24
|
|
15
|
-
|
25
|
+
<?xml version="1.0"?>
|
26
|
+
<urlset>
|
27
|
+
<url>
|
28
|
+
<loc>/articles</loc>
|
29
|
+
<lastmod>2010-06-20T09:38:26+00:00</lastmod>
|
30
|
+
<changefreq>daily</changefreq>
|
31
|
+
<priority>0.7</priority>
|
32
|
+
</url>
|
33
|
+
<url>
|
34
|
+
<loc>/articles/title-0</loc>
|
35
|
+
<lastmod>2010-06-20T09:38:26+00:00</lastmod>
|
36
|
+
<changefreq>weekly</changefreq>
|
37
|
+
<priority>0.5</priority>
|
38
|
+
</url>
|
39
|
+
<url>
|
40
|
+
<loc>/articles/title-1</loc>
|
41
|
+
<lastmod>2010-06-20T09:38:26+00:00</lastmod>
|
42
|
+
<changefreq>weekly</changefreq>
|
43
|
+
<priority>0.5</priority>
|
44
|
+
</url>
|
45
|
+
...
|
46
|
+
</urlset>
|
47
|
+
|
48
|
+
### Feed
|
16
49
|
|
17
|
-
|
18
|
-
|
50
|
+
Feed do
|
51
|
+
title "My RSS Feed"
|
52
|
+
author "Lance Pollard"
|
53
|
+
description "Something nice and tidy"
|
54
|
+
|
55
|
+
Post.all.each do |a|
|
56
|
+
entry "/posts/#{a.to_param}", :updated_at => a.updated_at, :title => a.title
|
19
57
|
end
|
20
|
-
|
21
58
|
end
|
22
59
|
|
23
60
|
## Features
|
24
61
|
|
25
62
|
- Works on Heroku
|
63
|
+
- Pings Google, Bing, Yahoo!, and Ask whenever anything changes
|
26
64
|
|
27
65
|
## Alternatives
|
28
66
|
|
67
|
+
- [SitemapGenerator](http://github.com/kjvarga/sitemap_generator)
|
68
|
+
- [Sitemap](http://github.com/queso/sitemap)
|
69
|
+
- [BigSitemap](http://github.com/alexrabarts/big_sitemap)
|
70
|
+
- [Sitemap (diff than above)](http://github.com/flyerhzm/sitemap)
|
71
|
+
- [SitemapGenerator (diff than above)](http://github.com/christianhellsten/sitemap-generator)
|
72
|
+
- [Sitemapper](http://github.com/milk-it/sitemapper)
|
73
|
+
|
74
|
+
## Resources
|
75
|
+
|
76
|
+
- [Official Sitemap Protocol](http://sitemaps.org/protocol.php)
|
77
|
+
- [Official RSS Feed Spec](http://cyber.law.harvard.edu/rss/rss.html)
|
78
|
+
- [Sitemap Engine List](http://en.wikipedia.org/wiki/Sitemap_index)
|
79
|
+
- [Yahoo Site Explorer](http://developer.yahoo.com/search/siteexplorer/V1/updateNotification.html)
|
80
|
+
- [Comparison of Feed Aggregators](http://en.wikipedia.org/wiki/Comparison_of_feed_aggregators)
|
data/Rakefile
CHANGED
@@ -5,7 +5,7 @@ require 'rake/gempackagetask'
|
|
5
5
|
spec = Gem::Specification.new do |s|
|
6
6
|
s.name = "crawlable"
|
7
7
|
s.authors = ["Lance Pollard"]
|
8
|
-
s.version = "0.0.1.
|
8
|
+
s.version = "0.0.1.6"
|
9
9
|
s.summary = "Crawlable: Super DRY Sitemaps for Rails and Sinatra Apps"
|
10
10
|
s.homepage = "http://github.com/viatropos/crawlable"
|
11
11
|
s.email = "lancejpollard@gmail.com"
|
@@ -14,6 +14,7 @@ spec = Gem::Specification.new do |s|
|
|
14
14
|
s.rubyforge_project = "crawlable"
|
15
15
|
s.platform = Gem::Platform::RUBY
|
16
16
|
s.files = %w(README.markdown Rakefile init.rb MIT-LICENSE) + Dir["{lib,rails,test}/**/*"] - Dir["test/tmp"]
|
17
|
+
s.add_dependency("nokogiri", ">= 1.4.1")
|
17
18
|
s.require_path = "lib"
|
18
19
|
end
|
19
20
|
|
data/lib/crawlable.rb
CHANGED
@@ -1,22 +1,20 @@
|
|
1
1
|
require 'rubygems'
|
2
2
|
require 'active_support'
|
3
3
|
require 'active_record'
|
4
|
+
require 'nokogiri'
|
5
|
+
require 'open-uri'
|
4
6
|
|
5
7
|
this = File.dirname(__FILE__)
|
6
|
-
Dir["#{this}/crawlable/*"].each { |c| require c }
|
7
|
-
|
8
|
-
class Settings
|
9
|
-
include Cockpit::Configuration
|
10
|
-
end
|
8
|
+
Dir["#{this}/crawlable/*"].each { |c| require c if File.extname(c) == ".rb" }
|
11
9
|
|
12
10
|
def Sitemap(*args, &block)
|
13
|
-
Sitemap.
|
11
|
+
Crawlable::Sitemap.define!(*args, &block)
|
14
12
|
end
|
15
13
|
|
16
|
-
def
|
17
|
-
|
14
|
+
def Feed(*args, &block)
|
15
|
+
Crawlable::Feed.define!(*args, &block)
|
18
16
|
end
|
19
17
|
|
20
|
-
def
|
21
|
-
|
22
|
-
end
|
18
|
+
def Crawlable(type, *args, &block)
|
19
|
+
type.to_s.constantize(*args, &block)
|
20
|
+
end
|
@@ -0,0 +1,168 @@
|
|
1
|
+
module Crawlable
|
2
|
+
class Feed
|
3
|
+
|
4
|
+
class << self
|
5
|
+
attr_accessor :options, :call_options
|
6
|
+
|
7
|
+
def define!(*args, &block)
|
8
|
+
self.options = args.extract_options!#.merge(:run => args.shift)
|
9
|
+
|
10
|
+
instance_eval(&block) if block_given?
|
11
|
+
end
|
12
|
+
|
13
|
+
def parse!(path, options = {})
|
14
|
+
path ||= File.join(::Rails.root, 'config/initializers/feeds.rb')
|
15
|
+
self.call_options = options.symbolize_keys
|
16
|
+
eval(IO.read(path))
|
17
|
+
self.call_options = nil
|
18
|
+
end
|
19
|
+
|
20
|
+
def method_missing(meth, *args, &block)
|
21
|
+
if block_given?
|
22
|
+
if !self.call_options.blank?
|
23
|
+
if call_options.has_key?(meth.to_sym)
|
24
|
+
self.new(meth, *args, &block)
|
25
|
+
else
|
26
|
+
super(meth, *args, &block)
|
27
|
+
end
|
28
|
+
else
|
29
|
+
self.new(meth, *args, &block)
|
30
|
+
end
|
31
|
+
else
|
32
|
+
super(meth, *args, &block)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
end
|
37
|
+
|
38
|
+
if defined?(::Rails)
|
39
|
+
if ActionPack::VERSION::MAJOR == 3
|
40
|
+
include ::Rails.application.routes.url_helpers
|
41
|
+
else
|
42
|
+
require 'action_controller'
|
43
|
+
include ActionController::UrlWriter
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
attr_accessor :title, :url, :description, :master, :copyright, :updated_at
|
48
|
+
|
49
|
+
def initialize(*args, &block)
|
50
|
+
options = args.extract_options!
|
51
|
+
|
52
|
+
name = args.shift
|
53
|
+
|
54
|
+
options.each do |k, v|
|
55
|
+
self.send(k, v) if self.respond_to?(k)
|
56
|
+
end
|
57
|
+
|
58
|
+
instance_eval(&block)
|
59
|
+
end
|
60
|
+
|
61
|
+
def copyright(string = nil)
|
62
|
+
@copyright = string unless string.nil?
|
63
|
+
@copyright
|
64
|
+
end
|
65
|
+
|
66
|
+
def title(string = nil)
|
67
|
+
@title = string unless string.nil?
|
68
|
+
@title
|
69
|
+
end
|
70
|
+
|
71
|
+
def description(string = nil)
|
72
|
+
@description = string unless string.nil?
|
73
|
+
@description
|
74
|
+
end
|
75
|
+
|
76
|
+
def url(string = nil)
|
77
|
+
@url = string unless string.nil?
|
78
|
+
@url
|
79
|
+
end
|
80
|
+
|
81
|
+
def author(string = nil)
|
82
|
+
@author = string unless string.nil?
|
83
|
+
@author
|
84
|
+
end
|
85
|
+
|
86
|
+
def master(string = nil)
|
87
|
+
@master = string unless string.nil?
|
88
|
+
@master
|
89
|
+
end
|
90
|
+
|
91
|
+
def entries
|
92
|
+
@entries ||= []
|
93
|
+
end
|
94
|
+
|
95
|
+
def entry(path, *args, &block)
|
96
|
+
options = args.extract_options!
|
97
|
+
|
98
|
+
result = options.dup
|
99
|
+
result.merge!(:url => path)
|
100
|
+
|
101
|
+
self.entries.push(result)
|
102
|
+
end
|
103
|
+
|
104
|
+
def to_rss
|
105
|
+
builder = Nokogiri::XML::Builder.new do |xml|
|
106
|
+
xml.rss "xmlns:dc" => "http://purl.org/dc/elements/1.1/" do
|
107
|
+
xml.channel do
|
108
|
+
xml.title self.title
|
109
|
+
xml.link self.url
|
110
|
+
xml.description self.description
|
111
|
+
|
112
|
+
self.entries.each do |entry|
|
113
|
+
puts entry.inspect
|
114
|
+
xml.item do
|
115
|
+
xml.title entry[:title]
|
116
|
+
xml.link entry[:url]
|
117
|
+
xml.description entry[:description]
|
118
|
+
xml.guid entry[:guid] || entry[:url]
|
119
|
+
xml.author entry[:author] unless entry[:author].blank?
|
120
|
+
entry[:categories].each do |category|
|
121
|
+
xml.category category
|
122
|
+
end unless entry[:categories].blank?
|
123
|
+
xml.comments entry[:comments] unless entry[:comments].blank?
|
124
|
+
xml.enclosure(
|
125
|
+
:url => entry[:asset],
|
126
|
+
:length => IO.size(entry[:asset]).to_s,
|
127
|
+
:type => entry[:asset_type]
|
128
|
+
) unless entry[:asset].blank?
|
129
|
+
xml.webMaster self.master unless self.master.blank?
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
134
|
+
end
|
135
|
+
builder.to_xml
|
136
|
+
end
|
137
|
+
|
138
|
+
def to_atom
|
139
|
+
builder = Nokogiri::XML::Builder.new do |xml|
|
140
|
+
xml.feed "xmlns" => "http://www.w3.org/2005/Atom" do
|
141
|
+
xml.title self.title
|
142
|
+
xml.link "rel" => "self", "href" => url_for(:only_path => false, :controller => 'feeds', :action => 'atom')
|
143
|
+
xml.link "rel" => "alternate", "href" => url_for(:only_path => false, :controller => 'posts')
|
144
|
+
xml.id url_for(:only_path => false, :controller => 'posts')
|
145
|
+
xml.updated self.updated_at.strftime "%Y-%m-%dT%H:%M:%SZ" if self.updated_at
|
146
|
+
xml.author { xml.name self.author }
|
147
|
+
|
148
|
+
self.entries.each do |entries|
|
149
|
+
xml.entry do
|
150
|
+
xml.title entries.title
|
151
|
+
xml.link "rel" => "alternate", "href" => url_for(:only_path => false, :controller => 'posts', :action => 'show', :id => entries.id)
|
152
|
+
xml.id url_for(:only_path => false, :controller => 'posts', :action => 'show', :id => entries.id)
|
153
|
+
xml.updated entries.updated_at.strftime "%Y-%m-%dT%H:%M:%SZ"
|
154
|
+
xml.author { xml.name entries.author.name }
|
155
|
+
xml.summary "Post summary"
|
156
|
+
xml.content "type" => "html" do
|
157
|
+
#xml.text! render(:partial => "posts/post", :post => post)
|
158
|
+
end
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
162
|
+
end
|
163
|
+
end
|
164
|
+
builder.to_xml
|
165
|
+
end
|
166
|
+
|
167
|
+
end
|
168
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module Crawlable
|
2
|
+
class Rack
|
3
|
+
|
4
|
+
def initialize(app)
|
5
|
+
@app = app
|
6
|
+
end
|
7
|
+
|
8
|
+
def call(env)
|
9
|
+
if ENV["HEROKU"]
|
10
|
+
return sitmap if env['REQUEST_PATH'] =~ /\/sitemap\.xml/i
|
11
|
+
end
|
12
|
+
@app.call(env)
|
13
|
+
end
|
14
|
+
|
15
|
+
def sitemap
|
16
|
+
file = File.join(heroku_writable_directory, "sitemap.xml.gz")
|
17
|
+
[200, { 'Cache-Control' => 'public, max-age=86400', 'Content-Length' => File.size(file).to_s, 'Content-Type' => 'text/xml' }, IO.read(file)]
|
18
|
+
end
|
19
|
+
|
20
|
+
def heroku_writable_directory
|
21
|
+
"#{Rails.root}/tmp"
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,195 @@
|
|
1
|
+
module Crawlable
|
2
|
+
class Sitemap
|
3
|
+
|
4
|
+
class << self
|
5
|
+
attr_accessor :instance
|
6
|
+
|
7
|
+
def define!(*args, &block)
|
8
|
+
self.instance = self.new(*args, &block)
|
9
|
+
end
|
10
|
+
|
11
|
+
def parse!(path)
|
12
|
+
path ||= File.join(::Rails.root, 'config/sitemap.rb')
|
13
|
+
eval(IO.read(path))
|
14
|
+
end
|
15
|
+
|
16
|
+
def write(to, compress = false)
|
17
|
+
self.instance.write(to, compress)
|
18
|
+
end
|
19
|
+
|
20
|
+
def process(from, to, compress = false, &block)
|
21
|
+
parse!(from)
|
22
|
+
write(to, compress)
|
23
|
+
end
|
24
|
+
|
25
|
+
def inspect
|
26
|
+
self.instance.inspect
|
27
|
+
end
|
28
|
+
|
29
|
+
def to_xml
|
30
|
+
self.instance.to_xml
|
31
|
+
end
|
32
|
+
|
33
|
+
def clear
|
34
|
+
self.instance.clear
|
35
|
+
self.instance = nil
|
36
|
+
end
|
37
|
+
|
38
|
+
end
|
39
|
+
|
40
|
+
attr_accessor :links, :host, :ping, :yahoo_app_id
|
41
|
+
|
42
|
+
def initialize(*args, &block)
|
43
|
+
self.host = args.shift
|
44
|
+
raise "Please define a host: 'Sitemap 'http://my-site.com' do ..." if self.host.blank?
|
45
|
+
options = args.extract_options!
|
46
|
+
|
47
|
+
options.each do |k, v|
|
48
|
+
self.send(k, v) if self.respond_to?(k)
|
49
|
+
end
|
50
|
+
|
51
|
+
instance_eval(&block)
|
52
|
+
end
|
53
|
+
|
54
|
+
def yahoo_app_id(string = nil)
|
55
|
+
@yahoo_app_id = string unless string.nil?
|
56
|
+
@yahoo_app_id
|
57
|
+
end
|
58
|
+
|
59
|
+
def links
|
60
|
+
@links ||= []
|
61
|
+
end
|
62
|
+
|
63
|
+
def host(*args)
|
64
|
+
@host = args unless args.empty?
|
65
|
+
@host
|
66
|
+
end
|
67
|
+
|
68
|
+
def ping(*args)
|
69
|
+
@ping = args unless args.empty?
|
70
|
+
@ping
|
71
|
+
end
|
72
|
+
|
73
|
+
def sitemap_path(string = nil)
|
74
|
+
@sitemap_path = string || "public/sitemap.xml"
|
75
|
+
end
|
76
|
+
|
77
|
+
def link(path, *args, &block)
|
78
|
+
options = args.extract_options!
|
79
|
+
options.assert_valid_keys(:priority, :changes, :updated_at, :host)
|
80
|
+
options.reverse_merge!(
|
81
|
+
:priority => 0.5,
|
82
|
+
:changes => 'monthly',
|
83
|
+
:updated_at => Time.now,
|
84
|
+
:host => self.host
|
85
|
+
)
|
86
|
+
|
87
|
+
result = {
|
88
|
+
:host => options[:host],
|
89
|
+
:path => path,
|
90
|
+
:url => URI.join(options[:host], path).to_s,
|
91
|
+
:priority => options[:priority],
|
92
|
+
:changes => options[:changes],
|
93
|
+
:updated_at => options[:updated_at],
|
94
|
+
:images => []
|
95
|
+
}
|
96
|
+
|
97
|
+
self.links.push(result)
|
98
|
+
|
99
|
+
instance_eval(&block) if block_given?
|
100
|
+
|
101
|
+
result
|
102
|
+
end
|
103
|
+
|
104
|
+
def image(path, *args, &block)
|
105
|
+
options = args.extract_options!
|
106
|
+
options.assert_valid_keys(:priority, :changes, :updated_at, :host)
|
107
|
+
|
108
|
+
result = {
|
109
|
+
:path => path,
|
110
|
+
:caption => options[:caption],
|
111
|
+
:geo_location => options[:geo_location],
|
112
|
+
:title => options[:title],
|
113
|
+
:license => options[:license]
|
114
|
+
}
|
115
|
+
|
116
|
+
self.links.last[:images].push(result)
|
117
|
+
end
|
118
|
+
|
119
|
+
def w3c_date(date)
|
120
|
+
date.utc.strftime("%Y-%m-%dT%H:%M:%S+00:00")
|
121
|
+
end
|
122
|
+
|
123
|
+
def to_xml
|
124
|
+
namespaces = {
|
125
|
+
"xmlns" => "http://www.sitemaps.org/schemas/sitemap/0.9",
|
126
|
+
"xmlns:image" => "http://www.google.com/schemas/sitemap-image/1.1"
|
127
|
+
}
|
128
|
+
builder = Nokogiri::XML::Builder.new do |xml|
|
129
|
+
xml.urlset(namespaces) do
|
130
|
+
self.links.each do |link|
|
131
|
+
xml.url do
|
132
|
+
xml.loc link[:path]
|
133
|
+
xml.lastmod w3c_date(link[:updated_at]) if link[:updated_at]
|
134
|
+
xml.changefreq link[:changes] if link[:changes]
|
135
|
+
xml.priority link[:priority] if link[:priority]
|
136
|
+
link[:images].each do |image|
|
137
|
+
xml["image"].image do
|
138
|
+
xml["image"].loc image[:path]
|
139
|
+
xml["image"].caption image[:caption] if image[:caption]
|
140
|
+
xml["image"].geo_location image[:geo_location] if image[:geo_location]
|
141
|
+
xml["image"].title image[:title] if image[:title]
|
142
|
+
xml["image"].license image[:license] if image[:license]
|
143
|
+
end
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|
149
|
+
builder.to_xml
|
150
|
+
end
|
151
|
+
|
152
|
+
def write(path, compress)
|
153
|
+
to = path
|
154
|
+
if compress
|
155
|
+
to << ".gz" unless File.extname(path) == ".gz"
|
156
|
+
File.open(to, 'wb') do |file|
|
157
|
+
gz = Zlib::GzipWriter.new(file)
|
158
|
+
gz.write to_xml
|
159
|
+
gz.close
|
160
|
+
end
|
161
|
+
else
|
162
|
+
File.open(to, 'wb') do |file|
|
163
|
+
file.puts to_xml
|
164
|
+
end
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
def notify
|
169
|
+
engines = {
|
170
|
+
:google => "http://www.google.com/webmasters/sitemaps/ping?sitemap=#{path}",
|
171
|
+
:yahoo => "http://search.yahooapis.com/SiteExplorerService/V1/ping?sitemap=#{path}&appid=#{yahoo_app_id}",
|
172
|
+
:ask => "http://submissions.ask.com/ping?sitemap=#{path}",
|
173
|
+
:bing => "http://www.bing.com/webmaster/ping.aspx?siteMap=#{path}",
|
174
|
+
:sitemap_writer => "http://www.sitemapwriter.com/notify.php?crawler=all&url=#{path}"
|
175
|
+
}
|
176
|
+
engines.each do |engine, link|
|
177
|
+
begin
|
178
|
+
open(link)
|
179
|
+
puts "Successful ping of #{engine.to_s.titleize}"
|
180
|
+
rescue Timeout::Error, StandardError => e
|
181
|
+
puts "Ping failed for #{engine.to_s.titleize}: #{e.inspect}"
|
182
|
+
end
|
183
|
+
end
|
184
|
+
end
|
185
|
+
|
186
|
+
def clear
|
187
|
+
@links = nil
|
188
|
+
end
|
189
|
+
|
190
|
+
def inspect
|
191
|
+
"<Sitemap @host='#{host.to_s}' @sitemap_path='#{sitemap_path.to_s}' @ping='#{ping.inspect}' @links='#{links.inspect}'/>"
|
192
|
+
end
|
193
|
+
|
194
|
+
end
|
195
|
+
end
|
data/lib/engine.rb
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
begin
|
2
|
+
ActiveRecord::Base.establish_connection(:adapter => "sqlite3", :database => ":memory:")
|
3
|
+
rescue ArgumentError
|
4
|
+
ActiveRecord::Base.establish_connection(:adapter => "sqlite3", :dbfile => ":memory:")
|
5
|
+
end
|
6
|
+
|
7
|
+
ActiveRecord::Base.configurations = true
|
8
|
+
|
9
|
+
ActiveRecord::Schema.define(:version => 1) do
|
10
|
+
|
11
|
+
create_table :posts, :force => true do |t|
|
12
|
+
t.string :title
|
13
|
+
t.timestamps
|
14
|
+
end
|
15
|
+
|
16
|
+
end
|
data/test/lib/post.rb
ADDED
data/test/lib/routes.rb
ADDED
File without changes
|
data/test/test_feed.rb
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), 'test_helper')
|
2
|
+
|
3
|
+
class FeedTest < ActiveSupport::TestCase
|
4
|
+
|
5
|
+
context "Feed" do
|
6
|
+
|
7
|
+
setup do
|
8
|
+
create_posts
|
9
|
+
load_feed
|
10
|
+
end
|
11
|
+
|
12
|
+
should "create feed" do
|
13
|
+
puts Crawlable::Feed.to_rss
|
14
|
+
end
|
15
|
+
|
16
|
+
end
|
17
|
+
|
18
|
+
end
|
data/test/test_helper.rb
CHANGED
@@ -9,6 +9,10 @@ require 'active_record/fixtures'
|
|
9
9
|
require 'shoulda'
|
10
10
|
require 'shoulda/active_record'
|
11
11
|
|
12
|
+
this = File.expand_path(File.dirname(__FILE__))
|
13
|
+
require File.expand_path(File.join(this, '/../lib/crawlable'))
|
14
|
+
|
15
|
+
Dir["#{this}/lib/*"].each { |c| require c if File.extname(c) == ".rb" }
|
12
16
|
require File.expand_path(File.join(File.dirname(__FILE__), '/../lib/crawlable'))
|
13
17
|
|
14
18
|
ActiveRecord::Base.class_eval do
|
@@ -16,3 +20,42 @@ ActiveRecord::Base.class_eval do
|
|
16
20
|
all.map(&:destroy)
|
17
21
|
end
|
18
22
|
end
|
23
|
+
|
24
|
+
ActiveSupport::TestCase.class_eval do
|
25
|
+
|
26
|
+
def create_posts(many = 10)
|
27
|
+
many.times do |i|
|
28
|
+
Post.create!(:title => "title-#{i.to_s}")
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def load_sitemap
|
33
|
+
Sitemap "http://www.example.com" do
|
34
|
+
ping :yahoo, :google
|
35
|
+
|
36
|
+
link "/posts", :priority => 0.7, :changes => 'daily' do
|
37
|
+
3.times do |i|
|
38
|
+
image "/images/#{i}"
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
Post.all.each do |a|
|
43
|
+
link "/posts/#{a.to_param}", :updated_at => a.updated_at
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def load_feed
|
49
|
+
Feed do
|
50
|
+
posts do
|
51
|
+
title "My RSS Feed"
|
52
|
+
author "Lance Pollard"
|
53
|
+
description "Something nice and tidy"
|
54
|
+
|
55
|
+
Post.all.each do |a|
|
56
|
+
entry "/posts/#{a.to_param}", :updated_at => a.updated_at, :title => a.title
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
data/test/test_sitemap.rb
CHANGED
@@ -4,8 +4,20 @@ class SitemapTest < ActiveSupport::TestCase
|
|
4
4
|
|
5
5
|
context "Sitemap" do
|
6
6
|
|
7
|
+
setup do
|
8
|
+
create_posts
|
9
|
+
load_sitemap
|
10
|
+
end
|
7
11
|
|
12
|
+
should "create sitemap" do
|
13
|
+
Crawlable::Sitemap.write("test/sitemap.xml")
|
14
|
+
assert File.exists?("test/sitemap.xml.gz")
|
15
|
+
end
|
16
|
+
|
17
|
+
teardown do
|
18
|
+
File.delete("test/sitemap.xml.gz") if File.exists?("test/sitemap.xml.gz")
|
19
|
+
end
|
8
20
|
|
9
21
|
end
|
10
22
|
|
11
|
-
end
|
23
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: crawlable
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 71
|
5
5
|
prerelease: false
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 0
|
9
9
|
- 1
|
10
|
-
-
|
11
|
-
version: 0.0.1.
|
10
|
+
- 6
|
11
|
+
version: 0.0.1.6
|
12
12
|
platform: ruby
|
13
13
|
authors:
|
14
14
|
- Lance Pollard
|
@@ -16,10 +16,25 @@ autorequire:
|
|
16
16
|
bindir: bin
|
17
17
|
cert_chain: []
|
18
18
|
|
19
|
-
date: 2010-
|
19
|
+
date: 2010-07-02 00:00:00 -07:00
|
20
20
|
default_executable:
|
21
|
-
dependencies:
|
22
|
-
|
21
|
+
dependencies:
|
22
|
+
- !ruby/object:Gem::Dependency
|
23
|
+
name: nokogiri
|
24
|
+
prerelease: false
|
25
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
26
|
+
none: false
|
27
|
+
requirements:
|
28
|
+
- - ">="
|
29
|
+
- !ruby/object:Gem::Version
|
30
|
+
hash: 5
|
31
|
+
segments:
|
32
|
+
- 1
|
33
|
+
- 4
|
34
|
+
- 1
|
35
|
+
version: 1.4.1
|
36
|
+
type: :runtime
|
37
|
+
version_requirements: *id001
|
23
38
|
description: Super DRY Sitemaps for Rails and Sinatra Apps
|
24
39
|
email: lancejpollard@gmail.com
|
25
40
|
executables: []
|
@@ -33,9 +48,16 @@ files:
|
|
33
48
|
- Rakefile
|
34
49
|
- init.rb
|
35
50
|
- MIT-LICENSE
|
36
|
-
- lib/crawlable/
|
51
|
+
- lib/crawlable/feed.rb
|
52
|
+
- lib/crawlable/rack.rb
|
53
|
+
- lib/crawlable/sitemap.rb
|
37
54
|
- lib/crawlable.rb
|
55
|
+
- lib/engine.rb
|
38
56
|
- rails/init.rb
|
57
|
+
- test/lib/_database.rb
|
58
|
+
- test/lib/post.rb
|
59
|
+
- test/lib/routes.rb
|
60
|
+
- test/test_feed.rb
|
39
61
|
- test/test_helper.rb
|
40
62
|
- test/test_sitemap.rb
|
41
63
|
has_rdoc: true
|