feedtools 0.2.8 → 0.2.9
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG +6 -0
- data/lib/database_feed_cache.rb +145 -0
- data/lib/feed_tools.rb +91 -191
- data/rakefile +2 -1
- metadata +13 -2
data/CHANGELOG
CHANGED
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
== FeedTools 0.2.9
|
|
2
|
+
* changed ordering of elements checked for the item content
|
|
3
|
+
* added dependancy on uuidtools, uuids used in feed generation
|
|
4
|
+
* database feed cache implementation factored out into its own file
|
|
5
|
+
* http_fetch changed to use lambda
|
|
6
|
+
* fixed invalid rss 1.0/2.0 generation (missing some namespaces)
|
|
1
7
|
== FeedTools 0.2.8
|
|
2
8
|
* fixed bug with http headers being loaded from invalid YAML
|
|
3
9
|
* fixed uninitialized constant bug (you shouldn't have RUBYOPT set)
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
#= database_feed_cache.rb
|
|
2
|
+
#
|
|
3
|
+
# The <tt>DatabaseFeedCache</tt> is the default caching mechanism for
|
|
4
|
+
# FeedTools. This mechanism can be replaced easily by creating another
|
|
5
|
+
# class with the required set of methods and setting
|
|
6
|
+
# <tt>FeedTools#feed_cache</tt> to the new class.
|
|
7
|
+
module FeedTools
|
|
8
|
+
# The default caching mechanism for the FeedTools module
|
|
9
|
+
class DatabaseFeedCache < ActiveRecord::Base
|
|
10
|
+
# Overrides the default table name to use the "feeds" table.
|
|
11
|
+
def self.table_name() "feeds" end
|
|
12
|
+
|
|
13
|
+
# If ActiveRecord is not already connected, attempts to find a configuration file and use
|
|
14
|
+
# it to open a connection for ActiveRecord.
|
|
15
|
+
# This method is probably unnecessary for anything but testing and debugging purposes.
|
|
16
|
+
# In a Rails environment, the connection will already have been established
|
|
17
|
+
# and this method will simply do nothing.
|
|
18
|
+
#
|
|
19
|
+
# This method should not raise any exceptions because it's designed to be run only when
|
|
20
|
+
# the module is first loaded. If it fails, the user should get an exception when they
|
|
21
|
+
# try to perform some action that makes use of the caching functionality, and not until.
|
|
22
|
+
def DatabaseFeedCache.initialize_cache
|
|
23
|
+
# Establish a connection if we don't already have one
|
|
24
|
+
begin
|
|
25
|
+
ActiveRecord::Base.connection
|
|
26
|
+
rescue
|
|
27
|
+
begin
|
|
28
|
+
possible_config_files = [
|
|
29
|
+
"./config/database.yml",
|
|
30
|
+
"../config/database.yml",
|
|
31
|
+
"./database.yml",
|
|
32
|
+
"../database.yml"
|
|
33
|
+
]
|
|
34
|
+
database_config_file = nil
|
|
35
|
+
for file in possible_config_files
|
|
36
|
+
if File.exists? file
|
|
37
|
+
database_config_file = file
|
|
38
|
+
break
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
database_config_hash = File.open(database_config_file) do |file|
|
|
42
|
+
config_hash = YAML::load(file)
|
|
43
|
+
unless config_hash[FEED_TOOLS_ENV].nil?
|
|
44
|
+
config_hash = config_hash[FEED_TOOLS_ENV]
|
|
45
|
+
end
|
|
46
|
+
config_hash
|
|
47
|
+
end
|
|
48
|
+
ActiveRecord::Base.configurations = database_config_hash
|
|
49
|
+
ActiveRecord::Base.establish_connection(database_config_hash)
|
|
50
|
+
ActiveRecord::Base.connection
|
|
51
|
+
rescue
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
# Verify that the necessary database tables are in place
|
|
55
|
+
# and if they're missing, create them
|
|
56
|
+
unless DatabaseFeedCache.table_exists?
|
|
57
|
+
DatabaseFeedCache.create_table
|
|
58
|
+
end
|
|
59
|
+
return nil
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Returns true if a connection to the database has been established and the
|
|
63
|
+
# required table structure is in place.
|
|
64
|
+
def DatabaseFeedCache.connected?
|
|
65
|
+
begin
|
|
66
|
+
ActiveRecord::Base.connection
|
|
67
|
+
return false if ActiveRecord::Base.configurations.nil?
|
|
68
|
+
return false unless DatabaseFeedCache.table_exists?
|
|
69
|
+
rescue => error
|
|
70
|
+
return false
|
|
71
|
+
end
|
|
72
|
+
return true
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# True if the appropriate database table already exists
|
|
76
|
+
def DatabaseFeedCache.table_exists?
|
|
77
|
+
begin
|
|
78
|
+
ActiveRecord::Base.connection.execute "select id, url, title, " +
|
|
79
|
+
"link, xml_data, http_headers, last_retrieved " +
|
|
80
|
+
"from #{self.table_name()} limit 1"
|
|
81
|
+
rescue ActiveRecord::StatementInvalid
|
|
82
|
+
return false
|
|
83
|
+
rescue
|
|
84
|
+
return false
|
|
85
|
+
end
|
|
86
|
+
return true
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Creates the appropriate database table
|
|
90
|
+
# Deprecated.
|
|
91
|
+
def DatabaseFeedCache.create_table
|
|
92
|
+
warn("The table creation methods will be removed in a future version " +
|
|
93
|
+
"and any reliance on this method should be removed.")
|
|
94
|
+
unless DatabaseFeedCache.table_exists?
|
|
95
|
+
feeds_mysql = <<-SQL_END
|
|
96
|
+
CREATE TABLE `#{self.table_name()}` (
|
|
97
|
+
`id` int(10) unsigned NOT NULL auto_increment,
|
|
98
|
+
`url` varchar(255) default NULL,
|
|
99
|
+
`title` varchar(255) default NULL,
|
|
100
|
+
`link` varchar(255) default NULL,
|
|
101
|
+
`xml_data` longtext default NULL,
|
|
102
|
+
`http_headers` text default NULL,
|
|
103
|
+
`last_retrieved` datetime default NULL,
|
|
104
|
+
PRIMARY KEY (`id`)
|
|
105
|
+
)
|
|
106
|
+
SQL_END
|
|
107
|
+
feeds_sqlite = <<-SQL_END
|
|
108
|
+
CREATE TABLE '#{self.table_name()}' (
|
|
109
|
+
'id' INTEGER PRIMARY KEY NOT NULL,
|
|
110
|
+
'url' VARCHAR(255) DEFAULT NULL,
|
|
111
|
+
'title' VARCHAR(255) DEFAULT NULL,
|
|
112
|
+
'link' VARCHAR(255) DEFAULT NULL,
|
|
113
|
+
'xml_data' TEXT DEFAULT NULL,
|
|
114
|
+
'http_headers' TEXT DEFAULT NULL,
|
|
115
|
+
'last_retrieved' DATETIME DEFAULT NULL
|
|
116
|
+
);
|
|
117
|
+
SQL_END
|
|
118
|
+
feeds_psql = <<-SQL_END
|
|
119
|
+
CREATE TABLE #{self.table_name()} (
|
|
120
|
+
id SERIAL PRIMARY KEY NOT NULL,
|
|
121
|
+
url varchar(255) default NULL,
|
|
122
|
+
title varchar(255) default NULL,
|
|
123
|
+
link varchar(255) default NULL,
|
|
124
|
+
xml_data text default NULL,
|
|
125
|
+
http_headers text default NULL,
|
|
126
|
+
last_retrieved timestamp default NULL
|
|
127
|
+
);
|
|
128
|
+
SQL_END
|
|
129
|
+
table_creation_sql = nil
|
|
130
|
+
if configurations["adapter"] == "mysql"
|
|
131
|
+
table_creation_sql = feeds_mysql
|
|
132
|
+
elsif configurations["adapter"] == "sqlite"
|
|
133
|
+
table_creation_sql = feeds_sqlite
|
|
134
|
+
elsif configurations["adapter"] == "postgresql"
|
|
135
|
+
table_creation_sql = feeds_psql
|
|
136
|
+
end
|
|
137
|
+
if table_creation_sql.nil?
|
|
138
|
+
raise "Could not build #{self.table_name()} table."
|
|
139
|
+
else
|
|
140
|
+
connection.execute table_creation_sql
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
end
|
data/lib/feed_tools.rb
CHANGED
|
@@ -22,19 +22,30 @@
|
|
|
22
22
|
#++
|
|
23
23
|
|
|
24
24
|
if Object.const_defined?(:FEED_TOOLS_ENV)
|
|
25
|
-
|
|
26
|
-
"by the presence of the RUBYOPT environment variable
|
|
25
|
+
warn("FeedTools may have been loaded improperly. This may be caused " +
|
|
26
|
+
"by the presence of the RUBYOPT environment variable or by using " +
|
|
27
|
+
"load instead of require. This can also be caused by missing " +
|
|
28
|
+
"the Iconv library, which is common on Windows.")
|
|
27
29
|
end
|
|
28
30
|
|
|
29
31
|
FEED_TOOLS_ENV = ENV['FEED_TOOLS_ENV'] ||
|
|
30
32
|
ENV['RAILS_ENV'] ||
|
|
31
33
|
'production' # :nodoc:
|
|
32
34
|
|
|
33
|
-
FEED_TOOLS_VERSION = "0.2.
|
|
35
|
+
FEED_TOOLS_VERSION = "0.2.9"
|
|
34
36
|
|
|
35
37
|
$:.unshift(File.dirname(__FILE__))
|
|
36
38
|
$:.unshift(File.dirname(__FILE__) + "/feed_tools/vendor")
|
|
37
|
-
|
|
39
|
+
|
|
40
|
+
begin
|
|
41
|
+
require 'iconv'
|
|
42
|
+
rescue LoadError
|
|
43
|
+
warn("The Iconv library does not appear to be installed properly. " +
|
|
44
|
+
"FeedTools cannot function properly without it.")
|
|
45
|
+
raise
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
require 'rubygems'
|
|
38
49
|
|
|
39
50
|
begin
|
|
40
51
|
require 'builder'
|
|
@@ -57,15 +68,16 @@ require 'net/ftp'
|
|
|
57
68
|
|
|
58
69
|
require 'rexml/document'
|
|
59
70
|
|
|
60
|
-
require 'iconv'
|
|
61
71
|
require 'uri'
|
|
62
72
|
require 'time'
|
|
63
73
|
require 'cgi'
|
|
64
74
|
require 'pp'
|
|
65
75
|
require 'yaml'
|
|
66
76
|
|
|
67
|
-
require 'rubygems'
|
|
68
77
|
require_gem('activerecord', '>= 1.10.1')
|
|
78
|
+
require_gem('uuidtools', '>= 0.1.1')
|
|
79
|
+
|
|
80
|
+
require 'database_feed_cache'
|
|
69
81
|
|
|
70
82
|
#= feed_tools.rb
|
|
71
83
|
#
|
|
@@ -84,141 +96,6 @@ require_gem('activerecord', '>= 1.10.1')
|
|
|
84
96
|
# => "43,37,28,23,11,3,1"
|
|
85
97
|
module FeedTools
|
|
86
98
|
|
|
87
|
-
# The default caching mechanism for the FeedTools module
|
|
88
|
-
class DatabaseFeedCache < ActiveRecord::Base
|
|
89
|
-
# Overrides the default table name to use the "feeds" table.
|
|
90
|
-
def self.table_name() "feeds" end
|
|
91
|
-
|
|
92
|
-
# If ActiveRecord is not already connected, attempts to find a configuration file and use
|
|
93
|
-
# it to open a connection for ActiveRecord.
|
|
94
|
-
# This method is probably unnecessary for anything but testing and debugging purposes.
|
|
95
|
-
# In a Rails environment, the connection will already have been established
|
|
96
|
-
# and this method will simply do nothing.
|
|
97
|
-
#
|
|
98
|
-
# This method should not raise any exceptions because it's designed to be run only when
|
|
99
|
-
# the module is first loaded. If it fails, the user should get an exception when they
|
|
100
|
-
# try to perform some action that makes use of the caching functionality, and not until.
|
|
101
|
-
def DatabaseFeedCache.initialize_cache
|
|
102
|
-
# Establish a connection if we don't already have one
|
|
103
|
-
begin
|
|
104
|
-
ActiveRecord::Base.connection
|
|
105
|
-
rescue
|
|
106
|
-
begin
|
|
107
|
-
possible_config_files = [
|
|
108
|
-
"./config/database.yml",
|
|
109
|
-
"../config/database.yml",
|
|
110
|
-
"./database.yml",
|
|
111
|
-
"../database.yml"
|
|
112
|
-
]
|
|
113
|
-
database_config_file = nil
|
|
114
|
-
for file in possible_config_files
|
|
115
|
-
if File.exists? file
|
|
116
|
-
database_config_file = file
|
|
117
|
-
break
|
|
118
|
-
end
|
|
119
|
-
end
|
|
120
|
-
database_config_hash = File.open(database_config_file) do |file|
|
|
121
|
-
config_hash = YAML::load(file)
|
|
122
|
-
unless config_hash[FEED_TOOLS_ENV].nil?
|
|
123
|
-
config_hash = config_hash[FEED_TOOLS_ENV]
|
|
124
|
-
end
|
|
125
|
-
config_hash
|
|
126
|
-
end
|
|
127
|
-
ActiveRecord::Base.configurations = database_config_hash
|
|
128
|
-
ActiveRecord::Base.establish_connection(database_config_hash)
|
|
129
|
-
ActiveRecord::Base.connection
|
|
130
|
-
rescue
|
|
131
|
-
end
|
|
132
|
-
end
|
|
133
|
-
# Verify that the necessary database tables are in place
|
|
134
|
-
# and if they're missing, create them
|
|
135
|
-
unless DatabaseFeedCache.table_exists?
|
|
136
|
-
DatabaseFeedCache.create_table
|
|
137
|
-
end
|
|
138
|
-
return nil
|
|
139
|
-
end
|
|
140
|
-
|
|
141
|
-
# Returns true if a connection to the database has been established and the
|
|
142
|
-
# required table structure is in place.
|
|
143
|
-
def DatabaseFeedCache.connected?
|
|
144
|
-
begin
|
|
145
|
-
ActiveRecord::Base.connection
|
|
146
|
-
return false if ActiveRecord::Base.configurations.nil?
|
|
147
|
-
return false unless DatabaseFeedCache.table_exists?
|
|
148
|
-
rescue => error
|
|
149
|
-
return false
|
|
150
|
-
end
|
|
151
|
-
return true
|
|
152
|
-
end
|
|
153
|
-
|
|
154
|
-
# True if the appropriate database table already exists
|
|
155
|
-
def DatabaseFeedCache.table_exists?
|
|
156
|
-
begin
|
|
157
|
-
ActiveRecord::Base.connection.execute "select id, url, title, " +
|
|
158
|
-
"link, xml_data, http_headers, last_retrieved " +
|
|
159
|
-
"from #{self.table_name()} limit 1"
|
|
160
|
-
rescue ActiveRecord::StatementInvalid
|
|
161
|
-
return false
|
|
162
|
-
rescue
|
|
163
|
-
return false
|
|
164
|
-
end
|
|
165
|
-
return true
|
|
166
|
-
end
|
|
167
|
-
|
|
168
|
-
# Creates the appropriate database table
|
|
169
|
-
def DatabaseFeedCache.create_table
|
|
170
|
-
unless DatabaseFeedCache.table_exists?
|
|
171
|
-
feeds_mysql = <<-SQL_END
|
|
172
|
-
CREATE TABLE `#{self.table_name()}` (
|
|
173
|
-
`id` int(10) unsigned NOT NULL auto_increment,
|
|
174
|
-
`url` varchar(255) default NULL,
|
|
175
|
-
`title` varchar(255) default NULL,
|
|
176
|
-
`link` varchar(255) default NULL,
|
|
177
|
-
`xml_data` longtext default NULL,
|
|
178
|
-
`http_headers` text default NULL,
|
|
179
|
-
`last_retrieved` datetime default NULL,
|
|
180
|
-
PRIMARY KEY (`id`)
|
|
181
|
-
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
|
|
182
|
-
SQL_END
|
|
183
|
-
feeds_sqlite = <<-SQL_END
|
|
184
|
-
CREATE TABLE '#{self.table_name()}' (
|
|
185
|
-
'id' INTEGER PRIMARY KEY NOT NULL,
|
|
186
|
-
'url' VARCHAR(255) DEFAULT NULL,
|
|
187
|
-
'title' VARCHAR(255) DEFAULT NULL,
|
|
188
|
-
'link' VARCHAR(255) DEFAULT NULL,
|
|
189
|
-
'xml_data' TEXT DEFAULT NULL,
|
|
190
|
-
'http_headers' TEXT DEFAULT NULL,
|
|
191
|
-
'last_retrieved' DATETIME DEFAULT NULL
|
|
192
|
-
);
|
|
193
|
-
SQL_END
|
|
194
|
-
feeds_psql = <<-SQL_END
|
|
195
|
-
CREATE TABLE #{self.table_name()} (
|
|
196
|
-
id SERIAL PRIMARY KEY NOT NULL,
|
|
197
|
-
url varchar(255) default NULL,
|
|
198
|
-
title varchar(255) default NULL,
|
|
199
|
-
link varchar(255) default NULL,
|
|
200
|
-
xml_data text default NULL,
|
|
201
|
-
http_headers text default NULL,
|
|
202
|
-
last_retrieved timestamp default NULL
|
|
203
|
-
);
|
|
204
|
-
SQL_END
|
|
205
|
-
table_creation_sql = nil
|
|
206
|
-
if configurations["adapter"] == "mysql"
|
|
207
|
-
table_creation_sql = feeds_mysql
|
|
208
|
-
elsif configurations["adapter"] == "sqlite"
|
|
209
|
-
table_creation_sql = feeds_sqlite
|
|
210
|
-
elsif configurations["adapter"] == "postgresql"
|
|
211
|
-
table_creation_sql = feeds_psql
|
|
212
|
-
end
|
|
213
|
-
if table_creation_sql.nil?
|
|
214
|
-
raise "Could not build #{self.table_name()} table."
|
|
215
|
-
else
|
|
216
|
-
connection.execute table_creation_sql
|
|
217
|
-
end
|
|
218
|
-
end
|
|
219
|
-
end
|
|
220
|
-
end
|
|
221
|
-
|
|
222
99
|
# Error raised when a feed cannot be retrieved
|
|
223
100
|
class FeedAccessError < StandardError
|
|
224
101
|
end
|
|
@@ -446,6 +323,8 @@ module FeedTools
|
|
|
446
323
|
'/usr/lib/tidy.so',
|
|
447
324
|
'C:\Program Files\Tidy\tidy.dll',
|
|
448
325
|
'C:\Tidy\tidy.dll',
|
|
326
|
+
'C:\Ruby\bin\tidy.dll',
|
|
327
|
+
'C:\Ruby\tidy.dll',
|
|
449
328
|
'/usr/local/lib',
|
|
450
329
|
'/opt/local/lib',
|
|
451
330
|
'/usr/lib'
|
|
@@ -558,6 +437,7 @@ module FeedTools
|
|
|
558
437
|
if (feed_uri.path =~ /^[\/]+/) == 0
|
|
559
438
|
feed_uri.path.gsub!(/^[\/]+/, "/")
|
|
560
439
|
end
|
|
440
|
+
feed_uri.host.downcase!
|
|
561
441
|
return feed_uri.to_s
|
|
562
442
|
rescue URI::InvalidURIError
|
|
563
443
|
return normalized_url
|
|
@@ -586,18 +466,14 @@ module FeedTools
|
|
|
586
466
|
end
|
|
587
467
|
|
|
588
468
|
# Converts a url into a urn:uuid: uri
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
# require 'uuidtools'
|
|
598
|
-
# tag_uri = UUID.sha1_create(normalized_url).to_uri_string
|
|
599
|
-
# return tag_uri
|
|
600
|
-
# end
|
|
469
|
+
def FeedTools.build_urn_uri(url)
|
|
470
|
+
unless url.kind_of? String
|
|
471
|
+
raise ArgumentError, "Expected String, got #{url.class.name}"
|
|
472
|
+
end
|
|
473
|
+
normalized_url = normalize_url(url)
|
|
474
|
+
require 'uuidtools'
|
|
475
|
+
return UUID.sha1_create(normalized_url).to_uri_string
|
|
476
|
+
end
|
|
601
477
|
|
|
602
478
|
# Returns true if the parameter appears to be a valid uri
|
|
603
479
|
def FeedTools.is_uri?(url)
|
|
@@ -896,10 +772,10 @@ module FeedTools
|
|
|
896
772
|
@http_headers = YAML.load(self.cache_object.http_headers)
|
|
897
773
|
@http_headers = {} unless @http_headers.kind_of? Hash
|
|
898
774
|
end
|
|
899
|
-
if
|
|
900
|
-
load_remote_feed!
|
|
901
|
-
else
|
|
775
|
+
if FeedTools.cache_only? || self.expired? == false
|
|
902
776
|
@live = false
|
|
777
|
+
else
|
|
778
|
+
load_remote_feed!
|
|
903
779
|
end
|
|
904
780
|
end
|
|
905
781
|
|
|
@@ -955,8 +831,8 @@ module FeedTools
|
|
|
955
831
|
FeedTools.user_agent unless FeedTools.user_agent.nil?
|
|
956
832
|
|
|
957
833
|
# The http feed access method
|
|
958
|
-
|
|
959
|
-
response_chain
|
|
834
|
+
http_fetch = lambda do |feed_url, http_headers, redirect_limit,
|
|
835
|
+
response_chain|
|
|
960
836
|
raise FeedAccessError, 'Redirect too deep' if redirect_limit == 0
|
|
961
837
|
feed_uri = nil
|
|
962
838
|
begin
|
|
@@ -995,7 +871,7 @@ module FeedTools
|
|
|
995
871
|
break
|
|
996
872
|
end
|
|
997
873
|
end
|
|
998
|
-
|
|
874
|
+
response
|
|
999
875
|
when Net::HTTPRedirection
|
|
1000
876
|
if response.code.to_i == 304
|
|
1001
877
|
response.error!
|
|
@@ -1012,7 +888,7 @@ module FeedTools
|
|
|
1012
888
|
# TODO: deal with stupid people using relative urls
|
|
1013
889
|
# in Location header
|
|
1014
890
|
# =================================================
|
|
1015
|
-
http_fetch(new_location, http_headers,
|
|
891
|
+
http_fetch.call(new_location, http_headers,
|
|
1016
892
|
redirect_limit - 1, response_chain)
|
|
1017
893
|
end
|
|
1018
894
|
else
|
|
@@ -1022,7 +898,7 @@ module FeedTools
|
|
|
1022
898
|
end
|
|
1023
899
|
|
|
1024
900
|
begin
|
|
1025
|
-
@http_response = http_fetch(self.url, headers)
|
|
901
|
+
@http_response = http_fetch.call(self.url, headers, 10, [])
|
|
1026
902
|
@http_headers = {}
|
|
1027
903
|
self.http_response.each_header do |header|
|
|
1028
904
|
self.http_headers[header.first.downcase] = header.last
|
|
@@ -1142,6 +1018,9 @@ module FeedTools
|
|
|
1142
1018
|
def xml
|
|
1143
1019
|
if @xml_doc.nil?
|
|
1144
1020
|
begin
|
|
1021
|
+
# TODO: :ignore_whitespace_nodes => :all
|
|
1022
|
+
# Add that?
|
|
1023
|
+
# ======================================
|
|
1145
1024
|
@xml_doc = Document.new(xml_data)
|
|
1146
1025
|
rescue
|
|
1147
1026
|
# Something failed, attempt to repair the xml with htree.
|
|
@@ -2446,9 +2325,17 @@ module FeedTools
|
|
|
2446
2325
|
elsif feed_type == "atom" && (version == nil || version == 0.0)
|
|
2447
2326
|
version = 0.3
|
|
2448
2327
|
end
|
|
2449
|
-
if feed_type == "rss" && (version == 0.9 || version == 1.0 ||
|
|
2328
|
+
if feed_type == "rss" && (version == 0.9 || version == 1.0 ||
|
|
2329
|
+
version == 1.1)
|
|
2450
2330
|
# RDF-based rss format
|
|
2451
|
-
return xml_builder.tag!("rdf:RDF"
|
|
2331
|
+
return xml_builder.tag!("rdf:RDF",
|
|
2332
|
+
"xmlns" => "http://purl.org/rss/1.0/",
|
|
2333
|
+
"xmlns:rdf" => "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
|
|
2334
|
+
"xmlns:dc" => "http://purl.org/dc/elements/1.1/",
|
|
2335
|
+
"xmlns:syn" => "http://purl.org/rss/1.0/modules/syndication/",
|
|
2336
|
+
"xmlns:taxo" => "http://purl.org/rss/1.0/modules/taxonomy/",
|
|
2337
|
+
"xmlns:itunes" => "http://www.itunes.com/DTDs/Podcast-1.0.dtd",
|
|
2338
|
+
"xmlns:media" => "http://search.yahoo.com/mrss") do
|
|
2452
2339
|
channel_attributes = {}
|
|
2453
2340
|
unless self.link.nil?
|
|
2454
2341
|
channel_attributes["rdf:about"] = CGI.escapeHTML(self.link)
|
|
@@ -2530,22 +2417,32 @@ module FeedTools
|
|
|
2530
2417
|
end
|
|
2531
2418
|
elsif feed_type == "rss"
|
|
2532
2419
|
# normal rss format
|
|
2533
|
-
return xml_builder.rss("version" =>
|
|
2534
|
-
|
|
2535
|
-
|
|
2536
|
-
|
|
2537
|
-
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
|
|
2541
|
-
|
|
2542
|
-
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
|
|
2548
|
-
|
|
2420
|
+
return xml_builder.rss("version" => "2.0",
|
|
2421
|
+
"xmlns:rdf" => "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
|
|
2422
|
+
"xmlns:dc" => "http://purl.org/dc/elements/1.1/",
|
|
2423
|
+
"xmlns:taxo" => "http://purl.org/rss/1.0/modules/taxonomy/",
|
|
2424
|
+
"xmlns:trackback" =>
|
|
2425
|
+
"http://madskills.com/public/xml/rss/module/trackback/",
|
|
2426
|
+
"xmlns:itunes" => "http://www.itunes.com/DTDs/Podcast-1.0.dtd",
|
|
2427
|
+
"xmlns:media" => "http://search.yahoo.com/mrss") do
|
|
2428
|
+
xml_builder.channel do
|
|
2429
|
+
unless title.nil? || title == ""
|
|
2430
|
+
xml_builder.title(title)
|
|
2431
|
+
end
|
|
2432
|
+
unless link.nil? || link == ""
|
|
2433
|
+
xml_builder.link(link)
|
|
2434
|
+
end
|
|
2435
|
+
unless description.nil? || description == ""
|
|
2436
|
+
xml_builder.description(description)
|
|
2437
|
+
end
|
|
2438
|
+
xml_builder.ttl((time_to_live / 1.minute).to_s)
|
|
2439
|
+
xml_builder.generator(
|
|
2440
|
+
"http://www.sporkmonger.com/projects/feedtools")
|
|
2441
|
+
build_xml_hook(feed_type, version, xml_builder)
|
|
2442
|
+
unless items.nil?
|
|
2443
|
+
for item in items
|
|
2444
|
+
item.build_xml(feed_type, version, xml_builder)
|
|
2445
|
+
end
|
|
2549
2446
|
end
|
|
2550
2447
|
end
|
|
2551
2448
|
end
|
|
@@ -2647,16 +2544,16 @@ module FeedTools
|
|
|
2647
2544
|
"http://www.sporkmonger.com/projects/feedtools")
|
|
2648
2545
|
if self.id != nil
|
|
2649
2546
|
unless FeedTools.is_uri? self.id
|
|
2650
|
-
|
|
2651
|
-
|
|
2652
|
-
|
|
2547
|
+
if self.link != nil
|
|
2548
|
+
xml_builder.id(FeedTools.build_urn_uri(self.link))
|
|
2549
|
+
else
|
|
2653
2550
|
raise "The unique id must be a valid URI."
|
|
2654
|
-
|
|
2551
|
+
end
|
|
2655
2552
|
else
|
|
2656
2553
|
xml_builder.id(self.id)
|
|
2657
2554
|
end
|
|
2658
|
-
|
|
2659
|
-
|
|
2555
|
+
elsif self.link != nil
|
|
2556
|
+
xml_builder.id(FeedTools.build_urn_uri(self.link))
|
|
2660
2557
|
else
|
|
2661
2558
|
raise "Cannot build feed, missing feed unique id."
|
|
2662
2559
|
end
|
|
@@ -2880,6 +2777,9 @@ module FeedTools
|
|
|
2880
2777
|
# Returns a REXML Document of the xml_data
|
|
2881
2778
|
def xml
|
|
2882
2779
|
if @xml_doc.nil?
|
|
2780
|
+
# TODO: :ignore_whitespace_nodes => :all
|
|
2781
|
+
# Add that?
|
|
2782
|
+
# ======================================
|
|
2883
2783
|
@xml_doc = Document.new(xml_data)
|
|
2884
2784
|
end
|
|
2885
2785
|
return @xml_doc
|
|
@@ -2984,10 +2884,7 @@ module FeedTools
|
|
|
2984
2884
|
if @description.nil?
|
|
2985
2885
|
unless root_node.nil?
|
|
2986
2886
|
repair_entities = false
|
|
2987
|
-
description_node = XPath.first(root_node, "
|
|
2988
|
-
if description_node.nil?
|
|
2989
|
-
description_node = XPath.first(root_node, "content:encoded")
|
|
2990
|
-
end
|
|
2887
|
+
description_node = XPath.first(root_node, "content:encoded")
|
|
2991
2888
|
if description_node.nil?
|
|
2992
2889
|
description_node = XPath.first(root_node, "content")
|
|
2993
2890
|
end
|
|
@@ -3000,6 +2897,9 @@ module FeedTools
|
|
|
3000
2897
|
if description_node.nil?
|
|
3001
2898
|
description_node = XPath.first(root_node, "body")
|
|
3002
2899
|
end
|
|
2900
|
+
if description_node.nil?
|
|
2901
|
+
description_node = XPath.first(root_node, "description")
|
|
2902
|
+
end
|
|
3003
2903
|
if description_node.nil?
|
|
3004
2904
|
description_node = XPath.first(root_node, "tagline")
|
|
3005
2905
|
end
|
|
@@ -4178,7 +4078,7 @@ module FeedTools
|
|
|
4178
4078
|
xml_builder.tag!("dc:date", time.iso8601)
|
|
4179
4079
|
end
|
|
4180
4080
|
unless tags.nil? || tags.size == 0
|
|
4181
|
-
xml_builder.tag!("
|
|
4081
|
+
xml_builder.tag!("taxo:topics") do
|
|
4182
4082
|
xml_builder.tag!("rdf:Bag") do
|
|
4183
4083
|
for tag in tags
|
|
4184
4084
|
xml_builder.tag!("rdf:li", tag)
|
|
@@ -4205,7 +4105,7 @@ module FeedTools
|
|
|
4205
4105
|
xml_builder.pubDate(time.rfc822)
|
|
4206
4106
|
end
|
|
4207
4107
|
unless tags.nil? || tags.size == 0
|
|
4208
|
-
xml_builder.tag!("
|
|
4108
|
+
xml_builder.tag!("taxo:topics") do
|
|
4209
4109
|
xml_builder.tag!("rdf:Bag") do
|
|
4210
4110
|
for tag in tags
|
|
4211
4111
|
xml_builder.tag!("rdf:li", tag)
|
data/rakefile
CHANGED
|
@@ -7,7 +7,7 @@ require 'rake/gempackagetask'
|
|
|
7
7
|
require 'rake/contrib/rubyforgepublisher'
|
|
8
8
|
|
|
9
9
|
PKG_NAME = 'feedtools'
|
|
10
|
-
PKG_VERSION = '0.2.
|
|
10
|
+
PKG_VERSION = '0.2.9'
|
|
11
11
|
PKG_FILE_NAME = "#{PKG_NAME}-#{PKG_VERSION}"
|
|
12
12
|
|
|
13
13
|
RELEASE_NAME = "REL #{PKG_VERSION}"
|
|
@@ -59,6 +59,7 @@ spec = Gem::Specification.new do |s|
|
|
|
59
59
|
end
|
|
60
60
|
|
|
61
61
|
s.add_dependency('activerecord', '>= 1.10.1')
|
|
62
|
+
s.add_dependency('uuidtools', '>= 0.1.1')
|
|
62
63
|
|
|
63
64
|
s.require_path = 'lib'
|
|
64
65
|
s.autorequire = 'feed_tools'
|
metadata
CHANGED
|
@@ -3,8 +3,8 @@ rubygems_version: 0.8.11
|
|
|
3
3
|
specification_version: 1
|
|
4
4
|
name: feedtools
|
|
5
5
|
version: !ruby/object:Gem::Version
|
|
6
|
-
version: 0.2.
|
|
7
|
-
date: 2005-09-
|
|
6
|
+
version: 0.2.9
|
|
7
|
+
date: 2005-09-25 00:00:00 -04:00
|
|
8
8
|
summary: "Parsing, generation, and caching system for xml news feeds."
|
|
9
9
|
require_paths:
|
|
10
10
|
- lib
|
|
@@ -33,6 +33,7 @@ files:
|
|
|
33
33
|
- install.rb
|
|
34
34
|
- README
|
|
35
35
|
- CHANGELOG
|
|
36
|
+
- lib/database_feed_cache.rb
|
|
36
37
|
- lib/feed_tools
|
|
37
38
|
- lib/feed_tools.rb
|
|
38
39
|
- lib/feed_tools/vendor
|
|
@@ -96,4 +97,14 @@ dependencies:
|
|
|
96
97
|
- ">="
|
|
97
98
|
- !ruby/object:Gem::Version
|
|
98
99
|
version: 1.10.1
|
|
100
|
+
version:
|
|
101
|
+
- !ruby/object:Gem::Dependency
|
|
102
|
+
name: uuidtools
|
|
103
|
+
version_requirement:
|
|
104
|
+
version_requirements: !ruby/object:Gem::Version::Requirement
|
|
105
|
+
requirements:
|
|
106
|
+
-
|
|
107
|
+
- ">="
|
|
108
|
+
- !ruby/object:Gem::Version
|
|
109
|
+
version: 0.1.1
|
|
99
110
|
version:
|